From 83de3ebd04bc55446d768eca7c0384e29fe7f931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9A=93?= Date: Mon, 25 Dec 2023 16:14:51 -0500 Subject: [PATCH] Archive manager (#80) * feat: start creating the UI for the archive manager * refactor(filesystem): add `luminol_core::spawn_future` for cross-platform spawning * chore(filesystem): fix redundant closures * refactor(project manager): move `frame` into `update_state` * refactor(filesystem): don't store in IndexedDB unless requested In web builds, we usually don't need to store folder handles in IndexedDB, so the old system of storing folder handles in IndexedDB by default and deleting them from IndexedDB if requested was clunky. This changes it so that folder handles are not stored by default. You need to call `.save_to_idb()` to save it and get the key to restore it. * feat(archive manager): start implementing a filesystem view * fix(archive manager): fix archive contents not displaying correctly * fix(archive manager): fix some UI oddities * fix(archive manager): don't persist collapsing header state We don't want the state of the collapsing headers in the filesystem view to be saved because we may be opening a different archive with the same name, or we may be opening a modified version of the same archive. That would lead to some errors when opening the view. * fix(archive manager): depersist the inner subtrees as well * style(archive manager): future proofing and use generic IDs This changes the filesystem view to use the indextree node IDs to generate the egui IDs for the collapsing headers instead of using the file paths to generate the egui IDs. This is so that we leave less junk objects in the memory if the user opens a lot of different archives with different names. Also, the response tuple for the collapsing headers is now read using pattern matching instead of by just indexing into the tuple, which makes it easier to debug errors if egui decides to change the format of the response tuple. * fix(archive manager): don't load folder contents until partly open * feat(archive manager): implement filesystem view file/folder selection * perf(archive manager): store `depersisted` in nodes, not hashset I may be stupid. * feat(archive manager): add stripes to filesystem view rows * feat(archive manager): implement `SelectedIter` This iterator allows iterating over the selected entries of a `FileSystemView` from top to bottom. * perf(filesystem): `create_dir` now creates missing parents This prevents us from having to check if a directory exists before creating it as well as simplifying the web filesystem implementation and making the `create_dir` function more consistent with `remove_dir`. * feat(archive manager): implement archive extraction * perf(archive manager): don't recurse into deselected dirs * feat(filesystem): implement file picker for saving files * refactor(filesystem): implement `File` for `&mut impl File` * refactor(filesystem): use builder pattern to save files * refactor(filesystem): make saving files less of a hassle * fix: archiver filesystem `create_dir` is now recursive I changed all of the other filesystems' `create_dir` implementations to be recursive, so this one needs to be recursive as well. * fix(filesystem): fix archiver not being able to create files * fix(filesystem): fix edge cases in trie.rs * fix(filesystem): fix archiver file creation using wrong magic value * feat(archive manager): implement archive creation * fix(filesystem): fix more edge cases in `get_dir_prefix` It would appear I need to do more fuzz testing. * fix(filesystem): fix magic calculation for RGSS3A archives * perf: improve archive creation speed The secret sauce is to buffer writes and not flush the archive until after all files have been written. * fix(filesystem): fix trie.rs directory merge algorithm This is the last edge case, I promise! * feat(archive manager): use better sorting in filesystem view * fix(archive manager): disable the UI when file/folder picker is active * fix(archive manager): use correct file extension when creating archives * fix(archive manager): use better error handling * feat(archive manager): implement Control and Shift for filesystem view You can now use the Control and Shift keys like how they normally work in file managers. * perf(archive manager): remove redundant stack from previous commit * feat(archive manager): shift-selection now deselects if pivot is deselected * feat(archive manager): select "Data" and "Graphics" when creating archives * feat(archive manager): load from current project by default * chore: clippy * fix(archive manager): `is_root` should be set to false for inner calls * fix(filesystem): fix archiver path separator bug on Windows * fix(filesystem): fix `FileSystemTrieDirIter` `ExactSizeIterator` implementation * fix(filesystem): fix trie.rs `create_dir` edge case I lied. THIS is the last edge case, for real this time! This fixes a bug where the trie registers the root directory as a child of itself if the user calls `create_dir` on the trie with any path and then calls `create_dir` on the trie a second time with the empty path. * feat(filesystem): implement asynchronous IO for host files Host files now implement `AsyncRead + AsyncWrite + AsyncSeek + Unpin` so that we can perform IO asynchronously. Native files do this by spawning a thread and performing IO there (using the `async-fs` crate). Web files do this by just asynchronously waiting for command responses instead of blocking. * feat(archive manager): archives are now created asynchronously * fix(filesystem): web file futures now clear on error * feat(archive manager): archives are now extracted asynchronously * feat(archive manager): add progress bars for extraction and creation * fix(filesystem): fix trie.rs `remove_dir` not removing entry from parent dir * style(filesystem): remove unnecessary `vec!` from `read_file_xor_async` * style(filesystem): `get_mut_file` -> `get_file_mut` * fix: fix app being drawn every frame on native * fix(archive manager): request repaint when progress bar updates * style(filesystem): remove poll-promise from filesystem dependencies * style(filesystem): fix formatting of imports in archiver/filesystem.rs * feat(archive manager): add tooltip explaining how to select files/folders * style(filesystem): fix formatting in archiver/filesystem.rs again --- Cargo.lock | 170 ++++- Cargo.toml | 4 + crates/audio/src/lib.rs | 10 +- crates/components/Cargo.toml | 4 + crates/components/src/filesystem_view.rs | 734 +++++++++++++++++++ crates/components/src/lib.rs | 3 + crates/core/src/lib.rs | 14 +- crates/core/src/project_manager.rs | 42 +- crates/filesystem/Cargo.toml | 11 +- crates/filesystem/src/archiver/file.rs | 61 +- crates/filesystem/src/archiver/filesystem.rs | 231 +++++- crates/filesystem/src/archiver/util.rs | 46 +- crates/filesystem/src/erased.rs | 1 + crates/filesystem/src/lib.rs | 19 +- crates/filesystem/src/native.rs | 204 +++++- crates/filesystem/src/project.rs | 27 +- crates/filesystem/src/trie.rs | 50 +- crates/filesystem/src/web/events.rs | 164 +++-- crates/filesystem/src/web/mod.rs | 235 +++++- crates/filesystem/src/web/util.rs | 26 +- crates/ui/Cargo.toml | 8 + crates/ui/src/windows/archive_manager.rs | 581 +++++++++++++++ crates/ui/src/windows/mod.rs | 2 + crates/ui/src/windows/new_project.rs | 43 +- crates/web/js/bindings.js | 10 + crates/web/src/bindings.rs | 19 + src/app/top_bar.rs | 26 +- 27 files changed, 2494 insertions(+), 251 deletions(-) create mode 100644 crates/components/src/filesystem_view.rs create mode 100644 crates/ui/src/windows/archive_manager.rs diff --git a/Cargo.lock b/Cargo.lock index 65c12cbb..2e6eb268 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,6 +228,12 @@ dependencies = [ "libc", ] +[[package]] +name = "any_ascii" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e" + [[package]] name = "anyhow" version = "1.0.75" @@ -332,6 +338,32 @@ dependencies = [ "futures-lite 1.13.0", ] +[[package]] +name = "async-fs" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1f344136bad34df1f83a47f3fd7f2ab85d75cb8a940af4ccf6d482a84ea01b" +dependencies = [ + "async-lock 3.2.0", + "blocking", + "futures-lite 2.1.0", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.1.1", + "async-executor", + "async-io 2.2.1", + "async-lock 3.2.0", + "blocking", + "futures-lite 2.1.0", + "once_cell", +] + [[package]] name = "async-io" version = "1.13.0" @@ -443,6 +475,32 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io 1.13.0", + "async-lock 2.8.0", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite 1.13.0", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + [[package]] name = "async-task" version = "4.5.0" @@ -460,6 +518,16 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "async_io_stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" +dependencies = [ + "futures", + "rustc_version", +] + [[package]] name = "atk-sys" version = "0.18.0" @@ -1823,9 +1891,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -2074,6 +2142,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "glow" version = "0.12.3" @@ -2515,6 +2595,12 @@ dependencies = [ "serde", ] +[[package]] +name = "indextree" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c40411d0e5c63ef1323c3d09ce5ec6d84d71531e18daed0743fccea279d7deb6" + [[package]] name = "instant" version = "0.1.12" @@ -2677,6 +2763,15 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lab" version = "0.11.0" @@ -2712,6 +2807,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "lexical-sort" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c09e4591611e231daf4d4c685a66cb0410cc1e502027a20ae55f2bb9e997207a" +dependencies = [ + "any_ascii", +] + [[package]] name = "libc" version = "0.2.150" @@ -2814,6 +2918,9 @@ name = "log" version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +dependencies = [ + "value-bag", +] [[package]] name = "loom" @@ -2907,7 +3014,9 @@ dependencies = [ "anyhow", "egui", "glam", + "indextree", "itertools", + "lexical-sort", "luminol-audio", "luminol-config", "luminol-core", @@ -2917,6 +3026,7 @@ dependencies = [ "luminol-graphics", "once_cell", "parking_lot", + "qp-trie", "serde", "slab", "strum", @@ -3040,11 +3150,15 @@ dependencies = [ name = "luminol-filesystem" version = "0.4.0" dependencies = [ + "async-fs 2.1.0", + "async-std", + "async_io_stream", "bitflags 2.4.1", "camino", "dashmap", "egui", "flume", + "futures-lite 2.1.0", "indexed_db_futures", "iter-read", "itertools", @@ -3054,6 +3168,7 @@ dependencies = [ "once_cell", "oneshot", "parking_lot", + "pin-project", "qp-trie", "rand", "rfd", @@ -3125,10 +3240,13 @@ name = "luminol-ui" version = "0.4.0" dependencies = [ "anyhow", + "async-std", + "camino", "catppuccin-egui", "egui", "egui_extras", "futures", + "futures-lite 2.1.0", "git-version", "itertools", "luminol-audio", @@ -3140,7 +3258,10 @@ dependencies = [ "luminol-graphics", "luminol-modals", "luminol-term", + "once_cell", + "pin-project", "poll-promise", + "qp-trie", "reqwest", "strum", "zip", @@ -4105,6 +4226,26 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -4663,6 +4804,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.20", +] + [[package]] name = "rustix" version = "0.37.27" @@ -4789,6 +4939,12 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" + [[package]] name = "semver-parser" version = "0.10.2" @@ -5389,7 +5545,7 @@ dependencies = [ "pest_derive", "phf 0.10.1", "regex", - "semver", + "semver 0.11.0", "sha2 0.9.9", "signal-hook", "siphasher", @@ -5907,6 +6063,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "value-bag" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" + [[package]] name = "vcpkg" version = "0.2.15" @@ -6822,7 +6984,7 @@ checksum = "31de390a2d872e4cd04edd71b425e29853f786dc99317ed72d73d6fcf5ebb948" dependencies = [ "async-broadcast", "async-executor", - "async-fs", + "async-fs 1.6.0", "async-io 1.13.0", "async-lock 2.8.0", "async-process", diff --git a/Cargo.toml b/Cargo.toml index 1ed2b573..098e9874 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,12 +111,16 @@ crossbeam = "0.8.2" dashmap = "5.5.3" flume = "0.11.0" oneshot = "0.1.6" +futures-lite = "2.1.0" +async-std = "1.12.0" +pin-project = "1" poll-promise = { version = "0.3.0" } camino = "1.1.6" slab = { version = "0.4.9", features = ["serde"] } +qp-trie = "0.8.2" itertools = "0.11.0" diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index 0c3639ac..5e75ee4e 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -86,14 +86,18 @@ impl Default for Audio { impl Audio { /// Play a sound on a source. - pub fn play( + pub fn play( &self, path: impl AsRef, - filesystem: &impl luminol_filesystem::FileSystem, // FIXME + filesystem: &T, volume: u8, pitch: u8, source: Source, - ) -> Result<()> { + ) -> Result<()> + where + T: luminol_filesystem::FileSystem, + T::File: 'static, + { let path = path.as_ref(); let file = filesystem.open_file(path, luminol_filesystem::OpenFlags::Read)?; diff --git a/crates/components/Cargo.toml b/crates/components/Cargo.toml index 166db9fc..7280f676 100644 --- a/crates/components/Cargo.toml +++ b/crates/components/Cargo.toml @@ -44,5 +44,9 @@ serde.workspace = true once_cell.workspace = true slab.workspace = true +qp-trie.workspace = true anyhow.workspace = true + +indextree = "4.6.0" +lexical-sort = "0.3.1" diff --git a/crates/components/src/filesystem_view.rs b/crates/components/src/filesystem_view.rs new file mode 100644 index 00000000..8e8655b4 --- /dev/null +++ b/crates/components/src/filesystem_view.rs @@ -0,0 +1,734 @@ +// Copyright (C) 2023 Lily Lyons +// +// This file is part of Luminol. +// +// Luminol is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Luminol is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Luminol. If not, see . +// +// Additional permission under GNU GPL version 3 section 7 +// +// If you modify this Program, or any covered work, by linking or combining +// it with Steamworks API by Valve Corporation, containing parts covered by +// terms of the Steamworks API by Valve Corporation, the licensors of this +// Program grant you additional permission to convey the resulting work. + +use itertools::Itertools; + +pub struct FileSystemView { + arena: indextree::Arena, + id: egui::Id, + filesystem: T, + root_name: String, + root_node_id: indextree::NodeId, + row_index: usize, + pivot_id: Option, + pivot_visited: bool, + show_tooltip: bool, +} + +#[derive(Debug)] +enum Entry { + File { + /// Name of this file with extension. + name: String, + /// Whether or not this file is selected in the filesystem view. + selected: bool, + }, + Dir { + /// Name of this directory. + name: String, + /// Whether or not we've cached the contents of this directory. + initialized: bool, + /// Whether or not the memory for this subtree's collapsing header has been deleted. + depersisted: bool, + /// Whether or not this directory is fully selected in the filesystem view. + selected: bool, + /// Whether or not the subtree for this directory is expanded. + expanded: bool, + /// Number of files and directories that are subentries of this one. Only includes direct + /// children, not indirect descendants. + total_children: usize, + /// Number of file and directories that are subentries of this one and are (fully) + /// selected. Only includes direct children, not indirect descendants. + selected_children: usize, + /// Number of subdirectories that are partially selected. Only includes direct children, + /// not indirect descendants. + partial_children: usize, + }, +} + +impl Entry { + fn name(&self) -> &str { + match self { + Entry::File { name, .. } => name, + Entry::Dir { name, .. } => name, + } + } + + fn selected(&self) -> bool { + match self { + Entry::File { selected, .. } => *selected, + Entry::Dir { selected, .. } => *selected, + } + } +} + +impl FileSystemView +where + T: luminol_filesystem::FileSystem, +{ + pub fn new(id: egui::Id, filesystem: T, root_name: String) -> Self { + let mut arena = indextree::Arena::new(); + let root_node_id = arena.new_node(Entry::Dir { + name: "".to_string(), + initialized: false, + depersisted: false, + selected: false, + expanded: true, + total_children: 0, + selected_children: 0, + partial_children: 0, + }); + Self { + arena, + id, + filesystem, + root_name, + root_node_id, + row_index: 0, + pivot_id: None, + pivot_visited: false, + show_tooltip: true, + } + } + + pub fn filesystem(&self) -> &T { + &self.filesystem + } + + pub fn root_name(&self) -> &str { + &self.root_name + } + + /// Returns an iterator over the selected entries in this view from top to bottom. + /// + /// The iterator does not recurse into directories that are completely selected - that is, if a + /// directory is completely selected, then this iterator will iterate over the directory but + /// none of its contents. + pub fn iter(&self) -> <&Self as IntoIterator>::IntoIter { + self.into_iter() + } + + pub fn ui( + &mut self, + ui: &mut egui::Ui, + update_state: &mut luminol_core::UpdateState<'_>, + default_selected_dirs: Option<&qp_trie::Trie>, + ) { + self.row_index = 0; + self.pivot_visited = false; + + let response = egui::Frame::none().show(ui, |ui| { + self.render_subtree( + ui, + update_state, + self.root_node_id, + &self.root_name.to_string(), + default_selected_dirs, + true, + ); + }); + + if self.show_tooltip { + response.response.on_hover_ui_at_pointer(|ui| { + ui.label("Click to select single entries"); + ui.label("Ctrl+click to select multiple entries or deselect entries"); + ui.label("Shift+click to select a range"); + ui.label("To select multiple ranges or deselect a range, Ctrl+click the first endpoint and Ctrl+Shift+click the second endpoint"); + }); + } + } + + fn render_subtree( + &mut self, + ui: &mut egui::Ui, + update_state: &mut luminol_core::UpdateState<'_>, + node_id: indextree::NodeId, + name: &str, + default_selected_dirs: Option<&qp_trie::Trie>, + is_root: bool, + ) { + let is_command_held = ui.input(|i| i.modifiers.command); + let is_shift_held = ui.input(|i| i.modifiers.shift); + let mut length = None; + + if let Entry::Dir { + initialized: initialized @ false, + selected, + expanded: true, + .. + } = self.arena[node_id].get_mut() + { + let selected = *selected; + *initialized = true; + + let mut ancestors = node_id + .ancestors(&self.arena) + .filter_map(|n| { + let name = self.arena[n].get().name(); + (!name.is_empty()).then_some(name) + }) + .collect_vec(); + ancestors.reverse(); + let path = ancestors.join("/"); + + let mut subentries = self.filesystem.read_dir(path).unwrap_or_else(|e| { + update_state.toasts.error(e.to_string()); + Vec::new() + }); + subentries.sort_unstable_by(|a, b| { + if a.metadata.is_file && !b.metadata.is_file { + std::cmp::Ordering::Greater + } else if b.metadata.is_file && !a.metadata.is_file { + std::cmp::Ordering::Less + } else { + let path_a = a.path.iter().next_back().unwrap(); + let path_b = b.path.iter().next_back().unwrap(); + lexical_sort::natural_lexical_cmp(path_a, path_b) + } + }); + length = Some(subentries.len()); + + for subentry in subentries { + let subentry_name = subentry.path.iter().next_back().unwrap().to_string(); + if subentry.metadata.is_file { + node_id.append_value( + Entry::File { + name: subentry_name, + selected, + }, + &mut self.arena, + ); + } else { + let should_select = is_root + && default_selected_dirs + .is_some_and(|dirs| dirs.contains_key_str(&subentry_name)); + let child_id = node_id.append_value( + Entry::Dir { + name: subentry_name, + selected, + initialized: false, + depersisted: false, + expanded: false, + total_children: 0, + selected_children: 0, + partial_children: 0, + }, + &mut self.arena, + ); + if should_select { + self.select(child_id); + } + } + } + } + + if let Some(length) = length { + if let Entry::Dir { + selected, + total_children, + selected_children, + .. + } = self.arena[node_id].get_mut() + { + *total_children = length; + if *selected { + *selected_children = length; + } + } + } + + let mut should_toggle = false; + + let mut frame = egui::containers::Frame::none(); + if self.row_index % 2 != 0 { + frame = frame.fill(ui.visuals().faint_bg_color); + } + self.row_index += 1; + + let mut header_response = None; + + match self.arena[node_id].get_mut() { + Entry::File { name, selected } => { + frame.show(ui, |ui| { + if ui + .add(egui::SelectableLabel::new(*selected, name.to_string())) + .clicked() + { + should_toggle = true; + }; + }); + } + Entry::Dir { + depersisted, + selected, + expanded, + selected_children, + partial_children, + .. + } => { + let id = self.id.with(node_id); + + // De-persist state of the collapsing headers since the underlying filesystem may + // have changed since this view was last used + if !*depersisted { + *depersisted = true; + if let Some(h) = egui::collapsing_header::CollapsingState::load(ui.ctx(), id) { + h.remove(ui.ctx()) + } + ui.ctx().animate_bool_with_time(id, *expanded, 0.); + } + + let header = egui::collapsing_header::CollapsingState::load_with_default_open( + ui.ctx(), + id, + *expanded, + ); + + *expanded = header.openness(ui.ctx()) >= 0.2; + + let layout = *ui.layout(); + header_response = Some(header.show_header(ui, |ui| { + ui.with_layout(layout, |ui| { + frame.show(ui, |ui| { + if ui + .add(egui::SelectableLabel::new( + *selected, + format!( + "{} {}", + if *selected { + '▣' + } else if *selected_children == 0 && *partial_children == 0 + { + '☐' + } else { + '⊟' + }, + name + ), + )) + .clicked() + { + should_toggle = true; + }; + }); + }); + })); + } + } + + if should_toggle { + self.show_tooltip = false; + + let is_pivot_selected = !is_command_held + || self + .pivot_id + .is_some_and(|pivot_id| self.arena[pivot_id].get().selected()); + + // Unless control is held, deselect all the nodes before doing anything + if !is_command_held { + self.deselect(self.root_node_id); + } + + // Select all the nodes between this one and the pivot node if shift is held and the + // pivot node is selected, or deselect them if the pivot node is deselected + if is_shift_held && self.pivot_id.is_some() { + let pivot_id = *self.pivot_id.as_ref().unwrap(); + let (starting_id, ending_id) = if self.pivot_visited { + (pivot_id, node_id) + } else { + (node_id, pivot_id) + }; + let mut edge = indextree::NodeEdge::Start(starting_id); + + loop { + match edge { + indextree::NodeEdge::Start(node_id) => { + let entry = self.arena[node_id].get(); + let first_child_id = node_id.children(&self.arena).next(); + edge = if let Some(first_child_id) = first_child_id { + indextree::NodeEdge::Start(first_child_id) + } else { + indextree::NodeEdge::End(node_id) + }; + + if node_id == starting_id + || matches!( + entry, + Entry::File { .. } + | Entry::Dir { + total_children: 0, + .. + } + ) + { + if is_pivot_selected { + self.select(node_id); + } else { + self.deselect(node_id); + } + } + } + + indextree::NodeEdge::End(node_id) => { + let next_sibling_id = node_id.following_siblings(&self.arena).nth(1); + + if let Some(next_sibling_id) = next_sibling_id { + edge = indextree::NodeEdge::Start(next_sibling_id) + } else if let Some(parent_id) = node_id.ancestors(&self.arena).nth(1) { + edge = indextree::NodeEdge::End(parent_id); + } else { + break; + } + + if node_id == ending_id { + break; + } + } + } + } + } else { + self.pivot_id = Some(node_id); + self.toggle(node_id); + } + } + + if self.pivot_id.is_some_and(|pivot_id| pivot_id == node_id) { + self.pivot_visited = true; + } + + // Draw the contents of the collapsing subtree if this node is a directory + if let Some(header_response) = header_response { + header_response.body(|ui| { + for node_id in node_id.children(&self.arena).collect_vec() { + self.render_subtree( + ui, + update_state, + node_id, + self.arena[node_id].get().name().to_string().as_str(), + default_selected_dirs, + false, + ); + } + }); + } + } + + fn toggle(&mut self, node_id: indextree::NodeId) { + match self.arena[node_id].get() { + Entry::File { selected, .. } => { + if *selected { + self.deselect(node_id) + } else { + self.select(node_id) + } + } + Entry::Dir { selected, .. } => { + if *selected { + self.deselect(node_id) + } else { + self.select(node_id) + } + } + } + } + + /// Marks the given node as (completely) selected. Also marks all descendant nodes as selected + /// and updates ancestor nodes correspondingly. + /// + /// When run m times in a row (without running `deselect`) on arbitrary nodes in a tree with n + /// nodes, this takes worst case O(m + n) time thanks to memoization. + fn select(&mut self, node_id: indextree::NodeId) { + // We can skip nodes that are marked as selected because they're guaranteed to have all of + // their subentries selected as well + if matches!(self.arena[node_id].get(), Entry::Dir { selected: true, .. }) { + return; + } + + // Select all of this node's descendants in a postorder traversal + for node_id in node_id.children(&self.arena).collect_vec() { + self.select(node_id); + } + + let mut child_is_selected = true; + let mut child_was_partial = false; + + // Select this node + match self.arena[node_id].get_mut() { + Entry::File { selected, .. } => { + if *selected { + return; + } + *selected = true; + } + Entry::Dir { + selected, + total_children, + selected_children, + partial_children, + .. + } => { + if *selected { + return; + } + *selected = true; + child_was_partial = *selected_children != 0 || *partial_children != 0; + *selected_children = *total_children; + *partial_children = 0; + } + } + + // Visit and update ancestor nodes until we either reach the root node or we reach an + // ancestor that does not change state (not selected / completely selected / partially + // selected) after updating it (that implies that the ancestors of *that* node will also + // not change state after updating, so visiting them would be redundant) + for node_id in node_id.ancestors(&self.arena).skip(1).collect_vec() { + if let Entry::Dir { + selected, + total_children, + selected_children, + partial_children, + .. + } = self.arena[node_id].get_mut() + { + let was_partial = *selected_children != 0 || *partial_children != 0; + if child_is_selected { + *selected_children += 1; + if child_was_partial { + *partial_children -= 1; + } + } else if !child_was_partial { + *partial_children += 1; + } + let is_selected = *selected_children == *total_children; + if is_selected { + *selected = true; + } else if was_partial { + break; + } + child_is_selected = is_selected; + child_was_partial = was_partial; + } + } + } + + /// Marks the given node as (completely) deselected. Also marks all descendant nodes as + /// deselected and updates ancestor nodes correspondingly. + /// + /// When run m times in a row (without running `select`) on arbitrary nodes in a tree with n + /// nodes, this takes worst case O(m + n) time thanks to memoization. + fn deselect(&mut self, node_id: indextree::NodeId) { + // We can skip nodes that are not marked as completely selected and have zero selected or + // partially selected children + match self.arena[node_id].get() { + Entry::File { selected, .. } => { + if !*selected { + return; + } + } + Entry::Dir { + selected, + selected_children, + partial_children, + .. + } => { + if !*selected && *selected_children == 0 && *partial_children == 0 { + return; + } + } + } + + // Deelect all of this node's descendants in a postorder traversal + for node_id in node_id.children(&self.arena).collect_vec() { + self.deselect(node_id); + } + + let mut child_is_deselected = true; + let mut child_was_partial = false; + + // Deselect this node + match self.arena[node_id].get_mut() { + Entry::File { selected, .. } => { + if !*selected { + return; + } + *selected = false; + } + Entry::Dir { + selected, + total_children, + selected_children, + partial_children, + .. + } => { + if !*selected && *selected_children == 0 && *partial_children == 0 { + return; + } + *selected = false; + child_was_partial = *selected_children != *total_children; + *selected_children = 0; + *partial_children = 0; + } + } + + // Visit and update ancestor nodes until we either reach the root node or we reach an + // ancestor that does not change state (not selected / completely selected / partially + // selected) after updating it (that implies that the ancestors of *that* node will also + // not change state after updating, so visiting them would be redundant) + for node_id in node_id.ancestors(&self.arena).skip(1).collect_vec() { + if let Entry::Dir { + selected, + total_children, + selected_children, + partial_children, + .. + } = self.arena[node_id].get_mut() + { + *selected = false; + let was_partial = *selected_children != *total_children; + if child_was_partial { + *partial_children -= 1; + } else { + *selected_children -= 1; + if !child_is_deselected { + *partial_children += 1; + } + } + let is_deselected = *selected_children == 0 && *partial_children == 0; + if !is_deselected && was_partial { + break; + } + child_is_deselected = is_deselected; + child_was_partial = was_partial; + } + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct Metadata { + pub path: String, + pub is_file: bool, +} + +/// An iterator over the selected entries of a `FileSystemView` from top to bottom. +/// +/// The iterator does not recurse into directories that are completely selected - that is, if a +/// directory is completely selected, then this iterator will iterate over the directory but +/// none of its contents. +pub struct SelectedIter<'a, T> +where + T: luminol_filesystem::FileSystem, +{ + view: &'a FileSystemView, + edge: Option, +} + +impl<'a, T> std::iter::FusedIterator for SelectedIter<'a, T> where T: luminol_filesystem::FileSystem {} + +impl<'a, T> Iterator for SelectedIter<'a, T> +where + T: luminol_filesystem::FileSystem, +{ + type Item = Metadata; + fn next(&mut self) -> Option { + loop { + match self.edge { + None => { + return None; + } + + Some(indextree::NodeEdge::Start(node_id)) => { + let entry = self.view.arena[node_id].get(); + let first_child_id = if entry.selected() + || matches!( + entry, + Entry::Dir { + selected: false, + selected_children: 0, + partial_children: 0, + .. + } + ) { + // No need to recurse into directories that are completely selected because + // we know all of its contents are selected too. + // No need to recurse into directories that are completely deselected + // either because all of its contents are deselected. + None + } else { + node_id.children(&self.view.arena).next() + }; + + self.edge = Some(if let Some(first_child_id) = first_child_id { + indextree::NodeEdge::Start(first_child_id) + } else { + indextree::NodeEdge::End(node_id) + }); + + if entry.selected() { + let mut ancestors = node_id + .ancestors(&self.view.arena) + .filter_map(|n| { + let name = self.view.arena[n].get().name(); + (!name.is_empty()).then_some(name) + }) + .collect_vec(); + ancestors.reverse(); + + return Some(Metadata { + path: ancestors.join("/"), + is_file: matches!(entry, Entry::File { .. }), + }); + } + } + + Some(indextree::NodeEdge::End(node_id)) => { + let next_sibling_id = node_id.following_siblings(&self.view.arena).nth(1); + + self.edge = if let Some(next_sibling_id) = next_sibling_id { + Some(indextree::NodeEdge::Start(next_sibling_id)) + } else { + node_id + .ancestors(&self.view.arena) + .nth(1) + .map(indextree::NodeEdge::End) + }; + } + } + } + } +} + +impl<'a, T> IntoIterator for &'a FileSystemView +where + T: luminol_filesystem::FileSystem, +{ + type Item = Metadata; + type IntoIter = SelectedIter<'a, T>; + fn into_iter(self) -> Self::IntoIter { + SelectedIter { + view: self, + edge: Some(indextree::NodeEdge::Start(self.root_node_id)), + } + } +} diff --git a/crates/components/src/lib.rs b/crates/components/src/lib.rs index 813c5feb..9ce22c94 100644 --- a/crates/components/src/lib.rs +++ b/crates/components/src/lib.rs @@ -37,6 +37,9 @@ pub use sound_tab::SoundTab; mod command_view; pub use command_view::CommandView; +mod filesystem_view; +pub use filesystem_view::FileSystemView; + pub struct EnumMenuButton<'e, T> { current_value: &'e mut T, id: egui::Id, diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 6d58283d..4b4ddbf1 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -41,6 +41,7 @@ mod toasts; pub use toasts::Toasts; pub mod project_manager; +pub use project_manager::spawn_future; pub use project_manager::ProjectManager; pub struct UpdateState<'res> { @@ -253,19 +254,12 @@ impl<'res> UpdateState<'res> { fn handle_project_loading(&mut self) { let mut filesystem_open_result = None; - #[cfg(target_arch = "wasm32")] - let mut idb_key = None; if let Some(p) = self.project_manager.load_filesystem_promise.take() { match p.try_take() { Ok(Ok(host)) => { self.close_project(); - #[cfg(target_arch = "wasm32")] - { - idb_key = host.idb_key().map(str::to_string); - } - filesystem_open_result = Some(self.filesystem.load_project( host, self.project_config, @@ -304,9 +298,6 @@ impl<'res> UpdateState<'res> { ) { self.toasts .error(format!("Error loading the project data: {why}")); - - #[cfg(target_arch = "wasm32")] - idb_key.map(luminol_filesystem::host::FileSystem::idb_drop); } else { self.toasts.info(format!( "Successfully opened {:?}", @@ -317,9 +308,6 @@ impl<'res> UpdateState<'res> { Some(Err(why)) => { self.toasts .error(format!("Error opening the project: {why}")); - - #[cfg(target_arch = "wasm32")] - idb_key.map(luminol_filesystem::host::FileSystem::idb_drop); } None => {} } diff --git a/crates/core/src/project_manager.rs b/crates/core/src/project_manager.rs index 99ed7341..1d8594c6 100644 --- a/crates/core/src/project_manager.rs +++ b/crates/core/src/project_manager.rs @@ -42,6 +42,24 @@ pub type CreateProjectPromiseResult = anyhow::Result; pub type FileSystemPromiseResult = luminol_filesystem::Result; pub type FileSystemOpenResult = luminol_filesystem::Result; +#[cfg(not(target_arch = "wasm32"))] +/// Spawns a future using `poll_promise::Promise::spawn_async` on native or +/// `poll_promise::Promise::spawn_local` on web. +pub fn spawn_future( + future: impl std::future::Future + Send + 'static, +) -> poll_promise::Promise { + poll_promise::Promise::spawn_async(future) +} + +#[cfg(target_arch = "wasm32")] +/// Spawns a future using `poll_promise::Promise::spawn_async` on native or +/// `poll_promise::Promise::spawn_local` on web. +pub fn spawn_future( + future: impl std::future::Future + 'static, +) -> poll_promise::Promise { + poll_promise::Promise::spawn_local(future) +} + impl ProjectManager { pub fn new(ctx: &egui::Context) -> Self { Self { @@ -86,21 +104,12 @@ impl ProjectManager { /// Opens a project picker after asking the user to save unsaved changes. pub fn open_project_picker(&mut self) { self.run_custom(|update_state| { - // maybe worthwhile to make an extension trait to select spawn_async or spawn_local based on the target? #[cfg(not(target_arch = "wasm32"))] - { - update_state.project_manager.load_filesystem_promise = - Some(poll_promise::Promise::spawn_async( - luminol_filesystem::host::FileSystem::from_file_picker(), - )); - } + let promise = spawn_future(luminol_filesystem::host::FileSystem::from_file_picker()); #[cfg(target_arch = "wasm32")] - { - update_state.project_manager.load_filesystem_promise = - Some(poll_promise::Promise::spawn_local( - luminol_filesystem::host::FileSystem::from_folder_picker(), - )); - } + let promise = spawn_future(luminol_filesystem::host::FileSystem::from_folder_picker()); + + update_state.project_manager.load_filesystem_promise = Some(promise); }); } @@ -123,10 +132,9 @@ impl ProjectManager { #[cfg(target_arch = "wasm32")] { - update_state.project_manager.load_filesystem_promise = - Some(poll_promise::Promise::spawn_local( - luminol_filesystem::host::FileSystem::from_idb_key(key), - )); + update_state.project_manager.load_filesystem_promise = Some(spawn_future( + luminol_filesystem::host::FileSystem::from_idb_key(key), + )); } }); } diff --git a/crates/filesystem/Cargo.toml b/crates/filesystem/Cargo.toml index 74f471bd..a34595b5 100644 --- a/crates/filesystem/Cargo.toml +++ b/crates/filesystem/Cargo.toml @@ -28,6 +28,9 @@ itertools.workspace = true dashmap.workspace = true parking_lot.workspace = true +futures-lite.workspace = true +async-std.workspace = true +pin-project.workspace = true egui.workspace = true @@ -42,13 +45,16 @@ luminol-config.workspace = true rand.workspace = true iter-read = "1.0.1" -qp-trie = "0.8.2" +async_io_stream = "0.3.3" + +qp-trie.workspace = true [target.'cfg(windows)'.dependencies] winreg = "0.51.0" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tempfile = "3.8.1" +async-fs = "2.1.0" [target.'cfg(target_arch = "wasm32")'.dependencies] once_cell.workspace = true @@ -77,7 +83,10 @@ web-sys = { version = "0.3", features = [ "FileSystemWritableFileStream", "WritableStream", + "Element", + "HtmlAnchorElement", "Navigator", "StorageManager", + "Url", "Window", ] } diff --git a/crates/filesystem/src/archiver/file.rs b/crates/filesystem/src/archiver/file.rs index 0a761d76..f3636b44 100644 --- a/crates/filesystem/src/archiver/file.rs +++ b/crates/filesystem/src/archiver/file.rs @@ -15,12 +15,14 @@ // You should have received a copy of the GNU General Public License // along with Luminol. If not, see . +use pin_project::pin_project; use std::io::{ prelude::*, BufReader, ErrorKind::{InvalidData, PermissionDenied}, SeekFrom, }; +use std::{pin::Pin, task::Poll}; use super::util::{move_file_and_truncate, read_file_xor, regress_magic}; use super::Trie; @@ -28,6 +30,7 @@ use crate::File as _; use crate::Metadata; #[derive(Debug)] +#[pin_project] pub struct File where T: crate::File, @@ -36,21 +39,11 @@ where pub(super) trie: Option>>, pub(super) path: camino::Utf8PathBuf, pub(super) read_allowed: bool, - pub(super) tmp: crate::host::File, pub(super) modified: parking_lot::Mutex, pub(super) version: u8, pub(super) base_magic: u32, -} - -impl Drop for File -where - T: crate::File, -{ - fn drop(&mut self) { - if self.archive.is_some() { - let _ = self.flush(); - } - } + #[pin] + pub(super) tmp: crate::host::File, } impl std::io::Write for File @@ -134,7 +127,7 @@ where } // Write the new length of the file to the trie - trie.get_mut_file(&self.path).ok_or(InvalidData)?.size = new_size; + trie.get_file_mut(&self.path).ok_or(InvalidData)?.size = new_size; } // Now write the new contents of the file @@ -191,6 +184,35 @@ where } } +impl futures_lite::AsyncRead for File +where + T: crate::File + futures_lite::AsyncRead, +{ + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> Poll> { + if self.read_allowed { + self.project().tmp.poll_read(cx, buf) + } else { + Poll::Ready(Err(PermissionDenied.into())) + } + } + + fn poll_read_vectored( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + bufs: &mut [std::io::IoSliceMut<'_>], + ) -> Poll> { + if self.read_allowed { + self.project().tmp.poll_read_vectored(cx, bufs) + } else { + Poll::Ready(Err(PermissionDenied.into())) + } + } +} + impl std::io::Seek for File where T: crate::File, @@ -204,6 +226,19 @@ where } } +impl futures_lite::AsyncSeek for File +where + T: crate::File + futures_lite::AsyncSeek, +{ + fn poll_seek( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + pos: SeekFrom, + ) -> Poll> { + self.project().tmp.poll_seek(cx, pos) + } +} + impl crate::File for File where T: crate::File, diff --git a/crates/filesystem/src/archiver/filesystem.rs b/crates/filesystem/src/archiver/filesystem.rs index f63c7c37..4377d927 100644 --- a/crates/filesystem/src/archiver/filesystem.rs +++ b/crates/filesystem/src/archiver/filesystem.rs @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with Luminol. If not, see . +use async_std::io::{BufReader as AsyncBufReader, BufWriter as AsyncBufWriter}; use itertools::Itertools; use rand::Rng; use std::io::{ @@ -25,9 +26,10 @@ use std::io::{ }; use super::util::{ - advance_magic, move_file_and_truncate, read_file_xor, read_header, read_u32_xor, regress_magic, + advance_magic, move_file_and_truncate, read_file_xor, read_file_xor_async, read_header, + read_u32_xor, regress_magic, }; -use super::{Entry, File, Trie, MAGIC}; +use super::{Entry, File, Trie, HEADER, MAGIC}; use crate::{DirEntry, Error, Metadata, OpenFlags}; #[derive(Debug, Default)] @@ -38,11 +40,24 @@ pub struct FileSystem { pub(super) base_magic: u32, } +impl Clone for FileSystem { + fn clone(&self) -> Self { + Self { + trie: self.trie.clone(), + archive: self.archive.clone(), + version: self.version, + base_magic: self.base_magic, + } + } +} + impl FileSystem where T: crate::File, { + /// Creates a new archiver filesystem from a file containing an existing archive. pub fn new(mut file: T) -> Result { + file.seek(SeekFrom::Start(0))?; let mut reader = BufReader::new(&mut file); let version = read_header(&mut reader)?; @@ -90,7 +105,7 @@ where reader.read_exact(&mut u32_buf)?; base_magic = u32::from_le_bytes(u32_buf); - base_magic = (base_magic * 9) + 3; + base_magic = base_magic.wrapping_mul(9).wrapping_add(3); while let Ok(body_offset) = read_u32_xor(&mut reader, base_magic) { if body_offset == 0 { @@ -129,13 +144,198 @@ where _ => return Err(Error::InvalidHeader), } - Ok(FileSystem { + Ok(Self { trie: std::sync::Arc::new(parking_lot::RwLock::new(trie)), archive: std::sync::Arc::new(parking_lot::Mutex::new(file)), version, base_magic, }) } + + /// Creates a new archiver filesystem from the given files. + /// The contents of the archive itself will be stored in `buffer`. + pub async fn from_buffer_and_files<'a, I, P, R>( + mut buffer: T, + version: u8, + files: I, + ) -> Result + where + T: futures_lite::AsyncWrite + futures_lite::AsyncSeek + Unpin, + I: Iterator>, + P: AsRef + 'a, + R: futures_lite::AsyncRead + Unpin, + { + use futures_lite::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; + + buffer.set_len(0)?; + AsyncSeekExt::seek(&mut buffer, SeekFrom::Start(0)).await?; + + let mut writer = AsyncBufWriter::new(&mut buffer); + writer.write_all(HEADER).await?; + writer.write_all(&[version]).await?; + + let mut trie = Trie::new(); + + match version { + 1 | 2 => { + let mut magic = MAGIC; + let mut header_offset = 8; + + for result in files { + let (path, size, file) = result?; + let reader = AsyncBufReader::new(file.take(size as u64)); + let path = path.as_ref(); + let header_size = path.as_str().bytes().len() as u64 + 8; + + // Write the header + writer + .write_all( + &(path.as_str().bytes().len() as u32 ^ advance_magic(&mut magic)) + .to_le_bytes(), + ) + .await?; + writer + .write_all( + &path + .as_str() + .bytes() + .map(|b| { + let b = if b == b'/' { b'\\' } else { b }; + b ^ advance_magic(&mut magic) as u8 + }) + .collect_vec(), + ) + .await?; + writer + .write_all(&(size ^ advance_magic(&mut magic)).to_le_bytes()) + .await?; + + // Write the file contents + async_std::io::copy(&mut read_file_xor_async(reader, magic), &mut writer) + .await?; + + trie.create_file( + path, + Entry { + header_offset, + body_offset: header_offset + header_size, + size: size as u64, + start_magic: magic, + }, + ); + + header_offset += header_size + size as u64; + } + + writer.flush().await?; + drop(writer); + Ok(Self { + trie: std::sync::Arc::new(parking_lot::RwLock::new(trie)), + archive: std::sync::Arc::new(parking_lot::Mutex::new(buffer)), + version, + base_magic: MAGIC, + }) + } + + 3 => { + let mut tmp = crate::host::File::new()?; + let mut tmp_writer = AsyncBufWriter::new(&mut tmp); + let mut entries = if let (_, Some(upper_bound)) = files.size_hint() { + Vec::with_capacity(upper_bound) + } else { + Vec::new() + }; + + let base_magic: u32 = rand::thread_rng().gen(); + writer + .write_all(&(base_magic.wrapping_sub(3).wrapping_mul(954437177)).to_le_bytes()) + .await?; + let mut header_offset = 12; + let mut body_offset = 0; + + for result in files { + let (path, size, file) = result?; + let reader = AsyncBufReader::new(file.take(size as u64)); + let path = path.as_ref(); + let entry_magic: u32 = rand::thread_rng().gen(); + + // Write the header to the buffer, except for the offset + writer.seek(SeekFrom::Current(4)).await?; + writer.write_all(&(size ^ base_magic).to_le_bytes()).await?; + writer + .write_all(&(entry_magic ^ base_magic).to_le_bytes()) + .await?; + writer + .write_all(&(path.as_str().bytes().len() as u32 ^ base_magic).to_le_bytes()) + .await?; + writer + .write_all( + &path + .as_str() + .bytes() + .enumerate() + .map(|(i, b)| { + let b = if b == b'/' { b'\\' } else { b }; + b ^ (base_magic >> (8 * (i % 4))) as u8 + }) + .collect_vec(), + ) + .await?; + + // Write the actual file contents to a temporary file + async_std::io::copy( + &mut read_file_xor_async(reader, entry_magic), + &mut tmp_writer, + ) + .await?; + + entries.push(( + path.to_owned(), + Entry { + header_offset, + body_offset, + size: size as u64, + start_magic: entry_magic, + }, + )); + + header_offset += path.as_str().bytes().len() as u64 + 16; + body_offset += size as u64; + } + + // Write the terminator at the end of the buffer + writer.write_all(&base_magic.to_le_bytes()).await?; + + // Write the contents of the temporary file to the buffer after the terminator + tmp_writer.flush().await?; + drop(tmp_writer); + AsyncSeekExt::seek(&mut tmp, SeekFrom::Start(0)).await?; + async_std::io::copy(&mut tmp, &mut writer).await?; + + // Write the offsets into the header now that we know the total size of the files + let header_size = header_offset + 4; + for (path, mut entry) in entries { + entry.body_offset += header_size; + writer.seek(SeekFrom::Start(entry.header_offset)).await?; + writer + .write_all(&(entry.body_offset as u32 ^ base_magic).to_le_bytes()) + .await?; + trie.create_file(path, entry); + } + + writer.flush().await?; + drop(writer); + Ok(Self { + trie: std::sync::Arc::new(parking_lot::RwLock::new(trie)), + archive: std::sync::Arc::new(parking_lot::Mutex::new(buffer)), + version, + base_magic, + }) + } + + _ => Err(Error::NotSupported), + } + } } impl crate::FileSystem for FileSystem @@ -156,8 +356,6 @@ where { let mut archive = self.archive.lock(); let mut trie = self.trie.write(); - let entry = *trie.get_file(path).ok_or(Error::NotExist)?; - archive.seek(SeekFrom::Start(entry.body_offset))?; if flags.contains(OpenFlags::Create) && !trie.contains_file(path) { created = true; @@ -177,6 +375,7 @@ where reader.seek(SeekFrom::Current(entry_len as i64))?; } drop(reader); + regress_magic(&mut magic); let archive_len = archive.seek(SeekFrom::End(0))?; let mut writer = BufWriter::new(archive.as_file()); @@ -279,12 +478,17 @@ where _ => return Err(Error::NotSupported), } } else if !flags.contains(OpenFlags::Truncate) { + let entry = *trie.get_file(path).ok_or(Error::NotExist)?; + archive.seek(SeekFrom::Start(entry.body_offset))?; + let mut adapter = BufReader::new(archive.as_file().take(entry.size)); std::io::copy( &mut read_file_xor(&mut adapter, entry.start_magic), &mut tmp, )?; tmp.flush()?; + } else if !trie.contains_file(path) { + return Err(Error::NotExist); } } @@ -480,7 +684,7 @@ where let current_body_offset = (current_body_offset as u64) .checked_add_signed(to_len as i64 - from_len as i64) .ok_or(Error::IoError(InvalidData.into()))?; - trie.get_mut_file(current_path) + trie.get_file_mut(current_path) .ok_or(Error::IoError(InvalidData.into()))? .body_offset = current_body_offset; headers.push((current_header_offset, current_body_offset as u32)); @@ -551,14 +755,9 @@ where fn create_dir(&self, path: impl AsRef) -> Result<(), Error> { let path = path.as_ref(); let mut trie = self.trie.write(); - if trie.contains(path) { + if trie.contains_file(path) { return Err(Error::IoError(AlreadyExists.into())); } - if let Some(parent) = path.parent() { - if !trie.contains_dir(parent) { - return Err(Error::NotExist); - } - } trie.create_dir(path); Ok(()) } @@ -644,7 +843,11 @@ where let trie = self.trie.read(); if let Some(iter) = trie.iter_dir(path) { iter.map(|(name, _)| { - let path = path.join(name); + let path = if path == "" { + name.into() + } else { + format!("{path}/{name}").into() + }; let metadata = self.metadata(&path)?; Ok(DirEntry { path, metadata }) }) diff --git a/crates/filesystem/src/archiver/util.rs b/crates/filesystem/src/archiver/util.rs index 5ebdfb93..70bcadf8 100644 --- a/crates/filesystem/src/archiver/util.rs +++ b/crates/filesystem/src/archiver/util.rs @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with Luminol. If not, see . +use async_std::stream::StreamExt; use itertools::Itertools; use std::io::{prelude::*, BufReader, BufWriter, ErrorKind::InvalidData, SeekFrom}; @@ -47,9 +48,34 @@ pub(super) fn read_file_xor(file: impl Read, start_magic: u32) -> impl Read { *state = (magic, j); Some(byte) }); + iter_read::IterRead::new(iter) } +pub(super) fn read_file_xor_async( + file: impl futures_lite::AsyncRead + Unpin, + start_magic: u32, +) -> impl futures_lite::AsyncRead + Unpin { + use futures_lite::AsyncReadExt; + + let stream = file.bytes().scan((start_magic, 0), |state, maybe_byte| { + let Ok(byte) = maybe_byte else { return None }; + let (mut magic, mut j) = *state; + + if j == 4 { + j = 0; + magic = magic.wrapping_mul(7).wrapping_add(3); + } + let byte = byte ^ magic.to_le_bytes()[j]; + j += 1; + + *state = (magic, j); + Some(Ok([byte])) + }); + + async_io_stream::IoStream::new(stream) +} + pub(super) fn advance_magic(magic: &mut u32) -> u32 { let old = *magic; @@ -130,7 +156,7 @@ where } } let current_path = String::from_utf8(current_path).map_err(|_| InvalidData)?; - let current_entry = trie.get_mut_file(¤t_path).ok_or(InvalidData)?; + let current_entry = trie.get_file_mut(¤t_path).ok_or(InvalidData)?; reader.seek(SeekFrom::Start(current_entry.body_offset))?; advance_magic(&mut reader_magic); @@ -152,10 +178,7 @@ where )?; std::io::copy( &mut read_file_xor( - &mut read_file_xor( - &mut (&mut reader).take(current_entry.size), - reader_magic, - ), + read_file_xor(&mut (&mut reader).take(current_entry.size), reader_magic), writer_magic, ), &mut writer, @@ -203,7 +226,7 @@ where .checked_sub(path_len + 8) .ok_or(InvalidData)?; entry.size = 0; - *trie.get_mut_file(path).ok_or(InvalidData)? = entry; + *trie.get_file_mut(path).ok_or(InvalidData)? = entry; Ok(()) } @@ -224,6 +247,7 @@ where // Find all of the files in the archive with offsets greater than the original // offset of the modified file and decrement them accordingly + let mut current_header_offset = 12; archive.seek(SeekFrom::Start(12))?; let mut reader = BufReader::new(archive.as_file()); let mut headers = Vec::new(); @@ -231,10 +255,6 @@ where if current_body_offset == 0 { break; } - let current_header_offset = reader - .stream_position()? - .checked_sub(4) - .ok_or(InvalidData)?; let current_body_offset = current_body_offset as u64; reader.seek(SeekFrom::Current(8))?; let current_path_len = read_u32_xor(&mut reader, base_magic)?; @@ -264,7 +284,7 @@ where .ok_or(InvalidData)? }; - trie.get_mut_file(current_path) + trie.get_file_mut(current_path) .ok_or(InvalidData)? .body_offset = current_body_offset; headers.push(( @@ -272,6 +292,8 @@ where current_body_offset as u32, should_truncate, )); + + current_header_offset += current_path_len as u64 + 16; } drop(reader); let mut writer = BufWriter::new(archive.as_file()); @@ -285,7 +307,7 @@ where writer.flush()?; drop(writer); - trie.get_mut_file(path).ok_or(InvalidData)?.size = 0; + trie.get_file_mut(path).ok_or(InvalidData)?.size = 0; Ok(()) } diff --git a/crates/filesystem/src/erased.rs b/crates/filesystem/src/erased.rs index 606dfb6c..c94996ee 100644 --- a/crates/filesystem/src/erased.rs +++ b/crates/filesystem/src/erased.rs @@ -46,6 +46,7 @@ pub trait ErasedFilesystem: Send + Sync { impl ErasedFilesystem for T where T: crate::FileSystem, + T::File: 'static, { fn open_file(&self, path: &camino::Utf8Path, flags: OpenFlags) -> Result> { let file = self.open_file(path, flags)?; diff --git a/crates/filesystem/src/lib.rs b/crates/filesystem/src/lib.rs index 7871edaa..f645eb61 100644 --- a/crates/filesystem/src/lib.rs +++ b/crates/filesystem/src/lib.rs @@ -45,6 +45,8 @@ pub enum Error { IoError(#[from] std::io::Error), #[error("UTF-8 Error {0}")] Utf8Error(#[from] std::string::FromUtf8Error), + #[error("Path is not valid UTF-8")] + PathUtf8Error, #[error("Project not loaded")] NotLoaded, #[error("Operation not supported by this filesystem")] @@ -113,7 +115,7 @@ bitflags::bitflags! { } } -pub trait File: std::io::Read + std::io::Write + std::io::Seek + Send + Sync + 'static { +pub trait File: std::io::Read + std::io::Write + std::io::Seek + Send + Sync { fn metadata(&self) -> std::io::Result; /// Truncates or extends the size of the file. If the file is extended, the file will be @@ -130,7 +132,20 @@ pub trait File: std::io::Read + std::io::Write + std::io::Seek + Send + Sync + ' } } -pub trait FileSystem: Send + Sync + 'static { +impl File for &mut T +where + T: File + ?Sized, +{ + fn metadata(&self) -> std::io::Result { + (**self).metadata() + } + + fn set_len(&self, new_size: u64) -> std::io::Result<()> { + (**self).set_len(new_size) + } +} + +pub trait FileSystem: Send + Sync { type File: File; fn open_file(&self, path: impl AsRef, flags: OpenFlags) diff --git a/crates/filesystem/src/native.rs b/crates/filesystem/src/native.rs index 31e9b326..247a4a3c 100644 --- a/crates/filesystem/src/native.rs +++ b/crates/filesystem/src/native.rs @@ -17,6 +17,12 @@ use itertools::Itertools; use crate::{DirEntry, Metadata, OpenFlags, Result}; +use pin_project::pin_project; +use std::{ + io::ErrorKind::{InvalidInput, PermissionDenied}, + pin::Pin, + task::Poll, +}; #[derive(Debug, Clone)] pub struct FileSystem { @@ -24,7 +30,19 @@ pub struct FileSystem { } #[derive(Debug)] -pub struct File(std::fs::File); +#[pin_project] +pub struct File { + file: Inner, + path: camino::Utf8PathBuf, + #[pin] + async_file: async_fs::File, +} + +#[derive(Debug)] +enum Inner { + StdFsFile(std::fs::File), + NamedTempFile(tempfile::NamedTempFile), +} impl FileSystem { pub fn new(root_path: impl AsRef) -> Self { @@ -39,7 +57,8 @@ impl FileSystem { pub async fn from_folder_picker() -> Result { if let Some(path) = rfd::AsyncFileDialog::default().pick_folder().await { - let path = camino::Utf8Path::from_path(path.path()).expect("path not utf-8"); + let path = + camino::Utf8Path::from_path(path.path()).ok_or(crate::Error::PathUtf8Error)?; Ok(Self::new(path)) } else { Err(crate::Error::CancelledLoading) @@ -53,7 +72,7 @@ impl FileSystem { .await { let path = camino::Utf8Path::from_path(path.path()) - .expect("path not utf-8") + .ok_or(crate::Error::PathUtf8Error)? .parent() .expect("path does not have parent"); Ok(Self::new(path)) @@ -77,9 +96,15 @@ impl crate::FileSystem for FileSystem { .write(flags.contains(OpenFlags::Write)) .read(flags.contains(OpenFlags::Read)) .truncate(flags.contains(OpenFlags::Truncate)) - .open(path) - .map_err(Into::into) - .map(File) + .open(&path) + .map(|file| { + let clone = file.try_clone()?; + Ok(File { + file: Inner::StdFsFile(file), + path, + async_file: clone.into(), + }) + })? } fn metadata(&self, path: impl AsRef) -> Result { @@ -108,7 +133,7 @@ impl crate::FileSystem for FileSystem { fn create_dir(&self, path: impl AsRef) -> Result<()> { let path = self.root_path.join(path); - std::fs::create_dir(path).map_err(Into::into) + std::fs::create_dir_all(path).map_err(Into::into) } fn remove_dir(&self, path: impl AsRef) -> Result<()> { @@ -146,13 +171,91 @@ impl crate::FileSystem for FileSystem { impl File { /// Creates a new empty temporary file with read-write permissions. pub fn new() -> std::io::Result { - tempfile::tempfile().map(File) + let file = tempfile::NamedTempFile::new()?; + let path = file.path().to_str().ok_or(InvalidInput)?.into(); + let clone = file.as_file().try_clone()?; + Ok(Self { + file: Inner::NamedTempFile(file), + path, + async_file: clone.into(), + }) + } + + /// Attempts to prompt the user to choose a file from their local machine. + /// Then creates a `File` allowing read-write access to that directory if they chose one + /// successfully, along with the name of the file including the extension. + /// + /// `extensions` should be a list of accepted file extensions for the file, without the leading + /// `.` + pub async fn from_file_picker( + filter_name: &str, + extensions: &[impl ToString], + ) -> Result<(Self, String)> { + if let Some(path) = rfd::AsyncFileDialog::default() + .add_filter(filter_name, extensions) + .pick_file() + .await + { + let file = std::fs::OpenOptions::new() + .read(true) + .open(path.path()) + .map_err(crate::Error::IoError)?; + let path = path + .path() + .iter() + .last() + .unwrap() + .to_os_string() + .into_string() + .map_err(|_| crate::Error::PathUtf8Error)?; + let clone = file.try_clone()?; + Ok(( + File { + file: Inner::StdFsFile(file), + path: path.clone().into(), + async_file: clone.into(), + }, + path, + )) + } else { + Err(crate::Error::CancelledLoading) + } + } + + /// Saves this file to a location of the user's choice. + /// + /// In native, this will open a file picker dialog, wait for the user to choose a location to + /// save a file, and then copy this file to the new location. This function will wait for the + /// user to finish picking a file location before returning. + /// + /// In web, this will use the browser's native file downloading method to save the file, which + /// may or may not open a file picker. Due to platform limitations, this function will return + /// immediately after making a download request and will not wait for the user to pick a file + /// location if a file picker is shown. + /// + /// You must flush the file yourself before saving. It will not be flushed for you. + /// + /// `filename` should be the default filename, with extension, to show in the file picker if + /// one is shown. `filter_name` should be the name of the file type shown in the part of the + /// file picker where the user selects a file extension. `filter_name` works only in native + /// builds; it is ignored in web builds. + pub async fn save(&self, filename: &str, filter_name: &str) -> Result<()> { + let mut dialog = rfd::AsyncFileDialog::default().set_file_name(filename); + if let Some((_, extension)) = filename.rsplit_once('.') { + dialog = dialog.add_filter(filter_name, &[extension]); + } + let path = dialog + .save_file() + .await + .ok_or(crate::Error::CancelledLoading)?; + std::fs::copy(&self.path, path.path())?; + Ok(()) } } impl crate::File for File { fn metadata(&self) -> std::io::Result { - let metdata = self.0.metadata()?; + let metdata = self.file.as_file().metadata()?; Ok(Metadata { is_file: metdata.is_file(), size: metdata.len(), @@ -160,36 +263,105 @@ impl crate::File for File { } fn set_len(&self, new_size: u64) -> std::io::Result<()> { - self.0.set_len(new_size) + self.file.as_file().set_len(new_size) } } impl std::io::Read for File { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.0.read(buf) + self.file.as_file().read(buf) } fn read_vectored(&mut self, bufs: &mut [std::io::IoSliceMut<'_>]) -> std::io::Result { - self.0.read_vectored(bufs) + self.file.as_file().read_vectored(bufs) + } +} + +impl futures_lite::AsyncRead for File { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> Poll> { + self.project().async_file.poll_read(cx, buf) + } + + fn poll_read_vectored( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + bufs: &mut [std::io::IoSliceMut<'_>], + ) -> Poll> { + self.project().async_file.poll_read_vectored(cx, bufs) } } impl std::io::Write for File { fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.0.write(buf) + self.file.as_file().write(buf) } fn flush(&mut self) -> std::io::Result<()> { - self.0.flush() + self.file.as_file().flush() } fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result { - self.0.write_vectored(bufs) + self.file.as_file().write_vectored(bufs) + } +} + +impl futures_lite::AsyncWrite for File { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + self.project().async_file.poll_write(cx, buf) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + self.project().async_file.poll_write_vectored(cx, bufs) + } + + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + self.project().async_file.poll_flush(cx) + } + + fn poll_close( + self: Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> Poll> { + Poll::Ready(Err(PermissionDenied.into())) } } impl std::io::Seek for File { fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { - self.0.seek(pos) + self.file.as_file().seek(pos) + } +} + +impl futures_lite::AsyncSeek for File { + fn poll_seek( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + pos: std::io::SeekFrom, + ) -> Poll> { + self.project().async_file.poll_seek(cx, pos) + } +} + +impl Inner { + fn as_file(&self) -> &std::fs::File { + match self { + Inner::StdFsFile(file) => file, + Inner::NamedTempFile(file) => file.as_file(), + } } } diff --git a/crates/filesystem/src/project.rs b/crates/filesystem/src/project.rs index 74f114be..d6c19328 100644 --- a/crates/filesystem/src/project.rs +++ b/crates/filesystem/src/project.rs @@ -29,6 +29,7 @@ pub enum FileSystem { HostLoaded(host::FileSystem), Loaded { filesystem: path_cache::FileSystem, + host_filesystem: host::FileSystem, project_path: camino::Utf8PathBuf, }, } @@ -99,9 +100,7 @@ impl FileSystem { } fn load_project_config(&self) -> Result { - if !self.exists(".luminol")? { - self.create_dir(".luminol")?; - } + self.create_dir(".luminol")?; let project = match self .read_to_string(".luminol/config") @@ -200,6 +199,16 @@ impl FileSystem { Ok(result) } + + pub fn host(&self) -> Option { + match self { + FileSystem::Unloaded => None, + FileSystem::HostLoaded(host) => Some(host.clone()), + FileSystem::Loaded { + host_filesystem, .. + } => Some(host_filesystem.clone()), + } + } } // Specific to windows @@ -329,6 +338,7 @@ impl FileSystem { project_config: &luminol_config::project::Config, global_config: &mut luminol_config::global::Config, ) -> Result { + let host_clone = host.clone(); let project_path = host.root_path().to_path_buf(); let mut list = list::FileSystem::new(); @@ -359,6 +369,7 @@ impl FileSystem { *self = FileSystem::Loaded { filesystem: path_cache, + host_filesystem: host_clone, project_path: project_path.to_path_buf(), }; @@ -443,7 +454,6 @@ impl FileSystem { }; let root_path = host.root_path().to_path_buf(); - let idb_key = host.idb_key().map(|k| k.to_string()); let mut list = list::FileSystem::new(); @@ -465,7 +475,7 @@ impl FileSystem { .map(archiver::FileSystem::new) .transpose()?; - list.push(host); + list.push(host.clone()); for filesystem in rtp_filesystems { list.push(filesystem) } @@ -477,17 +487,18 @@ impl FileSystem { *self = Self::Loaded { filesystem: path_cache, + host_filesystem: host.clone(), project_path: root_path.clone(), }; - if let Some(idb_key) = idb_key { + if let Ok(idb_key) = host.save_to_idb() { let mut projects: std::collections::VecDeque<_> = global_config .recent_projects .iter() - .filter(|(_, k)| k.as_str() != idb_key.as_str()) + .filter(|(_, k)| k.as_str() != idb_key) .cloned() .collect(); - projects.push_front((root_path.to_string(), idb_key)); + projects.push_front((root_path.to_string(), idb_key.to_string())); global_config.recent_projects = projects; } diff --git a/crates/filesystem/src/trie.rs b/crates/filesystem/src/trie.rs index 26ec283b..4b688b1c 100644 --- a/crates/filesystem/src/trie.rs +++ b/crates/filesystem/src/trie.rs @@ -42,9 +42,11 @@ impl<'a, T> Iterator for FileSystemTrieDirIter<'a, T> { type Item = (&'a str, Option<&'a T>); fn next(&mut self) -> Option { match &mut self.0 { - FileSystemTrieDirIterInner::Direct(iter, _) => iter - .next() - .map(|(key, value)| (key.as_str(), value.as_ref())), + FileSystemTrieDirIterInner::Direct(iter, len) => { + *len = len.saturating_sub(1); + iter.next() + .map(|(key, value)| (key.as_str(), value.as_ref())) + } FileSystemTrieDirIterInner::Prefix(iter) => iter.next(), } } @@ -135,7 +137,7 @@ impl FileSystemTrie { } else { &prefix_with_trailing_slash }) - .nth(1) + .next() { // If there is a different path in the trie that has this as a prefix (there can be // at most one), add it to this directory entry @@ -153,11 +155,10 @@ impl FileSystemTrie { } // Add the new path to the entry at this prefix - let prefix_trie = self.0.get_mut_str(&prefix).unwrap(); - prefix_trie.insert_str( - path.strip_prefix(&prefix).unwrap().iter().next().unwrap(), - None, - ); + if let Some(dirname) = path.strip_prefix(&prefix).unwrap().iter().next() { + let prefix_trie = self.0.get_mut_str(&prefix).unwrap(); + prefix_trie.insert_str(dirname, None); + } // Add the actual entry for the new path self.0.insert_str(path.as_str(), DirTrie::new()); @@ -209,9 +210,12 @@ impl FileSystemTrie { return false; }; let dir = path.parent().unwrap_or(camino::Utf8Path::new("")); - self.0 - .get_str(dir.as_str()) - .map_or(false, |dir_trie| dir_trie.contains_key_str(filename)) + self.0.get_str(dir.as_str()).map_or(false, |dir_trie| { + dir_trie + .get_str(filename) + .and_then(|o| o.as_ref()) + .is_some() + }) } /// Returns whether or not a file or directory exists at the given path. @@ -252,7 +256,7 @@ impl FileSystemTrie { } /// Gets a mutable reference to the value in the file at the given path, if it exists. - pub fn get_mut_file(&mut self, path: impl AsRef) -> Option<&mut T> { + pub fn get_file_mut(&mut self, path: impl AsRef) -> Option<&mut T> { let path = path.as_ref(); let Some(filename) = path.iter().next_back() else { return None; @@ -266,11 +270,11 @@ impl FileSystemTrie { /// Gets the longest prefix of the given path that is the path of a directory in the trie. pub fn get_dir_prefix(&self, path: impl AsRef) -> &camino::Utf8Path { - let path = path.as_ref().as_str(); - if self.0.contains_key_str(path) { + let path = path.as_ref(); + if self.0.contains_key_str(path.as_str()) { return self .0 - .longest_common_prefix::(path.into()) + .longest_common_prefix::(path.as_str().into()) .as_str() .into(); } @@ -278,8 +282,8 @@ impl FileSystemTrie { .0 .longest_common_prefix::(format!("{path}/").as_str().into()) .as_str(); - let prefix = if !self.0.contains_key_str(prefix) { - prefix.rsplit_once('/').map(|(s, _)| s).unwrap_or(prefix) + let prefix = if !self.0.contains_key_str(prefix) || !path.starts_with(prefix) { + prefix.rsplit_once('/').map(|(s, _)| s).unwrap_or_default() } else { prefix }; @@ -312,6 +316,11 @@ impl FileSystemTrie { self.0.remove_str(path_str); if let Some(parent) = path.parent() { self.create_dir(parent); + if let (Some(parent_trie), Some(dirname)) = + (self.0.get_mut_str(parent.as_str()), path.iter().next_back()) + { + parent_trie.remove_str(dirname); + } } true } else if self @@ -324,6 +333,11 @@ impl FileSystemTrie { self.0.remove_prefix_str(&format!("{path_str}/")); if let Some(parent) = path.parent() { self.create_dir(parent); + if let (Some(parent_trie), Some(dirname)) = + (self.0.get_mut_str(parent.as_str()), path.iter().next_back()) + { + parent_trie.remove_str(dirname); + } } true } else { diff --git a/crates/filesystem/src/web/events.rs b/crates/filesystem/src/web/events.rs index f65df28a..dfef6f49 100644 --- a/crates/filesystem/src/web/events.rs +++ b/crates/filesystem/src/web/events.rs @@ -15,19 +15,23 @@ // You should have received a copy of the GNU General Public License // along with Luminol. If not, see . -use super::util::{generate_key, get_subdir, get_tmp_dir, handle_event, idb, to_future}; +use super::util::{ + generate_key, get_subdir, get_subdir_create, get_tmp_dir, handle_event, idb, to_future, +}; use super::FileSystemCommand; use crate::{DirEntry, Error, Metadata, OpenFlags}; use indexed_db_futures::prelude::*; -use std::io::ErrorKind::{AlreadyExists, InvalidInput, PermissionDenied}; +use std::io::ErrorKind::{InvalidInput, PermissionDenied}; use wasm_bindgen::prelude::*; pub fn setup_main_thread_hooks(main_channels: super::MainChannels) { wasm_bindgen_futures::spawn_local(async move { - let storage = web_sys::window() - .expect("cannot run `setup_main_thread_hooks()` outside of main thread") - .navigator() - .storage(); + let window = web_sys::window() + .expect("cannot run `setup_main_thread_hooks()` outside of main thread"); + let document = window + .document() + .expect("cannot run `setup_main_thread_hooks()` outside of main thread"); + let storage = window.navigator().storage(); struct FileHandle { offset: usize, @@ -100,20 +104,8 @@ pub fn setup_main_thread_hooks(main_channels: super::MainChannels) { FileSystemCommand::DirPicker(tx) => { handle_event(tx, async { let dir = luminol_web::bindings::show_directory_picker().await.ok()?; - - // Try to insert the handle into IndexedDB - let idb_key = generate_key(); - let idb_ok = { - let idb_key = idb_key.as_str(); - idb(IdbTransactionMode::Readwrite, |store| { - store.put_key_val_owned(idb_key, &dir) - }) - .await - .is_ok() - }; - let name = dir.name(); - Some((dirs.insert(dir), name, idb_ok.then_some(idb_key))) + Some((dirs.insert(dir), name)) }) .await; } @@ -129,12 +121,31 @@ pub fn setup_main_thread_hooks(main_channels: super::MainChannels) { .ok() .flatten()?; let dir = dir.unchecked_into::(); - luminol_web::bindings::request_permission(&dir) - .await - .then(|| { - let name = dir.name(); - (dirs.insert(dir), name) - }) + let key = + luminol_web::bindings::request_permission(&dir) + .await + .then(|| { + let name = dir.name(); + (dirs.insert(dir), name) + })?; + idb(IdbTransactionMode::Readwrite, |store| { + store.delete_owned(&idb_key) + }) + .await + .ok()?; + Some(key) + }) + .await; + } + + FileSystemCommand::DirToIdb(key, idb_key, tx) => { + handle_event(tx, async { + let dir = dirs.get(key).unwrap(); + idb(IdbTransactionMode::Readwrite, |store| { + store.put_key_val_owned(idb_key, dir) + }) + .await + .is_ok() }) .await; } @@ -145,31 +156,8 @@ pub fn setup_main_thread_hooks(main_channels: super::MainChannels) { let dir = get_subdir(dirs.get(key).unwrap(), &mut iter) .await .ok_or(Error::NotExist)?; - - // Try to insert the handle into IndexedDB - let idb_key = generate_key(); - let idb_ok = { - let idb_key = idb_key.as_str(); - idb(IdbTransactionMode::Readwrite, |store| { - store.put_key_val_owned(idb_key, &dir) - }) - .await - .is_ok() - }; - let name = dir.name(); - Ok((dirs.insert(dir), name, idb_ok.then_some(idb_key))) - }) - .await; - } - - FileSystemCommand::DirIdbDrop(idb_key, tx) => { - handle_event(tx, async { - idb(IdbTransactionMode::Readwrite, |store| { - store.delete_owned(&idb_key) - }) - .await - .is_ok() + Ok((dirs.insert(dir), name)) }) .await; } @@ -283,36 +271,10 @@ pub fn setup_main_thread_hooks(main_channels: super::MainChannels) { FileSystemCommand::DirCreateDir(key, path, tx) => { handle_event(tx, async { let mut iter = path.iter(); - let dirname = iter - .next_back() - .ok_or(Error::IoError(AlreadyExists.into()))?; - let subdir = get_subdir(dirs.get(key).unwrap(), &mut iter) + get_subdir_create(dirs.get(key).unwrap(), &mut iter) .await - .ok_or(Error::NotExist)?; - - if to_future::( - subdir.get_file_handle(dirname), - ) - .await - .is_ok() - || to_future::( - subdir.get_directory_handle(dirname), - ) - .await - .is_ok() - { - // If there is already a file or directory at the given path - return Err(Error::IoError(PermissionDenied.into())); - } - - let mut options = web_sys::FileSystemGetDirectoryOptions::new(); - options.create(true); - to_future::( - subdir.get_directory_handle_with_options(dirname, &options), - ) - .await - .map(|_| ()) - .map_err(|_| Error::IoError(PermissionDenied.into())) + .ok_or(Error::IoError(PermissionDenied.into()))?; + Ok(()) }) .await; } @@ -516,6 +478,52 @@ pub fn setup_main_thread_hooks(main_channels: super::MainChannels) { .await; } + FileSystemCommand::FilePicker(filter_name, extensions, tx) => { + handle_event(tx, async { + let file_handle = + luminol_web::bindings::show_file_picker(&filter_name, &extensions) + .await + .ok()?; + let name = file_handle.name(); + + Some(( + files.insert(FileHandle { + offset: 0, + file_handle, + read_allowed: true, + write_handle: None, + }), + name, + )) + }) + .await; + } + + FileSystemCommand::FileSave(key, filename, tx) => { + handle_event(tx, async { + let file = files.get(key).unwrap(); + let file_handle = file.read_allowed.then_some(&file.file_handle)?; + + let blob = to_future::(file_handle.get_file()) + .await + .ok()?; + let url = web_sys::Url::create_object_url_with_blob(&blob).ok()?; + + let anchor = document + .create_element("a") + .ok()? + .unchecked_into::(); + anchor.set_href(&url); + anchor.set_download(&filename); + anchor.click(); + anchor.remove(); + let _ = web_sys::Url::revoke_object_url(&url); + + Some(()) + }) + .await; + } + FileSystemCommand::FileRead(key, max_length, tx) => { handle_event(tx, async { let file = files.get_mut(key).unwrap(); diff --git a/crates/filesystem/src/web/mod.rs b/crates/filesystem/src/web/mod.rs index d8a5fc36..b5aadc6b 100644 --- a/crates/filesystem/src/web/mod.rs +++ b/crates/filesystem/src/web/mod.rs @@ -15,13 +15,17 @@ // You should have received a copy of the GNU General Public License // along with Luminol. If not, see . +use itertools::Itertools; + mod events; mod util; pub use events::setup_main_thread_hooks; use super::FileSystem as FileSystemTrait; use super::{DirEntry, Error, Metadata, OpenFlags, Result}; -use util::{send_and_await, send_and_recv}; +use std::io::ErrorKind::PermissionDenied; +use std::task::Poll; +use util::{generate_key, send_and_await, send_and_recv, send_and_wake}; static WORKER_CHANNELS: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); @@ -59,6 +63,15 @@ pub struct FileSystem { pub struct File { key: usize, temp_file_name: Option, + futures: parking_lot::Mutex, +} + +#[derive(Debug, Default)] +pub struct FileFutures { + read: Option>>>, + write: Option>>, + flush: Option>>, + seek: Option>>, } #[derive(Debug)] @@ -69,14 +82,14 @@ enum FileSystemCommand { camino::Utf8PathBuf, oneshot::Sender>, ), - DirPicker(oneshot::Sender)>>), + DirPicker(oneshot::Sender>), DirFromIdb(String, oneshot::Sender>), + DirToIdb(usize, String, oneshot::Sender), DirSubdir( usize, camino::Utf8PathBuf, - oneshot::Sender)>>, + oneshot::Sender>, ), - DirIdbDrop(String, oneshot::Sender), DirOpenFile( usize, camino::Utf8PathBuf, @@ -96,6 +109,12 @@ enum FileSystemCommand { DirClone(usize, oneshot::Sender), FileCreateTemp(oneshot::Sender>), FileSetLength(usize, u64, oneshot::Sender>), + FilePicker( + String, + Vec, + oneshot::Sender>, + ), + FileSave(usize, String, oneshot::Sender>), FileRead(usize, usize, oneshot::Sender>>), FileWrite(usize, Vec, oneshot::Sender>), FileFlush(usize, oneshot::Sender>), @@ -137,11 +156,16 @@ impl FileSystem { } send_and_await(|tx| FileSystemCommand::DirPicker(tx)) .await - .map(|(key, name, idb_key)| FileSystem { key, name, idb_key }) + .map(|(key, name)| Self { + key, + name, + idb_key: None, + }) .ok_or(Error::CancelledLoading) } - /// Attempts to restore a previously created `FileSystem` using its `.idb_key()`. + /// Attempts to restore a previously created `FileSystem` using its IndexedDB key returned by + /// `.save_to_idb()`. pub async fn from_idb_key(idb_key: String) -> Result { if !Self::filesystem_supported() { return Err(Error::Wasm32FilesystemNotSupported); @@ -159,12 +183,25 @@ impl FileSystem { /// Creates a new `FileSystem` from a subdirectory of this one. pub fn subdir(&self, path: impl AsRef) -> Result { send_and_recv(|tx| FileSystemCommand::DirSubdir(self.key, path.as_ref().to_path_buf(), tx)) - .map(|(key, name, idb_key)| FileSystem { key, name, idb_key }) + .map(|(key, name)| FileSystem { + key, + name, + idb_key: None, + }) } - /// Drops the directory with the given key from IndexedDB if it exists in there. - pub fn idb_drop(idb_key: String) -> bool { - send_and_recv(|tx| FileSystemCommand::DirIdbDrop(idb_key, tx)) + /// Stores this `FileSystem` to IndexedDB. If successful, consumes this `Filesystem` and + /// returns the key needed to restore this `FileSystem` using `FileSystem::from_idb()`. + /// Otherwise, returns ownership of this `FileSystem`. + pub fn save_to_idb(mut self) -> std::result::Result { + let idb_key_is_some = self.idb_key.is_some(); + let idb_key = self.idb_key.take().unwrap_or_else(generate_key); + if send_and_recv(|tx| FileSystemCommand::DirToIdb(self.key, idb_key.clone(), tx)) { + Ok(idb_key) + } else { + self.idb_key = idb_key_is_some.then_some(idb_key); + Err(self) + } } /// Returns a path consisting of a single element: the name of the root directory of this @@ -172,11 +209,6 @@ impl FileSystem { pub fn root_path(&self) -> &camino::Utf8Path { self.name.as_str().into() } - - /// Returns the key needed to restore this `FileSystem` using `FileSystem::from_idb()`. - pub fn idb_key(&self) -> Option<&str> { - self.idb_key.as_deref() - } } impl Drop for FileSystem { @@ -190,7 +222,7 @@ impl Clone for FileSystem { Self { key: send_and_recv(|tx| FileSystemCommand::DirClone(self.key, tx)), name: self.name.clone(), - idb_key: self.idb_key.clone(), + idb_key: None, } } } @@ -209,6 +241,7 @@ impl FileSystemTrait for FileSystem { .map(|key| File { key, temp_file_name: None, + futures: Default::default(), }) } @@ -262,9 +295,69 @@ impl File { Self { key, temp_file_name: Some(temp_file_name), + futures: Default::default(), } }) } + + /// Attempts to prompt the user to choose a file from their local machine using the + /// JavaScript File System API. + /// Then creates a `File` allowing read access to that file if they chose one + /// successfully. + /// If the File System API is not supported, this always returns `None` without doing anything. + /// + /// `extensions` should be a list of accepted file extensions for the file, without the leading + /// `.` + pub async fn from_file_picker( + filter_name: &str, + extensions: &[impl ToString], + ) -> Result<(Self, String)> { + if !FileSystem::filesystem_supported() { + return Err(Error::Wasm32FilesystemNotSupported); + } + send_and_await(|tx| { + FileSystemCommand::FilePicker( + filter_name.to_string(), + extensions.iter().map(|e| e.to_string()).collect_vec(), + tx, + ) + }) + .await + .map(|(key, name)| { + ( + Self { + key, + temp_file_name: None, + futures: Default::default(), + }, + name, + ) + }) + .ok_or(Error::CancelledLoading) + } + + /// Saves this file to a location of the user's choice. + /// + /// In native, this will open a file picker dialog, wait for the user to choose a location to + /// save a file, and then copy this file to the new location. This function will wait for the + /// user to finish picking a file location before returning. + /// + /// In web, this will use the browser's native file downloading method to save the file, which + /// may or may not open a file picker. Due to platform limitations, this function will return + /// immediately after making a download request and will not wait for the user to pick a file + /// location if a file picker is shown. + /// + /// You must flush the file yourself before saving. It will not be flushed for you. + /// + /// `filename` should be the default filename, with extension, to show in the file picker if + /// one is shown. `filter_name` should be the name of the file type shown in the part of the + /// file picker where the user selects a file extension. `filter_name` works only in native + /// builds; it is ignored in web builds. + pub async fn save(&self, filename: &str, _filter_name: &str) -> Result<()> { + send_and_await(|tx| FileSystemCommand::FileSave(self.key, filename.to_string(), tx)) + .await + .ok_or(Error::IoError(PermissionDenied.into())) + } } impl Drop for File { @@ -298,6 +391,34 @@ impl std::io::Read for File { } } +impl futures_lite::AsyncRead for File { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let mut futures = self.futures.lock(); + if futures.read.is_none() { + futures.read = Some(send_and_wake(cx, |tx| { + FileSystemCommand::FileRead(self.key, buf.len(), tx) + })); + } + match futures.read.as_mut().unwrap().try_recv() { + Ok(Ok(vec)) => { + futures.read = None; + let length = vec.len(); + buf[..length].copy_from_slice(&vec[..]); + Poll::Ready(Ok(length)) + } + Ok(Err(e)) => { + futures.read = None; + Poll::Ready(Err(e)) + } + Err(_) => Poll::Pending, + } + } +} + impl std::io::Write for File { fn write(&mut self, buf: &[u8]) -> std::io::Result { send_and_recv(|tx| FileSystemCommand::FileWrite(self.key, buf.to_vec(), tx))?; @@ -309,8 +430,90 @@ impl std::io::Write for File { } } +impl futures_lite::AsyncWrite for File { + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + let mut futures = self.futures.lock(); + if futures.write.is_none() { + futures.write = Some(send_and_wake(cx, |tx| { + FileSystemCommand::FileWrite(self.key, buf.to_vec(), tx) + })); + } + match futures.write.as_mut().unwrap().try_recv() { + Ok(Ok(())) => { + futures.write = None; + Poll::Ready(Ok(buf.len())) + } + Ok(Err(e)) => { + futures.write = None; + Poll::Ready(Err(e)) + } + Err(_) => Poll::Pending, + } + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + let mut futures = self.futures.lock(); + if futures.flush.is_none() { + futures.flush = Some(send_and_wake(cx, |tx| { + FileSystemCommand::FileFlush(self.key, tx) + })); + } + match futures.flush.as_mut().unwrap().try_recv() { + Ok(Ok(())) => { + futures.flush = None; + Poll::Ready(Ok(())) + } + Ok(Err(e)) => { + futures.flush = None; + Poll::Ready(Err(e)) + } + Err(_) => Poll::Pending, + } + } + + fn poll_close( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> Poll> { + Poll::Ready(Err(PermissionDenied.into())) + } +} + impl std::io::Seek for File { fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { send_and_recv(|tx| FileSystemCommand::FileSeek(self.key, pos, tx)) } } + +impl futures_lite::AsyncSeek for File { + fn poll_seek( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + pos: std::io::SeekFrom, + ) -> Poll> { + let mut futures = self.futures.lock(); + if futures.seek.is_none() { + futures.seek = Some(send_and_wake(cx, |tx| { + FileSystemCommand::FileSeek(self.key, pos, tx) + })); + } + match futures.seek.as_mut().unwrap().try_recv() { + Ok(Ok(offset)) => { + futures.seek = None; + Poll::Ready(Ok(offset)) + } + Ok(Err(e)) => { + futures.seek = None; + Poll::Ready(Err(e)) + } + Err(_) => Poll::Pending, + } + } +} diff --git a/crates/filesystem/src/web/util.rs b/crates/filesystem/src/web/util.rs index a3d9b5da..549cdc7e 100644 --- a/crates/filesystem/src/web/util.rs +++ b/crates/filesystem/src/web/util.rs @@ -99,7 +99,7 @@ pub(super) fn generate_key() -> String { /// `IdbTransactionMode`. pub(super) async fn idb( mode: IdbTransactionMode, - f: impl Fn(IdbObjectStore<'_>) -> std::result::Result, + f: impl FnOnce(IdbObjectStore<'_>) -> std::result::Result, ) -> std::result::Result { let mut db_req = IdbDatabase::open_u32("astrabit.luminol", 1)?; @@ -153,3 +153,27 @@ pub(super) async fn send_and_await( ) -> R { send(f).await.unwrap() } + +/// Helper function to send a filesystem command from the worker thread to the main thread, wait +/// asynchronously for the response, send the response to the receiver returned by this function +/// and then wake up a task. +pub(super) fn send_and_wake( + cx: &std::task::Context<'_>, + f: impl FnOnce(oneshot::Sender) -> super::FileSystemCommand, +) -> oneshot::Receiver +where + R: 'static, +{ + let command_rx = send(f); + let (task_tx, task_rx) = oneshot::channel(); + + let waker = cx.waker().clone(); + + wasm_bindgen_futures::spawn_local(async move { + let response = command_rx.await.unwrap(); + let _ = task_tx.send(response); + waker.wake(); + }); + + task_rx +} diff --git a/crates/ui/Cargo.toml b/crates/ui/Cargo.toml index d7fc40e8..b194e50e 100644 --- a/crates/ui/Cargo.toml +++ b/crates/ui/Cargo.toml @@ -31,16 +31,24 @@ egui_extras.workspace = true catppuccin-egui = { version = "3.1.0", git = "https://github.com/catppuccin/egui", rev = "bcb5849b6f96b56aa4982ec3366e238371de473e" } +camino.workspace = true + strum.workspace = true git-version = "0.3.5" poll-promise.workspace = true +async-std.workspace = true +pin-project.workspace = true +futures-lite.workspace = true futures = "0.3.28" reqwest = "0.11.22" zip = { version = "0.6.6", default-features = false, features = ["deflate"] } +once_cell.workspace = true +qp-trie.workspace = true + itertools.workspace = true anyhow.workspace = true diff --git a/crates/ui/src/windows/archive_manager.rs b/crates/ui/src/windows/archive_manager.rs new file mode 100644 index 00000000..2da6e488 --- /dev/null +++ b/crates/ui/src/windows/archive_manager.rs @@ -0,0 +1,581 @@ +// Copyright (C) 2023 Lily Lyons +// +// This file is part of Luminol. +// +// Luminol is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Luminol is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Luminol. If not, see . +// +// Additional permission under GNU GPL version 3 section 7 +// +// If you modify this Program, or any covered work, by linking or combining +// it with Steamworks API by Valve Corporation, containing parts covered by +// terms of the Steamworks API by Valve Corporation, the licensors of this +// Program grant you additional permission to convey the resulting work. + +use luminol_filesystem::{File, FileSystem, OpenFlags}; + +static CREATE_DEFAULT_SELECTED_DIRS: once_cell::sync::Lazy< + qp_trie::Trie, +> = once_cell::sync::Lazy::new(|| { + let mut trie = qp_trie::Trie::new(); + trie.insert_str("Data", ()); + trie.insert_str("Graphics", ()); + trie +}); + +/// The archive manager for creating and extracting RGSSAD archives. +pub struct Window { + mode: Mode, + initialized: bool, + progress: std::sync::Arc, +} + +enum Mode { + Extract { + view: Option< + luminol_components::FileSystemView< + luminol_filesystem::archiver::FileSystem, + >, + >, + load_promise: Option< + poll_promise::Promise< + luminol_filesystem::Result<(luminol_filesystem::host::File, String)>, + >, + >, + save_promise: Option>>, + progress_total: usize, + }, + Create { + view: Option>, + load_promise: Option< + poll_promise::Promise>, + >, + save_promise: Option>>, + version: u8, + progress_total: usize, + }, +} + +impl Default for Window { + fn default() -> Self { + Self { + mode: Mode::Extract { + view: None, + load_promise: None, + save_promise: None, + progress_total: 0, + }, + initialized: false, + progress: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(usize::MAX)), + } + } +} + +impl luminol_core::Window for Window { + fn id(&self) -> egui::Id { + egui::Id::new("RGSSAD Archive Manager") + } + + fn show( + &mut self, + ctx: &egui::Context, + open: &mut bool, + update_state: &mut luminol_core::UpdateState<'_>, + ) { + // Open the currently loaded project by default + if !self.initialized { + self.initialized = true; + if let Some(host) = update_state.filesystem.host() { + match &mut self.mode { + Mode::Extract { view, .. } => { + if let Ok(Some((entry, archive))) = (|| { + host.read_dir("")? + .into_iter() + .find(|entry| { + entry.metadata.is_file + && matches!( + entry.path.extension(), + Some("rgssad" | "rgss2a" | "rgss3a") + ) + }) + .map(|entry| { + host.open_file(&entry.path, OpenFlags::Read) + .and_then(luminol_filesystem::archiver::FileSystem::new) + .map(|archive| (entry, archive)) + }) + .transpose() + })() { + *view = Some(luminol_components::FileSystemView::new( + "luminol_archive_manager_extract_view".into(), + archive, + entry.path.to_string(), + )) + } + } + + Mode::Create { view, .. } => { + let name = host.root_path().to_string(); + *view = Some(luminol_components::FileSystemView::new( + "luminol_archive_manager_create_view".into(), + host, + name, + )); + } + } + } + } + + let mut window_open = true; + egui::Window::new("RGSSAD Archive Manager") + .open(&mut window_open) + .show(ctx, |ui| { + let enabled = match &self.mode { + Mode::Extract { + load_promise, + save_promise, + .. + } => load_promise.is_none() && save_promise.is_none(), + Mode::Create { + load_promise, + save_promise, + .. + } => load_promise.is_none() && save_promise.is_none(), + }; + ui.add_enabled_ui(enabled, |ui| { + ui.columns(2, |columns| { + if columns[0] + .add(egui::SelectableLabel::new( + matches!(self.mode, Mode::Extract { .. }), + "Extract from archive", + )) + .clicked() + { + self.initialized = false; + self.progress = std::sync::Arc::new( + std::sync::atomic::AtomicUsize::new(usize::MAX), + ); + self.mode = Mode::Extract { + view: None, + load_promise: None, + save_promise: None, + progress_total: 0, + }; + } + if columns[1] + .add(egui::SelectableLabel::new( + matches!(self.mode, Mode::Create { .. }), + "Create new archive", + )) + .clicked() + { + self.initialized = false; + self.progress = std::sync::Arc::new( + std::sync::atomic::AtomicUsize::new(usize::MAX), + ); + self.mode = Mode::Create { + view: None, + load_promise: None, + save_promise: None, + version: 1, + progress_total: 0, + }; + } + }); + + ui.separator(); + + self.show_inner(ui, update_state); + + ui.with_layout( + egui::Layout { + cross_justify: true, + ..Default::default() + }, + |ui| { + ui.group(|ui| { + ui.set_width(ui.available_width()); + ui.set_height(ui.available_height()); + egui::ScrollArea::both().show(ui, |ui| match &mut self.mode { + Mode::Extract { view, .. } => { + if let Some(v) = view { + v.ui(ui, update_state, None); + } else { + ui.add( + egui::Label::new("No archive chosen").wrap(false), + ); + } + } + Mode::Create { view, .. } => { + if let Some(v) = view { + v.ui( + ui, + update_state, + Some(&CREATE_DEFAULT_SELECTED_DIRS), + ); + } else { + ui.add( + egui::Label::new("No source folder chosen") + .wrap(false), + ); + } + } + }); + }); + }, + ); + }); + }); + + *open = window_open; + } + + fn requires_filesystem(&self) -> bool { + false + } +} + +impl Window { + fn show_inner(&mut self, ui: &mut egui::Ui, update_state: &mut luminol_core::UpdateState<'_>) { + let progress = self.progress.clone(); + + match &mut self.mode { + Mode::Extract { + view, + load_promise, + save_promise, + progress_total, + } => { + if let Some(p) = load_promise.take() { + match p.try_take() { + Ok(Ok((handle, name))) => { + match luminol_filesystem::archiver::FileSystem::new(handle) { + Ok(archiver) => { + *view = Some(luminol_components::FileSystemView::new( + "luminol_archive_manager_extract_view".into(), + archiver, + name, + )) + } + Err(e) => update_state.toasts.error(e.to_string()), + } + } + Ok(Err(e)) => { + if !matches!(e, luminol_filesystem::Error::CancelledLoading) { + update_state.toasts.error(e.to_string()) + } + } + Err(p) => *load_promise = Some(p), + } + } + + let progress_amount = progress.load(std::sync::atomic::Ordering::Relaxed); + + if progress_amount == usize::MAX + || progress_amount == *progress_total + || save_promise.is_none() + { + ui.columns(2, |columns| { + columns[0].with_layout( + egui::Layout { + cross_align: egui::Align::Center, + cross_justify: true, + ..Default::default() + }, + |ui| { + if load_promise.is_none() && ui.button("Choose archive").clicked() { + *load_promise = Some(luminol_core::spawn_future( + luminol_filesystem::host::File::from_file_picker( + "RGSSAD archives", + &["rgssad", "rgss2a", "rgss3a"], + ), + )); + } else if load_promise.is_some() { + ui.spinner(); + } + }, + ); + + columns[1].with_layout( + egui::Layout { + cross_align: egui::Align::Center, + cross_justify: true, + ..Default::default() + }, + |ui| { + if save_promise.is_none() + && ui + .add_enabled( + view.as_ref() + .is_some_and(|view| view.iter().next().is_some()), + egui::Button::new("Extract selected files"), + ) + .clicked() + { + let view = view.as_ref().unwrap(); + match Self::find_files(view) { + Ok(file_paths) => { + let ctx = ui.ctx().clone(); + let progress = progress.clone(); + let view_filesystem = view.filesystem().clone(); + *progress_total = file_paths.len(); + progress.store(usize::MAX, std::sync::atomic::Ordering::Relaxed); + + *save_promise = Some(luminol_core::spawn_future(async move { + let dest_fs = luminol_filesystem::host::FileSystem::from_folder_picker().await?; + progress.store(0, std::sync::atomic::Ordering::Relaxed); + ctx.request_repaint(); + + for path in file_paths { + if let Some(parent) = path.parent() { + dest_fs.create_dir(parent)?; + } + let mut src_file = view_filesystem.open_file(&path, OpenFlags::Read)?; + let mut dest_file = dest_fs.open_file(&path, OpenFlags::Write | OpenFlags::Create | OpenFlags::Truncate)?; + async_std::io::copy(&mut src_file, &mut dest_file).await?; + + progress.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + ctx.request_repaint(); + } + + Ok(()) + })); + } + Err(e) => update_state.toasts.error(e.to_string()), + } + } else if save_promise.is_some() { + ui.spinner(); + } + }, + ); + }); + } else { + ui.add( + egui::ProgressBar::new(if *progress_total == 0 { + 0. + } else { + (progress_amount as f64 / *progress_total as f64) as f32 + }) + .show_percentage(), + ); + } + + if let Some(p) = save_promise.take() { + match p.try_take() { + Ok(Ok(())) => update_state.toasts.info("Extracted successfully!"), + Ok(Err(e)) => { + if !matches!(e, luminol_filesystem::Error::CancelledLoading) { + update_state.toasts.error(e.to_string()) + } + } + Err(p) => *save_promise = Some(p), + } + } + } + + Mode::Create { + view, + load_promise, + save_promise, + version, + progress_total, + } => { + if let Some(p) = load_promise.take() { + match p.try_take() { + Ok(Ok(handle)) => { + let name = handle.root_path().to_string(); + *view = Some(luminol_components::FileSystemView::new( + "luminol_archive_manager_create_view".into(), + handle, + name, + )); + } + Ok(Err(e)) => { + if !matches!(e, luminol_filesystem::Error::CancelledLoading) { + update_state.toasts.error(e.to_string()) + } + } + Err(p) => *load_promise = Some(p), + } + } + + ui.horizontal(|ui| { + ui.label("Version:"); + ui.columns(4, |columns| { + columns[1].radio_value(version, 1, "XP"); + columns[2].radio_value(version, 2, "VX"); + columns[3].radio_value(version, 3, "VX Ace"); + }); + }); + + ui.separator(); + + let progress_amount = progress.load(std::sync::atomic::Ordering::Relaxed); + + if progress_amount == usize::MAX + || progress_amount == *progress_total + || save_promise.is_none() + { + ui.columns(2, |columns| { + columns[0].with_layout( + egui::Layout { + cross_align: egui::Align::Center, + cross_justify: true, + ..Default::default() + }, + |ui| { + if load_promise.is_none() && ui.button("Choose source folder").clicked() + { + *load_promise = Some(luminol_core::spawn_future( + luminol_filesystem::host::FileSystem::from_folder_picker(), + )); + } else if load_promise.is_some() { + ui.spinner(); + } + }, + ); + + columns[1].with_layout( + egui::Layout { + cross_align: egui::Align::Center, + cross_justify: true, + ..Default::default() + }, + |ui| { + if save_promise.is_none() + && ui + .add_enabled( + view.as_ref() + .is_some_and(|view| view.iter().next().is_some()), + egui::Button::new("Create from selected files"), + ) + .clicked() + { + if let Some(view) = view { + let version = *version; + match Self::find_files(view) { + Ok(file_paths) => { + let ctx = ui.ctx().clone(); + let progress = progress.clone(); + let view_filesystem = view.filesystem().clone(); + *progress_total = file_paths.len(); + progress.store(usize::MAX, std::sync::atomic::Ordering::Relaxed); + + *save_promise = + Some(luminol_core::spawn_future(async move { + let mut file = luminol_filesystem::host::File::new()?; + + let mut is_first = true; + + progress.store(0, std::sync::atomic::Ordering::Relaxed); + ctx.request_repaint(); + + let _ = luminol_filesystem::archiver::FileSystem::from_buffer_and_files( + &mut file, + version, + file_paths.iter().map(|path| { + if is_first { + is_first = false; + } else { + progress.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + ctx.request_repaint(); + } + + let file = view_filesystem.open_file(path, OpenFlags::Read)?; + let size = file.metadata()?.size as u32; + Ok((path, size, file)) + }), + ).await?; + + progress.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + ctx.request_repaint(); + + file.save( + match version { + 1 => "Game.rgssad", + 2 => "Game.rgss2a", + 3 => "Game.rgss3a", + _ => unreachable!(), + }, + "RGSSAD archives", + ) + .await + })); + } + Err(e) => update_state.toasts.error(e.to_string()), + } + } + } else if save_promise.is_some() { + ui.spinner(); + } + }, + ); + }); + } else { + ui.add( + egui::ProgressBar::new(if *progress_total == 0 { + 0. + } else { + (progress_amount as f64 / *progress_total as f64) as f32 + }) + .show_percentage(), + ); + } + + if let Some(p) = save_promise.take() { + match p.try_take() { + Ok(Ok(())) => { + update_state.toasts.info("Created archive successfully!"); + } + Ok(Err(e)) => { + if !matches!(e, luminol_filesystem::Error::CancelledLoading) { + update_state.toasts.error(e.to_string()) + } + } + Err(p) => *save_promise = Some(p), + } + } + } + } + } + + fn find_files( + view: &luminol_components::FileSystemView, + ) -> luminol_filesystem::Result> { + let mut vec = Vec::new(); + for metadata in view { + Self::find_files_recurse( + &mut vec, + view.filesystem(), + metadata.path.as_str().into(), + metadata.is_file, + )?; + } + Ok(vec) + } + + fn find_files_recurse( + vec: &mut Vec, + src_fs: &impl luminol_filesystem::FileSystem, + path: &camino::Utf8Path, + is_file: bool, + ) -> luminol_filesystem::Result<()> { + if is_file { + vec.push(path.to_owned()); + } else { + for entry in src_fs.read_dir(path)? { + Self::find_files_recurse(vec, src_fs, &entry.path, entry.metadata.is_file)?; + } + } + Ok(()) + } +} diff --git a/crates/ui/src/windows/mod.rs b/crates/ui/src/windows/mod.rs index d6991ad1..5f103472 100644 --- a/crates/ui/src/windows/mod.rs +++ b/crates/ui/src/windows/mod.rs @@ -25,6 +25,8 @@ /// The about window. pub mod about; pub mod appearance; +/// The archive manager for creating and extracting RGSSAD archives. +pub mod archive_manager; /// The common event editor. pub mod common_event_edit; /// Config window diff --git a/crates/ui/src/windows/new_project.rs b/crates/ui/src/windows/new_project.rs index 82a0bc62..416cbde7 100644 --- a/crates/ui/src/windows/new_project.rs +++ b/crates/ui/src/windows/new_project.rs @@ -181,38 +181,17 @@ impl luminol_core::Window for Window { let branch_name = self.git_branch_name.clone(); - #[cfg(not(target_arch = "wasm32"))] - { - update_state - .project_manager - .run_custom(move |update_state| { - update_state.project_manager.create_project_promise = - Some(poll_promise::Promise::spawn_async( - Self::setup_project( - config, - download_executable, - init_git.then_some(branch_name), - progress, - ), - )); - }); - } - #[cfg(target_arch = "wasm32")] - { - update_state - .project_manager - .run_custom(move |update_state| { - update_state.project_manager.create_project_promise = - Some(poll_promise::Promise::spawn_local( - Self::setup_project( - config, - download_executable, - init_git.then_some(branch_name), - progress, - ), - )); - }); - } + update_state + .project_manager + .run_custom(move |update_state| { + update_state.project_manager.create_project_promise = + Some(luminol_core::spawn_future(Self::setup_project( + config, + download_executable, + init_git.then_some(branch_name), + progress, + ))); + }); } if ui.button("Cancel").clicked() { *open = false; diff --git a/crates/web/js/bindings.js b/crates/web/js/bindings.js index 91adb10f..4489509e 100644 --- a/crates/web/js/bindings.js +++ b/crates/web/js/bindings.js @@ -41,6 +41,16 @@ export async function _show_directory_picker() { return await window.showDirectoryPicker({ mode: 'readwrite' }); } +export async function _show_file_picker(filter_name, extensions) { + return (await window.showOpenFilePicker({ + types: [{ + description: filter_name, + accept: { 'application/x-empty': extensions }, + }], + excludeAcceptAllOption: true, + }))[0]; +} + export function dir_values(dir) { return dir.values(); } diff --git a/crates/web/src/bindings.rs b/crates/web/src/bindings.rs index e6d7b311..12ec599b 100644 --- a/crates/web/src/bindings.rs +++ b/crates/web/src/bindings.rs @@ -23,6 +23,11 @@ extern "C" { pub fn filesystem_supported() -> bool; #[wasm_bindgen(catch)] async fn _show_directory_picker() -> Result; + #[wasm_bindgen(catch)] + async fn _show_file_picker( + filter_name: &str, + extensions: &js_sys::Array, + ) -> Result; pub fn dir_values(dir: &web_sys::FileSystemDirectoryHandle) -> js_sys::AsyncIterator; async fn _request_permission(handle: &web_sys::FileSystemHandle) -> JsValue; pub fn cross_origin_isolated() -> bool; @@ -35,6 +40,20 @@ pub async fn show_directory_picker() -> Result], +) -> Result { + let array = js_sys::Array::new(); + for extension in extensions { + array.push(&JsValue::from(format!(".{}", extension.as_ref()))); + } + _show_file_picker(filter_name, &array) + .await + .map(|o| o.unchecked_into()) + .map_err(|e| e.unchecked_into()) +} + pub async fn request_permission(handle: &web_sys::FileSystemHandle) -> bool { _request_permission(handle).await.is_truthy() } diff --git a/src/app/top_bar.rs b/src/app/top_bar.rs index 3874dba6..efc3c2e4 100644 --- a/src/app/top_bar.rs +++ b/src/app/top_bar.rs @@ -39,10 +39,13 @@ impl TopBar { #[cfg(not(target_arch = "wasm32"))] { + let old_fullscreen = self.fullscreen; ui.checkbox(&mut self.fullscreen, "Fullscreen"); - update_state - .ctx - .send_viewport_cmd(egui::ViewportCommand::Fullscreen(self.fullscreen)); + if self.fullscreen != old_fullscreen { + update_state + .ctx + .send_viewport_cmd(egui::ViewportCommand::Fullscreen(self.fullscreen)); + } } let mut open_project = ui.input(|i| i.modifiers.command && i.key_pressed(egui::Key::O)) @@ -193,6 +196,23 @@ impl TopBar { ui.separator(); + ui.menu_button("Tools", |ui| { + // Hide this menu if the unsaved changes modal or a file/folder picker is open + if update_state.project_manager.is_modal_open() + || update_state.project_manager.is_picker_open() + { + ui.close_menu(); + } + + if ui.button("RGSSAD Archive Manager").clicked() { + update_state + .edit_windows + .add_window(luminol_ui::windows::archive_manager::Window::default()); + } + }); + + ui.separator(); + ui.menu_button("Help", |ui| { // Hide this menu if the unsaved changes modal or a file/folder picker is open if update_state.project_manager.is_modal_open()