From b41a6caa295b53becdd9514e8218cecc6fe79905 Mon Sep 17 00:00:00 2001 From: Xiphoseer Date: Thu, 28 Oct 2021 00:10:44 +0200 Subject: [PATCH] Start work on CRC/ResMgr subsystem --- Cargo.toml | 13 +- src/api/adapter.rs | 53 +++++-- src/api/files.rs | 47 +++++++ src/api/mod.rs | 102 ++++++++------ src/api/rev/behaviors.rs | 13 +- src/api/rev/common.rs | 48 ++++++- src/api/rev/data.rs | 8 +- src/api/rev/loot_table_index.rs | 4 +- src/api/rev/missions.rs | 26 ++-- src/api/rev/mod.rs | 5 +- src/api/rev/object_types.rs | 5 +- src/api/rev/skills.rs | 4 +- src/api/tables.rs | 2 +- src/config.rs | 4 + src/data/fs.rs | 238 ++++++++++++++++++++++++++++++++ src/data/locale.rs | 64 +++++++++ src/data/maps.rs | 1 + src/data/mod.rs | 5 +- src/data/skill_system.rs | 2 +- src/main.rs | 61 ++++++-- src/template/mod.rs | 65 ++++++--- 21 files changed, 640 insertions(+), 130 deletions(-) create mode 100644 src/api/files.rs create mode 100644 src/data/fs.rs create mode 100644 src/data/locale.rs create mode 100644 src/data/maps.rs diff --git a/Cargo.toml b/Cargo.toml index 27781a2..f8f8fcf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,11 @@ authors = ["Xiphoseer "] edition = "2018" [dependencies] +assembly-core = { git = "https://github.com/Xiphoseer/assembly_rs.git" } +assembly-maps = { git = "https://github.com/Xiphoseer/assembly_rs.git" } +assembly-pack = { git = "https://github.com/Xiphoseer/assembly_rs.git" } +assembly-fdb = { git = "https://github.com/Xiphoseer/assembly_rs.git", default-features = false, features = ["serde-derives"] } +assembly-xml = { git = "https://github.com/Xiphoseer/assembly_rs.git" } base64 = "0.13" handlebars = "3.5" pretty_env_logger = "0.4.0" @@ -27,14 +32,6 @@ version = "0.3" features = ["tls", "multipart"] default-features = false -[dependencies.assembly-core] -version = "0.2.1" - -[dependencies.assembly-data] -version = "0.3.0" -default-features = false -features = ["serde-derives"] - # Holds data in insertion order [dependencies.linked-hash-map] version = "0.5.3" diff --git a/src/api/adapter.rs b/src/api/adapter.rs index 1db8df7..c633d04 100644 --- a/src/api/adapter.rs +++ b/src/api/adapter.rs @@ -2,8 +2,8 @@ use std::iter::Copied; use std::slice::Iter; use std::{collections::BTreeMap, fmt}; -use assembly_data::xml::localization::LocaleNode; -use paradox_typed_db::typed_rows::TypedRow; +use assembly_xml::localization::LocaleNode; +use paradox_typed_db::TypedRow; use serde::{ser::SerializeMap, Serialize}; pub(crate) trait FindHash { @@ -90,24 +90,53 @@ where } } +pub(crate) struct TableMultiIter<'a, 'b, R, K, F> +where + K: Iterator, + F: FindHash, + R: TypedRow<'a, 'b>, +{ + pub(crate) index: F, + pub(crate) key_iter: K, + pub(crate) table: &'b R::Table, + pub(crate) id_col: usize, +} + +impl<'a, 'b, R, K, F> Iterator for TableMultiIter<'a, 'b, R, K, F> +where + K: Iterator, + F: FindHash, + R: TypedRow<'a, 'b>, +{ + type Item = (i32, R); + + fn next(&mut self) -> Option { + for key in &mut self.key_iter { + if let Some(hash) = self.index.find_hash(key) { + if let Some(r) = R::get(self.table, hash, key, self.id_col) { + return Some((key, r)); + } + } + } + None + } +} + impl<'b, 'a: 'b, R, F, K> TypedTableIterAdapter<'a, 'b, R, F, K> where R: TypedRow<'a, 'b> + 'b, { - pub(crate) fn to_iter(&self, id_col: usize) -> impl Iterator + 'b + pub(crate) fn to_iter(&self, id_col: usize) -> TableMultiIter<'a, 'b, R, K::IntoIter, F> where F: FindHash + Copy + 'b, K: IntoIterator + Clone + 'b, { - let table: &'b R::Table = self.table; - let i = self.index; - let iter = self.keys.clone().into_iter(); - let mapper = move |key| { - let hash = i.find_hash(key)?; - let r = R::get(table, hash, key, id_col)?; - Some((key, r)) - }; - iter.filter_map(mapper) + TableMultiIter { + index: self.index, + key_iter: self.keys.clone().into_iter(), + table: self.table, + id_col, + } } } diff --git a/src/api/files.rs b/src/api/files.rs new file mode 100644 index 0000000..e9727f7 --- /dev/null +++ b/src/api/files.rs @@ -0,0 +1,47 @@ +use assembly_pack::pki::core::PackFileRef; +use color_eyre::eyre::Context; +use serde::Serialize; +use std::{path::Path, sync::Arc}; +use tracing::error; + +use warp::{ + filters::BoxedFilter, + hyper::StatusCode, + reply::{json, with_status, Json, WithStatus}, + Filter, +}; + +use crate::data::fs::{Loader, Node}; + +#[derive(Serialize)] +struct CRCReply<'a> { + fs: Option<&'a Node>, + pk: Option<&'a PackFileRef>, +} + +/// Lookup information on a CRC i.e. a file path in the client +pub fn make_crc_lookup_filter( + res_path: &Path, + pki_path: Option<&Path>, +) -> BoxedFilter<(WithStatus,)> { + let mut loader = Loader::new(); + loader.load_dir(Path::new("client/res"), res_path); + if let Some(pki_path) = pki_path { + if let Err(e) = loader + .load_pki(pki_path) + .with_context(|| format!("Failed to load PKI file at '{}'", pki_path.display())) + { + error!("{}", e); + } + } + + let loader = Arc::new(loader); + + warp::path::param() + .map(move |crc: u32| { + let fs = loader.get(crc).map(|e| &e.public); + let pk = loader.get_pki(crc); + with_status(json(&CRCReply { fs, pk }), StatusCode::OK) + }) + .boxed() +} diff --git a/src/api/mod.rs b/src/api/mod.rs index 7c1ddb6..102afc5 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -2,11 +2,13 @@ use std::{ borrow::Borrow, convert::Infallible, error::Error, + path::Path, str::{FromStr, Utf8Error}, sync::Arc, }; -use assembly_data::{fdb::mem::Database, xml::localization::LocaleNode}; +use assembly_fdb::mem::Database; +use assembly_xml::localization::LocaleNode; use paradox_typed_db::TypedDatabase; use percent_encoding::percent_decode_str; use warp::{ @@ -16,16 +18,18 @@ use warp::{ Filter, Reply, }; -use crate::auth::AuthKind; +use crate::{auth::AuthKind, data::locale::LocaleRoot}; use self::{ adapter::{LocaleAll, LocalePod}, + files::make_crc_lookup_filter, rev::{make_api_rev, ReverseLookup}, tables::{make_api_tables, tables_api}, }; pub mod adapter; mod docs; +mod files; pub mod rev; pub mod tables; @@ -149,45 +153,57 @@ pub fn locale_api(lr: Arc) -> impl Fn(Tail) -> Option, - tydb: &'static TypedDatabase<'static>, - rev: &'static ReverseLookup, - lr: Arc, -) -> BoxedFilter<(WithStatus,)> { - let v0_base = warp::path("v0"); - let v0_tables = warp::path("tables").and(make_api_tables(db)); - let v0_locale = warp::path("locale") - .and(warp::path::tail()) - .map(locale_api(lr)) - .map(map_opt); - - let v0_rev = warp::path("rev").and(make_api_rev(tydb, rev)); - let v0_openapi = docs::openapi(url, auth_kind).unwrap(); - let v0 = v0_base.and( - v0_tables - .or(v0_locale) - .unify() - .or(v0_rev) - .unify() - .or(v0_openapi) - .unify(), - ); - - // v1 - let dbf = db_filter(db); - let v1_base = warp::path("v1"); - let v1_tables_base = dbf.and(warp::path("tables")); - let v1_tables = v1_tables_base - .and(warp::path::end()) - .map(tables_api) - .map(map_res); - let v1 = v1_base.and(v1_tables); - - // catch all - let catch_all = make_api_catch_all(); - - v0.or(v1).unify().or(catch_all).unify().boxed() +pub(crate) struct ApiFactory<'a> { + pub url: String, + pub auth_kind: AuthKind, + pub db: Database<'static>, + pub tydb: &'static TypedDatabase<'static>, + pub rev: &'static ReverseLookup, + pub lr: Arc, + pub res_path: &'a Path, + pub pki_path: Option<&'a Path>, +} + +impl<'a> ApiFactory<'a> { + pub(crate) fn make_api(self) -> BoxedFilter<(WithStatus,)> { + let loc = LocaleRoot::new(self.lr.clone()); + + let v0_base = warp::path("v0"); + let v0_tables = warp::path("tables").and(make_api_tables(self.db)); + let v0_locale = warp::path("locale") + .and(warp::path::tail()) + .map(locale_api(self.lr)) + .map(map_opt); + + let v0_crc = warp::path("crc").and(make_crc_lookup_filter(self.res_path, self.pki_path)); + + let v0_rev = warp::path("rev").and(make_api_rev(self.tydb, loc, self.rev)); + let v0_openapi = docs::openapi(self.url, self.auth_kind).unwrap(); + let v0 = v0_base.and( + v0_tables + .or(v0_crc) + .unify() + .or(v0_locale) + .unify() + .or(v0_rev) + .unify() + .or(v0_openapi) + .unify(), + ); + + // v1 + let dbf = db_filter(self.db); + let v1_base = warp::path("v1"); + let v1_tables_base = dbf.and(warp::path("tables")); + let v1_tables = v1_tables_base + .and(warp::path::end()) + .map(tables_api) + .map(map_res); + let v1 = v1_base.and(v1_tables); + + // catch all + let catch_all = make_api_catch_all(); + + v0.or(v1).unify().or(catch_all).unify().boxed() + } } diff --git a/src/api/rev/behaviors.rs b/src/api/rev/behaviors.rs index 14fb12f..6751e06 100644 --- a/src/api/rev/behaviors.rs +++ b/src/api/rev/behaviors.rs @@ -1,8 +1,9 @@ use assembly_core::buffer::CastError; use paradox_typed_db::{ - typed_rows::{BehaviorTemplateRow, TypedRow}, - typed_tables::{BehaviorParameterTable, BehaviorTemplateTable}, - TypedDatabase, + columns::BehaviorTemplateColumn, + rows::BehaviorTemplateRow, + tables::{BehaviorParameterTable, BehaviorTemplateTable}, + TypedDatabase, TypedRow, }; use serde::ser::SerializeMap; use serde::Serialize; @@ -56,6 +57,10 @@ impl Serialize for EmbeddedBehaviors<'_, '_> { S: serde::Serializer, { let mut m = serializer.serialize_map(Some(self.keys.len()))?; + let col_behavior_id = self + .table_templates + .get_col(BehaviorTemplateColumn::BehaviorId) + .unwrap(); for &behavior_id in self.keys { m.serialize_key(&behavior_id)?; let b = Behavior { @@ -63,7 +68,7 @@ impl Serialize for EmbeddedBehaviors<'_, '_> { self.table_templates, behavior_id, behavior_id, - self.table_templates.col_behavior_id, + col_behavior_id, ), parameters: BehaviorParameters { key: behavior_id, diff --git a/src/api/rev/common.rs b/src/api/rev/common.rs index 92c2cea..ac094cc 100644 --- a/src/api/rev/common.rs +++ b/src/api/rev/common.rs @@ -1,12 +1,16 @@ use std::collections::HashMap; +use assembly_fdb::common::Latin1Str; use paradox_typed_db::{ - typed_rows::{MissionTaskRow, ObjectsRef}, - typed_tables::MissionTasksTable, + columns::ObjectsColumn, + rows::{MissionTasksRow, ObjectsRow}, + tables::{MissionTasksTable, ObjectsTable}, }; use serde::Serialize; -use crate::api::adapter::{FindHash, I32Slice, IdentityHash, TypedTableIterAdapter}; +use crate::api::adapter::{ + FindHash, I32Slice, IdentityHash, TableMultiIter, TypedTableIterAdapter, +}; use super::data::MissionTaskUIDLookup; @@ -16,8 +20,40 @@ pub struct MapFilter<'a, E> { keys: &'a [i32], } -pub(super) type ObjectsRefAdapter<'a, 'b> = - TypedTableIterAdapter<'a, 'b, ObjectsRef<'a, 'b>, IdentityHash, I32Slice<'b>>; +#[derive(Clone)] +pub struct ObjectsRefAdapter<'a, 'b> { + table: &'b ObjectsTable<'a>, + keys: &'b [i32], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize)] +struct ObjectRefData<'a> { + name: &'a Latin1Str, +} + +impl<'a, 'b> ObjectsRefAdapter<'a, 'b> { + pub fn new(table: &'b ObjectsTable<'a>, keys: &'b [i32]) -> Self { + Self { table, keys } + } +} + +impl<'a, 'b> serde::Serialize for ObjectsRefAdapter<'a, 'b> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let id_col = self.table.get_col(ObjectsColumn::Id).unwrap(); + serializer.collect_map( + TableMultiIter { + index: IdentityHash, + key_iter: self.keys.iter().copied(), + table: self.table, + id_col, + } + .map(|(id, row): (i32, ObjectsRow)| (id, ObjectRefData { name: row.name() })), + ) + } +} #[derive(Serialize)] pub(super) struct ObjectTypeEmbedded<'a, 'b> { @@ -44,7 +80,7 @@ impl<'a, E: Serialize> Serialize for MapFilter<'a, E> { pub(super) type MissionTaskHash<'b> = &'b HashMap; pub(super) type MissionTasks<'a, 'b> = - TypedTableIterAdapter<'a, 'b, MissionTaskRow<'a, 'b>, MissionTaskHash<'b>, I32Slice<'b>>; + TypedTableIterAdapter<'a, 'b, MissionTasksRow<'a, 'b>, MissionTaskHash<'b>, I32Slice<'b>>; pub(super) struct MissionTaskIconsAdapter<'a, 'b> { table: &'b MissionTasksTable<'a>, diff --git a/src/api/rev/data.rs b/src/api/rev/data.rs index 3df6218..70745a6 100644 --- a/src/api/rev/data.rs +++ b/src/api/rev/data.rs @@ -10,7 +10,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; -use assembly_data::fdb::common::Latin1Str; +use assembly_fdb::common::Latin1Str; use paradox_typed_db::TypedDatabase; use serde::Serialize; @@ -92,11 +92,7 @@ impl ReverseLookup { for m in db.missions.row_iter() { let id = m.id(); - let d_type = m - .defined_type() - .map(Latin1Str::decode) - .unwrap_or_default() - .into_owned(); + let d_type = m.defined_type().decode().into_owned(); let d_subtype = m .defined_subtype() .map(Latin1Str::decode) diff --git a/src/api/rev/loot_table_index.rs b/src/api/rev/loot_table_index.rs index a512004..56fa2bf 100644 --- a/src/api/rev/loot_table_index.rs +++ b/src/api/rev/loot_table_index.rs @@ -4,7 +4,7 @@ use crate::api::{ adapter::{AdapterLayout, TypedTableIterAdapter}, map_opt, }; -use paradox_typed_db::{typed_rows::LootTableRow, TypedDatabase}; +use paradox_typed_db::{columns::LootTableColumn, rows::LootTableRow, TypedDatabase}; use serde::Serialize; use warp::{ filters::BoxedFilter, @@ -30,7 +30,7 @@ fn rev_loop_table_index_api(db: &TypedDatabase, rev: Rev<'static>, index: i32) - index, keys, table: &db.loot_table, - id_col: db.loot_table.col_id, + id_col: db.loot_table.get_col(LootTableColumn::Id).unwrap(), layout: AdapterLayout::Seq, }; list diff --git a/src/api/rev/missions.rs b/src/api/rev/missions.rs index b74cf62..eaee749 100644 --- a/src/api/rev/missions.rs +++ b/src/api/rev/missions.rs @@ -1,8 +1,8 @@ use std::{borrow::Borrow, collections::BTreeMap, convert::Infallible}; use assembly_core::buffer::CastError; -use assembly_data::xml::localization::LocaleNode; -use paradox_typed_db::{typed_rows::MissionsRow, TypedDatabase}; +use assembly_xml::localization::LocaleNode; +use paradox_typed_db::{rows::MissionsRow, TypedDatabase}; use serde::{ser::SerializeMap, Serialize}; use warp::{ filters::BoxedFilter, @@ -10,9 +10,12 @@ use warp::{ Filter, }; -use crate::api::{ - adapter::{I32Slice, IdentityHash, LocaleTableAdapter, TypedTableIterAdapter}, - map_res, PercentDecoded, +use crate::{ + api::{ + adapter::{I32Slice, IdentityHash, LocaleTableAdapter, TypedTableIterAdapter}, + map_res, PercentDecoded, + }, + data::locale::LocaleRoot, }; use super::{common::MissionsTaskIconsAdapter, Api, Rev}; @@ -96,6 +99,7 @@ struct MissionIDList<'b> { fn missions_reply<'a, 'b>( db: &'b TypedDatabase<'a>, + loc: &'b LocaleRoot, mission_ids: &'b [i32], ) -> Api, MissionTypesEmbedded<'a, 'b>> { Api { @@ -103,7 +107,7 @@ fn missions_reply<'a, 'b>( embedded: MissionTypesEmbedded { missions: MissionsAdapter::new(&db.missions, mission_ids), mission_task_icons: MissionsTaskIconsAdapter::new(&db.mission_tasks, mission_ids), - locale: MissionLocale::new(&db.locale, mission_ids), + locale: MissionLocale::new(&loc.root, mission_ids), }, } } @@ -111,12 +115,13 @@ fn missions_reply<'a, 'b>( fn rev_mission_type_api( db: &TypedDatabase, rev: Rev, + loc: LocaleRoot, d_type: PercentDecoded, ) -> Result { let key: &String = d_type.borrow(); match rev.inner.mission_types.get(key) { Some(t) => match t.get("") { - Some(missions) => Ok(warp::reply::json(&missions_reply(db, missions))), + Some(mission_ids) => Ok(warp::reply::json(&missions_reply(db, &loc, mission_ids))), None => Ok(warp::reply::json(&Subtypes { subtypes: MissionSubtypesAdapter(t), })), @@ -128,6 +133,7 @@ fn rev_mission_type_api( fn rev_mission_subtype_api( db: &TypedDatabase, rev: Rev, + loc: LocaleRoot, d_type: PercentDecoded, d_subtype: PercentDecoded, ) -> Result { @@ -135,7 +141,7 @@ fn rev_mission_subtype_api( let t = rev.inner.mission_types.get(t_key); let s_key: &String = d_subtype.borrow(); let s = t.and_then(|t| t.get(s_key)); - let m = s.map(|missions| missions_reply(db, missions)); + let m = s.map(|mission_ids| missions_reply(db, &loc, mission_ids)); Ok(warp::reply::json(&m)) } @@ -147,7 +153,9 @@ pub(super) fn mission_types_api< F: Filter + Send + Sync + Clone + 'static, >( rev: &F, + loc: LocaleRoot, ) -> BoxedFilter<(WithStatus,)> { + let loc_filter = warp::any().map(move || loc.clone()); let rev_mission_types_base = rev.clone().and(warp::path("mission_types")); let rev_mission_types_full = rev_mission_types_base @@ -160,6 +168,7 @@ pub(super) fn mission_types_api< let rev_mission_type = rev_mission_types_base .clone() + .and(loc_filter.clone()) .and(warp::path::param()) .and(warp::path::end()) .map(rev_mission_type_api) @@ -168,6 +177,7 @@ pub(super) fn mission_types_api< let rev_mission_subtype = rev_mission_types_base .clone() + .and(loc_filter) .and(warp::path::param()) .and(warp::path::param()) .and(warp::path::end()) diff --git a/src/api/rev/mod.rs b/src/api/rev/mod.rs index 20a1dc2..f52a673 100644 --- a/src/api/rev/mod.rs +++ b/src/api/rev/mod.rs @@ -26,6 +26,8 @@ mod skills; pub use data::ReverseLookup; +use crate::data::locale::LocaleRoot; + use super::{map_res, tydb_filter}; #[derive(Debug, Clone, Serialize)] @@ -63,6 +65,7 @@ type Ext = (&'static TypedDatabase<'static>, Rev<'static>); pub(super) fn make_api_rev( db: &'static TypedDatabase<'static>, + loc: LocaleRoot, rev: &'static ReverseLookup, ) -> BoxedFilter<(WithStatus,)> { let db = tydb_filter(db); @@ -72,7 +75,7 @@ pub(super) fn make_api_rev( let rev_behaviors = behaviors::behaviors_api(&rev); let rev_component_types = component_types::component_types_api(&rev); let rev_loot_table_index = loot_table_index::loot_table_index_api(&rev); - let rev_mission_types = missions::mission_types_api(&rev); + let rev_mission_types = missions::mission_types_api(&rev, loc); let rev_object_types = object_types::object_types_api(&rev); let rev_skills = skills::skill_api(&rev); diff --git a/src/api/rev/object_types.rs b/src/api/rev/object_types.rs index 645b1b2..2746e1f 100644 --- a/src/api/rev/object_types.rs +++ b/src/api/rev/object_types.rs @@ -11,9 +11,8 @@ use warp::{ use super::{common::ObjectTypeEmbedded, Ext}; use crate::api::{ - adapter::TypedTableIterAdapter, map_opt_res, map_res, - rev::{Api, Rev}, + rev::{common::ObjectsRefAdapter, Api, Rev}, PercentDecoded, }; @@ -35,7 +34,7 @@ fn rev_object_type_api( object_ids: objects.as_ref(), }, embedded: ObjectTypeEmbedded { - objects: TypedTableIterAdapter::new(&db.objects, objects), + objects: ObjectsRefAdapter::new(&db.objects, objects), }, }; warp::reply::json(&rep) diff --git a/src/api/rev/skills.rs b/src/api/rev/skills.rs index 3615707..7878454 100644 --- a/src/api/rev/skills.rs +++ b/src/api/rev/skills.rs @@ -4,7 +4,7 @@ use crate::api::{ map_res, }; use assembly_core::buffer::CastError; -use paradox_typed_db::TypedDatabase; +use paradox_typed_db::{columns::MissionTasksColumn, TypedDatabase}; use serde::Serialize; use std::convert::Infallible; use warp::{ @@ -26,7 +26,7 @@ fn rev_skill_id_api(db: &'_ TypedDatabase, rev: Rev, skill_id: i32) -> Result, + /// The `versions` directory + pub versions: Option, /// The CDClient database FDB file pub cdclient: PathBuf, /// The lu-explorer static files diff --git a/src/data/fs.rs b/src/data/fs.rs new file mode 100644 index 0000000..d2b08bb --- /dev/null +++ b/src/data/fs.rs @@ -0,0 +1,238 @@ +use std::{ + collections::BTreeMap, + convert::TryFrom, + ffi::OsStr, + fs::{DirEntry, File}, + io::{self, ErrorKind}, + os::unix::prelude::OsStrExt, + path::{Component, Path, PathBuf}, +}; + +use assembly_fdb::common::Latin1Str; +use assembly_pack::{ + crc::calculate_crc, + pki::core::{PackFileRef, PackIndexFile}, +}; + +use serde::Serialize; +use tokio::sync::oneshot::Sender; +use tracing::error; +use warp::{ + filters::BoxedFilter, + hyper::StatusCode, + path::Tail, + reply::{json, with_status, Json, WithStatus}, + Filter, Rejection, +}; + +pub fn cleanup_path(url: &Latin1Str) -> Option { + let url = url.decode().replace('\\', "/").to_ascii_lowercase(); + let p = Path::new(&url); + + let mut path = Path::new("/textures/ui").to_owned(); + for comp in p.components() { + match comp { + Component::ParentDir => { + path.pop(); + } + Component::CurDir => {} + Component::Normal(seg) => path.push(seg), + Component::RootDir => return None, + Component::Prefix(_) => return None, + } + } + path.set_extension("png"); + Some(path) +} + +#[derive(Debug, Clone)] +pub struct LuRes { + prefix: String, +} + +impl LuRes { + pub fn new(prefix: &str) -> Self { + Self { + prefix: prefix.to_owned(), + } + } + + pub fn to_res_href(&self, path: &Path) -> String { + format!("{}{}", self.prefix, path.display()) + } +} + +enum Event { + Path(Tail, Sender, Rejection>>), +} + +#[derive(Serialize)] +struct Reply { + crc: u32, +} + +pub fn make_file_filter(_path: &Path) -> BoxedFilter<(WithStatus,)> { + let (tx, mut rx) = tokio::sync::mpsc::channel(1000); + tokio::spawn(async move { + loop { + match rx.recv().await { + None => break, + Some(Event::Path(tail, reply)) => { + let path = format!("client\\res\\{}", tail.as_str()); + let crc = calculate_crc(path.as_bytes()); + + let t = with_status(json(&Reply { crc }), StatusCode::OK); + // Ignore replies that get dropped + let _ = reply.send(Ok(t)); + } + } + } + }); + warp::path::tail() + .and_then(move |tail: Tail| { + let tx = tx.clone(); + async move { + let (otx, orx) = tokio::sync::oneshot::channel(); + if tx.send(Event::Path(tail, otx)).await.is_ok() { + match orx.await { + Ok(v) => v, + Err(e) => { + error!("{}", e); + Err(warp::reject::not_found()) + } + } + } else { + Err(warp::reject::not_found()) + } + } + }) + .boxed() +} + +/// A single file +#[derive(Debug, Copy, Clone, Serialize)] +pub enum NodeKind { + ZoneFile, + LevelFile, + DirectDrawSurface, + Script, +} + +#[derive(Debug, Clone, Serialize)] +pub struct Node { + /// Server side path: DO NOT SERIALIZE + pub rel_path: PathBuf, + /// The kind of this file + pub kind: NodeKind, +} + +pub struct ServerNode { + pub public: Node, + /// server path, DO NOT SERIALIZE + pub abs_path: PathBuf, +} + +pub struct Loader { + /// Maps path CRCs to a node + entries: BTreeMap, + /// The data from a pack index file + pki: PackIndexFile, +} + +impl Loader { + pub fn new() -> Self { + Self { + entries: BTreeMap::new(), + pki: PackIndexFile { + archives: Vec::new(), + files: BTreeMap::new(), + }, + } + } + + pub fn get(&self, crc: u32) -> Option<&ServerNode> { + self.entries.get(&crc) + } + + pub fn get_pki(&self, crc: u32) -> Option<&PackFileRef> { + self.pki + .files + .get(&crc) + .map(|r| &self.pki.archives[r.pack_file as usize]) + } + + fn error(&mut self, path: &Path, error: io::Error) { + error!("{} {}", path.display(), error) + } + + pub fn load_pki(&mut self, path: &Path) -> io::Result<()> { + let file = File::open(path)?; + self.pki = PackIndexFile::try_from(file) + .map_err(|error| io::Error::new(ErrorKind::Other, error))?; + Ok(()) + } + + /*pub fn load_luz(&mut self, path: &Path) -> Option> { + let file = File::open(&path).map_err(|e| self.error(path, e)).ok()?; + let mut buf = BufReader::new(file); + match match ZoneFile::try_from_luz(&mut buf) { + Ok(zf) => zf.parse_paths(), + Err(e) => { + self.error(path, io::Error::new(ErrorKind::Other, e)); + return None; + } + } { + Ok(zf) => Some(zf), + Err(_e) => { + /* TODO */ + None + } + } + }*/ + + pub fn load_node(&mut self, rel_parent: &Path, entry: DirEntry) { + let path = entry.path(); + let name = path.file_name().unwrap().to_string_lossy().to_lowercase(); + if path.is_dir() { + let relative = rel_parent.join(&name); + self.load_dir(&relative, &path); + } + if path.is_file() { + let ext = path.extension().and_then(OsStr::to_str); + if let Some(kind) = match ext { + Some("luz") => Some(NodeKind::ZoneFile), + Some("lvl") => Some(NodeKind::LevelFile), + Some("dds") => Some(NodeKind::DirectDrawSurface), + Some("lua") => Some(NodeKind::Script), + _ => None, + } { + let rel_path = rel_parent.join(&name); + let crc = calculate_crc(rel_path.as_os_str().as_bytes()); + self.entries.insert( + crc, + ServerNode { + abs_path: path, + public: Node { rel_path, kind }, + }, + ); + } + } + } + + pub fn load_dir(&mut self, relative: &Path, absolute: &Path) { + match std::fs::read_dir(absolute) { + Ok(read_dir) => { + for entry in read_dir { + match entry { + Ok(entry) => self.load_node(relative, entry), + Err(e) => { + self.error(absolute, e); + continue; + } + } + } + } + Err(e) => self.error(absolute, e), + } + } +} diff --git a/src/data/locale.rs b/src/data/locale.rs new file mode 100644 index 0000000..66311cb --- /dev/null +++ b/src/data/locale.rs @@ -0,0 +1,64 @@ +use std::sync::Arc; + +use assembly_xml::localization::LocaleNode; +use paradox_typed_db::ext::MissionKind; + +#[derive(Clone)] +pub struct LocaleRoot { + pub root: Arc, +} + +impl LocaleRoot { + pub fn new(root: Arc) -> Self { + Self { root } + } + + pub fn get_mission_name(&self, kind: MissionKind, id: i32) -> Option { + let missions = self.root.str_children.get("Missions").unwrap(); + if id > 0 { + if let Some(mission) = missions.int_children.get(&(id as u32)) { + if let Some(name_node) = mission.str_children.get("name") { + let name = name_node.value.as_ref().unwrap(); + return Some(format!("{} | {:?} #{}", name, kind, id)); + } + } + } + None + } + + pub fn get_item_set_name(&self, rank: i32, id: i32) -> Option { + let missions = self.root.str_children.get("ItemSets").unwrap(); + if id > 0 { + if let Some(mission) = missions.int_children.get(&(id as u32)) { + if let Some(name_node) = mission.str_children.get("kitName") { + let name = name_node.value.as_ref().unwrap(); + return Some(if rank > 0 { + format!("{} (Rank {}) | Item Set #{}", name, rank, id) + } else { + format!("{} | Item Set #{}", name, id) + }); + } + } + } + None + } + + pub fn get_skill_name_desc(&self, id: i32) -> (Option, Option) { + let skills = self.root.str_children.get("SkillBehavior").unwrap(); + let mut the_name = None; + let mut the_desc = None; + if id > 0 { + if let Some(skill) = skills.int_children.get(&(id as u32)) { + if let Some(name_node) = skill.str_children.get("name") { + let name = name_node.value.as_ref().unwrap(); + the_name = Some(format!("{} | Item Set #{}", name, id)); + } + if let Some(desc_node) = skill.str_children.get("descriptionUI") { + let desc = desc_node.value.as_ref().unwrap(); + the_desc = Some(desc.clone()); + } + } + } + (the_name, the_desc) + } +} diff --git a/src/data/maps.rs b/src/data/maps.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/data/maps.rs @@ -0,0 +1 @@ + diff --git a/src/data/mod.rs b/src/data/mod.rs index 2c00660..839063d 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -1,8 +1,11 @@ use std::{convert::TryFrom, fmt}; -use assembly_data::fdb::common::Latin1Str; +use assembly_fdb::common::Latin1Str; use serde::{Deserialize, Serialize}; +pub mod fs; +pub mod locale; +pub mod maps; pub mod skill_system; #[repr(u8)] diff --git a/src/data/skill_system.rs b/src/data/skill_system.rs index ae315e3..79c3ea3 100644 --- a/src/data/skill_system.rs +++ b/src/data/skill_system.rs @@ -1,4 +1,4 @@ -use assembly_data::fdb::common::Latin1Str; +use assembly_fdb::common::Latin1Str; use serde::Serialize; #[derive(Debug, Clone, PartialEq, Eq, Serialize)] diff --git a/src/main.rs b/src/main.rs index f837757..b45595b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,13 @@ use std::{ fs::File, + path::Path, str::FromStr, sync::{Arc, RwLock}, }; -use api::make_api; -use assembly_data::{fdb::mem::Database, xml::localization::load_locale}; +use api::ApiFactory; +use assembly_fdb::mem::Database; +use assembly_xml::localization::load_locale; use color_eyre::eyre::WrapErr; use config::{Config, Options}; use handlebars::Handlebars; @@ -15,6 +17,7 @@ use paradox_typed_db::TypedDatabase; use structopt::StructOpt; use template::make_spa_dynamic; use tokio::runtime::Handle; +use tracing::log::LevelFilter; use warp::{filters::BoxedFilter, hyper::Uri, path::FullPath, Filter, Reply}; mod api; @@ -29,6 +32,7 @@ use crate::{ api::rev::ReverseLookup, auth::AuthKind, config::AuthConfig, + data::{fs::LuRes, locale::LocaleRoot}, fallback::make_fallback, redirect::{add_host_filters, add_redirect_filters, base_filter}, template::{load_meta_template, FsEventHandler, TemplateUpdateTask}, @@ -36,7 +40,9 @@ use crate::{ #[tokio::main] async fn main() -> color_eyre::Result<()> { - pretty_env_logger::init(); + pretty_env_logger::formatted_builder() + .filter_level(LevelFilter::Info) + .init(); color_eyre::install()?; let opts = Options::from_args(); @@ -86,15 +92,34 @@ async fn main() -> color_eyre::Result<()> { .clone() .unwrap_or_else(|| format!("{}/lu-res", canonical_base_url)); - let lu_res_prefix = Box::leak(lu_res.clone().into_boxed_str()); - let tydb = TypedDatabase::new(lr.clone(), lu_res_prefix, tables)?; - let data = Box::leak(Box::new(tydb)); - let rev = Box::leak(Box::new(ReverseLookup::new(data))); + let res = LuRes::new(&lu_res); + + let tydb = TypedDatabase::new(tables)?; + let tydb = Box::leak(Box::new(tydb)); + let rev = Box::leak(Box::new(ReverseLookup::new(tydb))); + + // Load the files + // let mut f = Folder::default(); + // if let Some(res) = &cfg.data.res { + // let mut loader = Loader::new(); + // f = loader.load_dir(res); + // } // Make the API let lu_json_path = cfg.data.lu_json_cache.clone(); let fallback_routes = make_fallback(lu_json_path); + let res_path = cfg + .data + .res + .as_deref() + .unwrap_or_else(|| Path::new("client/res")); + let pki_path = cfg.data.versions.as_ref().map(|x| x.join("primary.pki")); + + let file_routes = warp::path("v1") + .and(warp::path("res")) + .and(data::fs::make_file_filter(res_path)); + let auth_kind = if matches!(cfg.auth, Some(AuthConfig { basic: Some(_) })) { AuthKind::Basic } else { @@ -115,8 +140,23 @@ async fn main() -> color_eyre::Result<()> { }) .boxed(); - let api_routes = make_api(api_url, auth_kind, db, data, rev, lr.clone()); - let api = warp::path("api").and(fallback_routes.or(api_swagger).or(api_routes)); + let api_routes = ApiFactory { + url: api_url, + auth_kind, + db, + tydb, + rev, + lr: lr.clone(), + res_path, + pki_path: pki_path.as_deref(), + } + .make_api(); + let api = warp::path("api").and( + fallback_routes + .or(api_swagger) + .or(file_routes) + .or(api_routes), + ); let spa_path = &cfg.data.explorer_spa; let spa_index = spa_path.join("index.html"); @@ -136,7 +176,8 @@ async fn main() -> color_eyre::Result<()> { rt.spawn(TemplateUpdateTask::new(rx, hb.clone())); - let spa_dynamic = make_spa_dynamic(data, hb, &cfg.general.domain); + let loc = LocaleRoot::new(lr); + let spa_dynamic = make_spa_dynamic(tydb, loc, res, hb, &cfg.general.domain); //let spa_file = warp::fs::file(spa_index); let spa = warp::fs::dir(spa_path.clone()).or(spa_dynamic); diff --git a/src/template/mod.rs b/src/template/mod.rs index 8892e92..39d7fc8 100644 --- a/src/template/mod.rs +++ b/src/template/mod.rs @@ -12,13 +12,18 @@ use std::{ }; use handlebars::Handlebars; -use paradox_typed_db::{typed_ext::MissionKind, TypedDatabase}; +use paradox_typed_db::{ext::MissionKind, TypedDatabase}; use regex::{Captures, Regex}; use serde::Serialize; use tokio::sync::mpsc::{Receiver, Sender}; use tracing::{debug, error, info}; use warp::{filters::BoxedFilter, path::FullPath, Filter}; +use crate::data::{ + fs::{cleanup_path, LuRes}, + locale::LocaleRoot, +}; + fn make_meta_template(text: &str) -> Cow { let re = Regex::new("").unwrap(); re.replace_all(text, |cap: &Captures| { @@ -146,7 +151,7 @@ where } /// Retrieve metadata for /missions/:id -fn mission_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { +fn mission_get_impl(data: &'_ TypedDatabase<'_>, loc: LocaleRoot, res: LuRes, id: i32) -> Meta { let mut image = None; let mut kind = MissionKind::Mission; if let Some(mission) = data.get_mission_data(id) { @@ -154,7 +159,7 @@ fn mission_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { kind = MissionKind::Achievement; if let Some(icon_id) = mission.mission_icon_id { if let Some(path) = data.get_icon_path(icon_id) { - image = Some(data.to_res_href(&path)); + image = cleanup_path(path).map(|p| res.to_res_href(&p)); } } } @@ -163,12 +168,12 @@ fn mission_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { let mut desc = String::new(); let tasks = data.get_mission_tasks(id); - let tasks_locale = data.locale.str_children.get("MissionTasks").unwrap(); + let tasks_locale = loc.root.str_children.get("MissionTasks").unwrap(); for task in tasks { if image == None { if let Some(icon_id) = task.icon_id { if let Some(path) = data.get_icon_path(icon_id) { - image = Some(data.to_res_href(&path)); + image = cleanup_path(path).map(|p| res.to_res_href(&p)); } } } @@ -188,7 +193,7 @@ fn mission_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { desc.pop(); } - let title = data + let title = loc .get_mission_name(kind, id) .unwrap_or(format!("Missing {:?} #{}", kind, id)); @@ -200,12 +205,13 @@ fn mission_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { } /// Retrieve metadata for /objects/:id -fn object_get_api(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { +fn object_get_api(data: &'_ TypedDatabase<'_>, _loc: LocaleRoot, res: LuRes, id: i32) -> Meta { let (title, description) = data .get_object_name_desc(id) .unwrap_or((format!("Missing Object #{}", id), String::new())); let comp = data.get_components(id); let image = comp.render.and_then(|id| data.get_render_image(id)); + let image = image.and_then(cleanup_path).map(|p| res.to_res_href(&p)); Meta { title: Cow::Owned(title), description: Cow::Owned(description), @@ -214,15 +220,15 @@ fn object_get_api(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { } /// Retrieve metadata for /objects/item-sets/:id -fn item_set_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { +fn item_set_get_impl(data: &'_ TypedDatabase<'_>, loc: LocaleRoot, res: LuRes, id: i32) -> Meta { let mut rank = 0; let mut image = None; let mut desc = String::new(); if let Some(item_set) = data.item_sets.get_data(id) { rank = item_set.kit_rank; if let Some(image_id) = item_set.kit_image { - if let Some(path) = data.get_icon_path(image_id) { - image = Some(data.to_res_href(&path)); + if let Some(path) = data.get_icon_path(image_id).and_then(cleanup_path) { + image = Some(res.to_res_href(&path)); } } @@ -237,7 +243,7 @@ fn item_set_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { desc.pop(); } - let title = data + let title = loc .get_item_set_name(rank, id) .unwrap_or(format!("Unnamed Item Set #{}", id)); @@ -249,8 +255,8 @@ fn item_set_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { } /// Retrieve metadata for /skills/:id -fn skill_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { - let (mut title, description) = data.get_skill_name_desc(id); +fn skill_get_impl(data: &'_ TypedDatabase<'_>, loc: LocaleRoot, res: LuRes, id: i32) -> Meta { + let (mut title, description) = loc.get_skill_name_desc(id); let description = description.map(Cow::Owned).unwrap_or(Cow::Borrowed("")); let mut image = None; @@ -259,8 +265,8 @@ fn skill_get_impl(data: &'_ TypedDatabase<'_>, id: i32) -> Meta { title = Some(format!("Skill #{}", id)) } if let Some(icon_id) = skill.skill_icon { - if let Some(path) = data.get_icon_path(icon_id) { - image = Some(data.to_res_href(&path)); + if let Some(path) = data.get_icon_path(icon_id).and_then(cleanup_path) { + image = Some(res.to_res_href(&path)); } } } @@ -296,8 +302,13 @@ struct Meta { fn meta<'r>( data: &'static TypedDatabase<'static>, + loc: LocaleRoot, + res: LuRes, ) -> impl Filter + Clone + 'r { - let base = warp::any().map(move || data); + let d = warp::any().map(move || data); + let l = warp::any().map(move || loc.clone()); + let r = warp::any().map(move || res.clone()); + let base = d.and(l).and(r); let dashboard = warp::path("dashboard").and(warp::path::end()).map(|| Meta { title: Cow::Borrowed("Dashboard"), @@ -310,13 +321,19 @@ fn meta<'r>( description: Cow::Borrowed("Check out the LEGO Universe Objects"), image: None, }); - let object_get = base.and(warp::path::param::()).map(object_get_api); + let object_get = base + .clone() + .and(warp::path::param::()) + .map(object_get_api); let item_sets_end = warp::path::end().map(|| Meta { title: Cow::Borrowed("Item Sets"), description: Cow::Borrowed("Check out the LEGO Universe Objects"), image: None, }); - let item_set_get = base.and(warp::path::param::()).map(item_set_get_impl); + let item_set_get = base + .clone() + .and(warp::path::param::()) + .map(item_set_get_impl); let item_sets = warp::path("item-sets").and(item_sets_end.or(item_set_get).unify()); let objects = warp::path("objects").and(objects_end.or(object_get).unify().or(item_sets).unify()); @@ -326,7 +343,10 @@ fn meta<'r>( description: Cow::Borrowed("Check out the LEGO Universe Missions"), image: None, }); - let mission_get = base.and(warp::path::param::()).map(mission_get_impl); + let mission_get = base + .clone() + .and(warp::path::param::()) + .map(mission_get_impl); let missions = warp::path("missions").and(missions_end.or(mission_get).unify()); let skills_end = warp::path::end().map(move || Meta { @@ -356,6 +376,8 @@ fn meta<'r>( #[allow(clippy::needless_lifetimes)] // false positive? pub(crate) fn make_spa_dynamic( data: &'static TypedDatabase<'static>, + loc: LocaleRoot, + res: LuRes, hb: Arc>>, domain: &str, // hnd: ArcHandle, @@ -366,8 +388,7 @@ pub(crate) fn make_spa_dynamic( }; // Prepare the default image - let mut default_img = data.lu_res_prefix.to_owned(); - default_img.push_str(DEFAULT_IMG); + let default_img = res.to_res_href(Path::new(DEFAULT_IMG)); let default_img: &'static str = Box::leak(default_img.into_boxed_str()); // Create a reusable closure to render template @@ -375,7 +396,7 @@ pub(crate) fn make_spa_dynamic( warp::any() .and(dom) - .and(meta(data)) + .and(meta(data, loc, res)) .and(warp::path::full()) .map( move |dom: &str, meta: Meta, full_path: FullPath| WithTemplate {