more db optimization

akarras · Jan 13, 2024 · 987e212 · 987e212
1 parent 90e789a
commit 987e212
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 88 deletions.
diff --git a/ultros-db/src/lib.rs b/ultros-db/src/lib.rs
@@ -19,9 +19,8 @@ pub use sea_orm::error::DbErr as SeaDbErr;
 pub use sea_orm::ActiveValue;
 
 use anyhow::Result;
-use chrono::{Duration, Utc};
-use futures::{future::try_join_all, Stream};
-use migration::{sea_orm::QueryOrder, DbErr, Migrator, MigratorTrait};
+use futures::future::try_join_all;
+use migration::{Migrator, MigratorTrait};
 
 use sea_orm::{
     ActiveModelTrait, ActiveValue::NotSet, ColumnTrait, ConnectOptions, Database,
@@ -58,14 +57,7 @@ impl UltrosDb {
                     .ok()
             })
             .unwrap_or(300);
-        opt.max_connections(max_connections)
-            .min_connections(0)
-            // .connect_timeout(Duration::from_secs(8))
-            // .idle_timeout(Duration::from_secs(8))
-            // .max_lifetime(Duration::from_secs(8))
-        //    .sqlx_logging(false)
-        //    .sqlx_logging_level(log::LevelFilter::Info)
-        ;
+        opt.max_connections(max_connections).min_connections(10);
         let db: DatabaseConnection = Database::connect(opt).await?;
         Migrator::up(&db, None).await?;
 
@@ -238,38 +230,6 @@ impl UltrosDb {
         Ok(listings)
     }
 
-    #[instrument(skip(self))]
-    pub async fn get_listings_for_world(
-        &self,
-        world: WorldId,
-        item: ItemId,
-    ) -> Result<Vec<active_listing::Model>> {
-        use active_listing::*;
-        let listings = Entity::find()
-            .filter(Column::ItemId.eq(item.0))
-            .filter(Column::WorldId.eq(world.0))
-            .all(&self.db)
-            .await?;
-        Ok(listings)
-    }
-
-    #[instrument(skip(self))]
-    pub async fn get_cheapest_listing_by_world(
-        &self,
-        world: i32,
-        item: i32,
-        is_hq: bool,
-    ) -> Result<Option<active_listing::Model>> {
-        use active_listing::*;
-        Ok(Entity::find()
-            .filter(Column::ItemId.eq(item))
-            .filter(Column::WorldId.eq(world))
-            .filter(Column::Hq.eq(is_hq))
-            .order_by_asc(Column::PricePerUnit)
-            .one(&self.db)
-            .await?)
-    }
-
     #[instrument(skip(self))]
     pub async fn create_alert(&self, owner: discord_user::Model) -> Result<alert::Model> {
         use alert::ActiveModel;
@@ -393,19 +353,6 @@ impl UltrosDb {
         Ok(retainers)
     }
 
-    #[instrument(skip(self))]
-    pub async fn stream_sales_within_days(
-        &self,
-        days: i64,
-        world_id: i32,
-    ) -> Result<impl Stream<Item = Result<sale_history::Model, DbErr>> + '_, anyhow::Error> {
-        Ok(sale_history::Entity::find()
-            .filter(sale_history::Column::WorldId.eq(world_id))
-            .filter(sale_history::Column::SoldDate.gt(Utc::now() - Duration::days(days)))
-            .stream(&self.db)
-            .await?)
-    }
-
     /// Stores a region. This generally assumes the regions haven't changed and really is just querying for region IDs
     #[instrument(skip(self))]
     pub async fn store_region(&self, region_name: &str) -> Result<region::Model> {

diff --git a/ultros-db/src/listings.rs b/ultros-db/src/listings.rs
@@ -1,8 +1,11 @@
 use anyhow::Result;
 use futures::{future::try_join_all, Stream};
+use itertools::Itertools;
 use metrics::{counter, histogram};
 use migration::DbErr;
-use sea_orm::{ColumnTrait, DbBackend, EntityTrait, FromQueryResult, QueryFilter, Statement};
+use sea_orm::{
+    ColumnTrait, DbBackend, EntityTrait, FromQueryResult, QueryFilter, QuerySelect, Statement,
+};
 use std::{
     collections::{hash_map::Entry, HashMap, HashSet},
     time::Instant,
@@ -94,17 +97,14 @@ impl UltrosDb {
             }),
         )
         .await?;
-        let retainers: HashSet<_> = items.iter().map(|i| i.retainer_id).collect();
-        let retainers: HashMap<i32, Retainer> = try_join_all(
-            retainers
-                .into_iter()
-                .map(|r| retainer::Entity::find_by_id(r).one(&self.db)),
-        )
-        .await?
-        .into_iter()
-        .flatten()
-        .map(|r| (r.id, r.into()))
-        .collect();
+        let retainers = items.iter().map(|i| i.retainer_id).unique();
+        let retainers: HashMap<i32, Retainer> = retainer::Entity::find()
+            .filter(retainer::Column::Id.is_in(retainers))
+            .all(&self.db)
+            .await?
+            .into_iter()
+            .map(|r| (r.id, r.into()))
+            .collect();
         Ok(items
             .into_iter()
             .flat_map(|i| retainers.get(&i.retainer_id).map(|r| (i.into(), r.clone())))
@@ -143,6 +143,21 @@ impl UltrosDb {
         Ok(data)
     }
 
+    #[instrument(skip(self))]
+    pub async fn get_listings_for_world(
+        &self,
+        world: WorldId,
+        item: ItemId,
+    ) -> Result<Vec<active_listing::Model>> {
+        use active_listing::*;
+        let listings = Entity::find()
+            .filter(Column::ItemId.eq(item.0))
+            .filter(Column::WorldId.eq(world.0))
+            .all(&self.db)
+            .await?;
+        Ok(listings)
+    }
+
     #[instrument(skip(self))]
     pub async fn get_all_listings_with_retainers(
         &self,

diff --git a/ultros-db/src/sales.rs b/ultros-db/src/sales.rs
@@ -9,7 +9,7 @@ use crate::{
     UltrosDb,
 };
 use anyhow::Result;
-use chrono::NaiveDateTime;
+use chrono::{Duration, NaiveDateTime, Utc};
 
 use futures::{future::try_join_all, Stream};
 use itertools::Itertools;
@@ -43,12 +43,9 @@ impl UltrosDb {
         }
 
         // check for any sales that have already been posted
-        let already_recorded_sales = Entity::find()
-            .filter(sale_history::Column::SoldItemId.eq(item_id.0))
-            .filter(sale_history::Column::WorldId.eq(world_id.0))
-            .order_by_desc(sale_history::Column::SoldDate)
-            .limit(sales.len() as u64)
-            .all(&self.db)
+        let limit = sales.len() as u64;
+        let already_recorded_sales = self
+            .get_sale_history_for_item(world_id.0, item_id.0, limit)
             .await?;
         let buyers = self.lookup_buyer_names(&sales).await?;
         sales.retain(|sale| {
@@ -123,25 +120,16 @@ impl UltrosDb {
         sales.sort_by_key(|sale| std::cmp::Reverse(sale.sold_date));
         sales.truncate(limit as usize);
 
-        let buyer_ids = sales.iter().map(|s| s.buying_character_id);
         let buyers = unknown_final_fantasy_character::Entity::find()
-            .filter(unknown_final_fantasy_character::Column::Id.is_in(buyer_ids))
+            .filter(
+                unknown_final_fantasy_character::Column::Id
+                    .is_in(sales.iter().map(|s| s.buying_character_id).unique()),
+            )
             .all(&self.db)
             .await?
             .into_iter()
             .map(|c| (c.id, c))
             .collect::<HashMap<_, _>>();
-        // let buyers = try_join_all(
-        //     sales
-        //         .iter()
-        //         .map(|s| s.buying_character_id)
-        //         .unique()
-        //         .map(|c| unknown_final_fantasy_character::Entity::find_by_id(c).one(&self.db)),
-        // )
-        // .await?
-        // .into_iter()
-        // .filter_map(|c| c.map(|c| (c.id, c)))
-        // .collect::<HashMap<_, _>>();
         let sales = sales
             .into_iter()
             .map(|sale| {
@@ -224,6 +212,19 @@ impl UltrosDb {
             .stream(&self.db)
             .await
     }
+
+    #[instrument(skip(self))]
+    pub async fn stream_sales_within_days(
+        &self,
+        days: i64,
+        world_id: i32,
+    ) -> Result<impl Stream<Item = Result<sale_history::Model, DbErr>> + '_, anyhow::Error> {
+        Ok(sale_history::Entity::find()
+            .filter(sale_history::Column::WorldId.eq(world_id))
+            .filter(sale_history::Column::SoldDate.gt(Utc::now() - Duration::days(days)))
+            .stream(&self.db)
+            .await?)
+    }
 }
 
 #[derive(Debug, FromQueryResult)]