add outlier filtering to charts, closes #19

ultros-frontend/ultros-app/src/components/price_history_chart.rs

@@ -6,27 +6,36 @@ use ultros_api_types::SaleHistory;
 
 use ultros_charts::draw_sale_history_scatter_plot;
 
-use crate::global_state::LocalWorldData;
+use crate::{components::toggle::Toggle, global_state::LocalWorldData};
 
 #[component]
 pub fn PriceHistoryChart(sales: MaybeSignal<Vec<SaleHistory>>) -> impl IntoView {
     let canvas = create_node_ref::<Canvas>();
     let local_world_data = use_context::<LocalWorldData>().unwrap();
     let helper = local_world_data.0.unwrap();
+    let (filter_outliers, set_filter_outliers) = create_signal(true);
     let hidden = create_memo(move |_| {
         if let Some(canvas) = canvas() {
             let backend = CanvasBackend::with_canvas_object(canvas.deref().clone()).unwrap();
             // if there's an error drawing, we should hide the canvas
             sales.with(|sales| {
-                draw_sale_history_scatter_plot(backend, helper.clone().as_ref(), sales).is_err()
+                draw_sale_history_scatter_plot(
+                    backend,
+                    helper.clone().as_ref(),
+                    filter_outliers(),
+                    sales,
+                )
+                .is_err()
             })
         } else {
             true
         }
     });
     view! {
-        <div class:hidden=hidden>
+        <div class="flex flex-col" class:hidden=hidden>
             <canvas width="750" height="450" _ref=canvas/>
+            <Toggle checked=filter_outliers set_checked=set_filter_outliers
+                checked_label="Filtering outliers" unchecked_label="No filter" />
         </div>
     }
 }

ultros-frontend/ultros-app/src/routes/item_view.rs

@@ -106,20 +106,20 @@ fn ListingsContent(item_id: Memo<i32>, world: Memo<String>) -> impl IntoView {
                     let lq_listings = currently_shown.listings.iter().cloned().filter(|(listing, _)| !listing.hq).collect::<Vec<_>>();
                     let sales = create_memo(move |_| currently_shown.sales.clone());
                     view! {
-                        <div class="content-well max-h-[30em] overflow-y-auto">
+                        <div class="content-well max-h-[35em] overflow-y-auto">
                             <PriceHistoryChart sales=MaybeSignal::from(sales) />
                         </div>
                         {(!hq_listings.is_empty()).then(move || {
-                            view!{ <div class="content-well max-h-[30em] overflow-y-auto">
+                            view!{ <div class="content-well max-h-[35em] overflow-y-auto">
                                 <span class="content-title">"high quality listings"</span>
                                 <ListingsTable listings=hq_listings />
                             </div> }.into_view()
                         })}
-                        <div class="content-well max-h-[30em] overflow-y-auto">
+                        <div class="content-well max-h-[35em] overflow-y-auto">
                             <span class="content-title">"low quality listings"</span>
                             <ListingsTable listings=lq_listings />
                         </div>
-                        <div class="content-well max-h-[30em] overflow-y-auto">
+                        <div class="content-well max-h-[35em] overflow-y-auto">
                             <span class="content-title">"sale history"</span>
                             <SaleHistoryTable sales=Signal::from(sales) />
                         </div>

ultros-frontend/ultros-charts/src/lib.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::collections::HashSet;
 
 use anyhow::anyhow;
@@ -35,18 +36,59 @@ enum DayLabelMode {
     Minute,
 }
 
+/// Returns a filter where Some((min, max))
+fn get_iqr_filter(sales: &[SaleHistory]) -> Option<(i32, i32)> {
+    if sales.len() < 10 {
+        return None;
+    }
+    let sales_prices = sales
+        .into_iter()
+        .map(|sales| sales.price_per_item)
+        .sorted()
+        .collect::<Vec<_>>();
+    let first_quartile_index = sales_prices.len() / 4;
+    let last_quartile_index = sales_prices.len() - first_quartile_index;
+    let first_quartile_value = sales_prices.get(first_quartile_index)?;
+    let third_quartile_value = sales_prices.get(last_quartile_index)?;
+    let interquartile_range = ((third_quartile_value - first_quartile_value) as f32 * 2.5) as i32;
+    Some((
+        first_quartile_value - interquartile_range,
+        third_quartile_value + interquartile_range,
+    ))
+}
+
+fn filter_outliers<'a>(sales: &'a [SaleHistory]) -> Cow<'a, [SaleHistory]> {
+    if let Some((min, max)) = get_iqr_filter(sales) {
+        let range = min..=max;
+        Cow::Owned(
+            sales
+                .into_iter()
+                .filter(|sales| range.contains(&sales.price_per_item))
+                .cloned()
+                .collect(),
+        )
+    } else {
+        Cow::Borrowed(sales)
+    }
+}
+
 pub fn draw_sale_history_scatter_plot<'a, T>(
     backend: T,
     world_helper: &WorldHelper,
+    remove_outliers: bool,
     sales: &[SaleHistory],
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync + 'a>>
 where
     T: 'a + DrawingBackend,
 {
+    let sales = if remove_outliers {
+        filter_outliers(sales)
+    } else {
+        Cow::Borrowed(sales)
+    };
     let root = backend.into_drawing_area();
-    root.fill(&RGBColor(16, 10, 18).mix(0.93))?;
-
-    let line = map_sale_history_to_line(world_helper, sales);
+    root.fill(&RGBColor(16, 10, 18))?;
+    let line = map_sale_history_to_line(world_helper, &sales);
     let item_name = &xiv_gen_db::data()
         .items
         .get(&ItemId(
@@ -59,6 +101,7 @@ where
         .flat_map(|(_, sales)| sales)
         .map(|(_, price, _)| price)
         .max()
+        .copied()
         .ok_or(anyhow!("price hidden"))?;
     let (first_sale, last_sale) = line
         .iter()
@@ -78,6 +121,7 @@ where
     } else {
         DayLabelMode::Minute
     };
+    let pad_top = (max_sale as f32 * 1.5).ceil() as i32;
     let mut chart = ChartBuilder::on(&root)
         .x_label_area_size(60)
         .y_label_area_size(100)
@@ -86,7 +130,7 @@ where
             format!("{} - Sale History", item_name),
             ("Jaldi, sans-serif", 25.0).into_font().color(&WHITE),
         )
-        .build_cartesian_2d(*first_sale..*last_sale, 0..*max_sale)?;
+        .build_cartesian_2d(*first_sale..*last_sale, 0..pad_top)?;
 
     chart
         .configure_mesh()
@@ -210,5 +254,6 @@ fn map_sale_history_to_line(
     selector_source
         .into_iter()
         .flat_map(|w| map_sales_in(world_helper, w, sales))
+        .sorted_by_cached_key(|(name, _)| name.clone())
         .collect()
 }

ultros/src/discord/ffxiv/item_prices.rs

@@ -138,7 +138,8 @@ async fn history(
         let backend = SVGBackend::with_string(&mut buffer, SIZE);
 
         let world_helper = &*ctx.data().world_helper;
-        if let Err(e) = ultros_charts::draw_sale_history_scatter_plot(backend, world_helper, &sales)
+        if let Err(e) =
+            ultros_charts::draw_sale_history_scatter_plot(backend, world_helper, true, &sales)
         {
             Err(anyhow!("can't draw scatter plot {e}"))?
         }