playtime

.env

@@ -84,7 +84,7 @@ BEEHIIV_API_KEY=none
 
 ANALYTICS_ALLOWED_ORIGINS='["http://127.0.0.1:3000", "http://localhost:3000", "https://modrinth.com", "https://www.modrinth.com", "*"]'
 
-CLICKHOUSE_URL=http:/localhost:8123
+CLICKHOUSE_URL=http://localhost:8123
 CLICKHOUSE_USER=default
 CLICKHOUSE_PASSWORD=
 CLICKHOUSE_DATABASE=staging_ariadne

src/models/analytics.rs

@@ -1,10 +1,10 @@
 use clickhouse::Row;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use std::hash::{Hash, Hasher};
 use std::net::Ipv6Addr;
 use uuid::Uuid;
 
-#[derive(Row, Serialize, Clone)]
+#[derive(Row, Serialize, Deserialize, Clone)]
 pub struct Download {
     #[serde(with = "uuid::serde::compact")]
     pub id: Uuid,
@@ -41,7 +41,7 @@ impl Hash for Download {
     }
 }
 
-#[derive(Row, Serialize, Clone)]
+#[derive(Row, Serialize, Deserialize, Clone)]
 pub struct PageView {
     #[serde(with = "uuid::serde::compact")]
     pub id: Uuid,
@@ -76,12 +76,12 @@ impl Hash for PageView {
     }
 }
 
-#[derive(Row, Serialize, Clone)]
+#[derive(Row, Serialize, Deserialize, Clone, Debug)]
 pub struct Playtime {
     #[serde(with = "uuid::serde::compact")]
     pub id: Uuid,
     pub recorded: i64,
-    pub seconds: u16,
+    pub seconds: u64,
 
     // Modrinth User ID for logged in users (unused atm)
     pub user_id: u64,

src/queue/analytics.rs

@@ -1,5 +1,15 @@
-use crate::models::analytics::{Download, PageView, Playtime};
+use std::sync::Arc;
+
+use crate::{
+    models::{
+        analytics::{Download, PageView, Playtime},
+        ids::{ProjectId, VersionId},
+    },
+    routes::ApiError,
+};
+use chrono::NaiveDate;
 use dashmap::DashSet;
+use serde::{Deserialize, Serialize};
 
 pub struct AnalyticsQueue {
     views_queue: DashSet<PageView>,
@@ -67,4 +77,114 @@ impl AnalyticsQueue {
 
         Ok(())
     }
+
+    // Only one of project_id or version_id should be used
+    pub async fn fetch_playtimes(
+        &self,
+        projects: Option<Vec<ProjectId>>,
+        versions: Option<Vec<VersionId>>,
+        start_date: NaiveDate,
+        end_date: NaiveDate,
+        client: Arc<clickhouse::Client>,
+    ) -> Result<Vec<ReturnPlaytimes>, ApiError> {
+        let project_or_version = if projects.is_some() && versions.is_none() {
+            "project_id"
+        } else if versions.is_some() {
+            "version_id"
+        } else {
+            return Err(ApiError::InvalidInput(
+                "Only one of 'project_id' or 'version_id' should be used.".to_string(),
+            ));
+        };
+
+        let mut query = client.query(&format!(
+                "
+            WITH
+            loader_grouping AS
+            (
+                SELECT
+                    toYYYYMMDD(recorded) AS day,
+                    project_id,
+                    version_id,
+                    loader,
+                    sum(seconds) AS temp_loader_seconds
+                FROM playtime
+                WHERE loader != ''
+                GROUP BY
+                    day,
+                    project_id,
+                    version_id,
+                    loader
+            ),
+            game_version_grouping AS
+            (
+                SELECT
+                    toYYYYMMDD(recorded) AS day,
+                    project_id,
+                    version_id,
+                    game_version,
+                    sum(seconds) AS temp_game_version_seconds
+                FROM playtime
+                WHERE game_version != '' 
+                GROUP BY
+                    day,
+                    project_id,
+                    version_id,
+                    game_version
+            ),
+            parent_grouping AS
+            (
+                SELECT
+                    toYYYYMMDD(recorded) AS day,
+                    project_id,
+                    version_id,
+                    parent,
+                    sum(seconds) AS temp_parent_seconds
+                FROM playtime
+                WHERE parent != 0
+                GROUP BY
+                    day,
+                    project_id,
+                    version_id,
+                    parent
+            )
+        SELECT
+            l.day,
+            l.project_id,
+            l.{project_or_version},
+            sum(l.temp_loader_seconds) AS total_seconds,
+            array_aggDistinct((l.loader, l.temp_loader_seconds)) AS loader_seconds,
+            array_aggDistinct((g.game_version, g.temp_game_version_seconds)) AS game_version_seconds,
+            array_aggDistinct((p.parent, p.temp_parent_seconds)) AS parent_seconds
+        FROM loader_grouping AS l
+        LEFT JOIN game_version_grouping AS g ON (l.day = g.day) AND (l.{project_or_version} = g.{project_or_version})
+        LEFT JOIN parent_grouping AS p   ON (l.day = p.day) AND (l.{project_or_version} = p.{project_or_version})
+        WHERE l.day >= toYYYYMMDD(toDate(?)) AND l.day <= toYYYYMMDD(toDate(?))
+        AND l.{project_or_version} IN ? 
+        GROUP BY
+            l.day,
+            l.project_id,
+            l.{project_or_version}
+                        "
+            )).bind(start_date).bind(end_date);
+
+        if projects.is_some() {
+            query = query.bind(projects.unwrap().iter().map(|x| x.0).collect::<Vec<_>>());
+        } else if versions.is_some() {
+            query = query.bind(versions.unwrap().iter().map(|x| x.0).collect::<Vec<_>>());
+        }
+
+        Ok(query.fetch_all().await?)
+    }
+}
+
+#[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)]
+pub struct ReturnPlaytimes {
+    pub day: u32,
+    pub project_id: u64,
+    pub id: u64,
+    pub total_seconds: u64,
+    pub loader_seconds: Vec<(String, u64)>,
+    pub game_version_seconds: Vec<(String, u64)>,
+    pub parent_seconds: Vec<(u64, u64)>,
 }

src/routes/analytics.rs

@@ -1,5 +1,6 @@
 use crate::auth::get_user_from_headers;
 use crate::models::analytics::{PageView, Playtime};
+
 use crate::models::pats::Scopes;
 use crate::queue::maxmind::MaxMindIndexer;
 use crate::queue::session::AuthQueue;
@@ -205,10 +206,10 @@ pub async fn playtime_ingest(
                 .add_playtime(Playtime {
                     id: Default::default(),
                     recorded: Utc::now().timestamp_nanos() / 100_000,
-                    seconds: playtime.seconds,
+                    seconds: playtime.seconds as u64,
                     user_id: user.id.0,
-                    project_id: version.inner.id.0 as u64,
-                    version_id: version.inner.project_id.0 as u64,
+                    project_id: version.inner.project_id.0 as u64,
+                    version_id: version.inner.id.0 as u64,
                     loader: playtime.loader,
                     game_version: playtime.game_version,
                     parent: playtime.parent.map(|x| x.0).unwrap_or(0),

src/routes/v2/analytics_get.rs

@@ -0,0 +1,184 @@
+use std::{collections::HashMap, sync::Arc};
+
+use actix_web::{get, web, HttpRequest, HttpResponse};
+use chrono::NaiveDate;
+use serde::{Deserialize, Serialize};
+use sqlx::PgPool;
+
+use crate::{
+    auth::{filter_authorized_projects, filter_authorized_versions, get_user_from_headers},
+    database::models::{project_item, version_item},
+    models::{
+        ids::{
+            base62_impl::{parse_base62, to_base62},
+            ProjectId, VersionId,
+        },
+        pats::Scopes,
+    },
+    queue::{analytics::AnalyticsQueue, session::AuthQueue},
+};
+
+use super::ApiError;
+
+pub fn config(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::scope("analytics").service(playtimes_get));
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct GetPlaytimes {
+    pub project_ids: Option<Vec<String>>,
+    pub version_ids: Option<Vec<String>>,
+    pub start_date: Option<NaiveDate>,
+    pub end_date: Option<NaiveDate>,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct FetchedPlaytime {
+    pub day: u32,
+    pub total_seconds: u64,
+    pub loader_seconds: HashMap<String, u64>,
+    pub game_version_seconds: HashMap<String, u64>,
+    pub parent_seconds: HashMap<VersionId, u64>,
+}
+
+/// Get playtime data for a set of projects or versions
+/// Data is returned as a hashmap of project/version ids to a hashmap of days to playtime data
+/// eg:
+/// {
+///     "4N1tEhnO": {
+///         "20230824": {
+///             "day": 20230824,
+///             "total_seconds": 23,
+///             "loader_seconds": {
+///                 "bukkit": 23
+///             },
+///             "game_version_seconds": {
+///                 "1.2.3": 23
+///             },
+///             "parent_seconds": {
+///                 "": 0
+///             }
+///         }
+///    }
+///}
+/// Either a list of project_ids or version_ids can be used, but not both. Unauthorized projects/versions will be filtered out.
+/// loader_seconds, game_version_seconds, and parent_seconds are a how many of the total seconds were spent in each loader, game version, and parent version respectively.
+#[get("playtime")]
+pub async fn playtimes_get(
+    req: HttpRequest,
+    analytics_queue: web::Data<Arc<AnalyticsQueue>>,
+    clickhouse: web::Data<clickhouse::Client>,
+    data: web::Json<GetPlaytimes>,
+    session_queue: web::Data<AuthQueue>,
+    pool: web::Data<PgPool>,
+    redis: web::Data<deadpool_redis::Pool>,
+) -> Result<HttpResponse, ApiError> {
+    let user_option = get_user_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &session_queue,
+        Some(&[Scopes::ANALYTICS]),
+    )
+    .await
+    .map(|x| x.1)
+    .ok();
+
+    if data.project_ids.is_some() == data.version_ids.is_some() {
+        return Err(ApiError::InvalidInput(
+            "Exactly one of 'project_ids' or 'version_ids' should be used.".to_string(),
+        ));
+    }
+
+    let start_date = data.start_date.unwrap_or(NaiveDate::MIN);
+    let end_date = data.end_date.unwrap_or(NaiveDate::MAX);
+
+    let mut hm = HashMap::new();
+
+    let playtimes = if let Some(project_ids) = data.project_ids.clone() {
+        // Submitted project_ids are filtered by the user's permissions
+        let ids = project_ids
+            .iter()
+            .map(|id| Ok(ProjectId(parse_base62(id)?).into()))
+            .collect::<Result<Vec<_>, ApiError>>()?;
+        let projects = project_item::Project::get_many_ids(&ids, &**pool, &redis).await?;
+        let ids: Vec<ProjectId> = filter_authorized_projects(projects, &user_option, &pool)
+            .await?
+            .into_iter()
+            .map(|x| x.id)
+            .collect::<Vec<_>>();
+
+        for id in &ids {
+            hm.insert(to_base62(id.0), HashMap::new());
+        }
+        // Get the playtimes
+        analytics_queue
+            .fetch_playtimes(
+                Some(ids),
+                None,
+                start_date,
+                end_date,
+                clickhouse.into_inner(),
+            )
+            .await?
+    } else if let Some(version_ids) = data.version_ids.clone() {
+        // Submitted version_ids are filtered by the user's permissions
+        let ids = version_ids
+            .iter()
+            .map(|id| Ok(VersionId(parse_base62(id)?).into()))
+            .collect::<Result<Vec<_>, ApiError>>()?;
+        let versions = version_item::Version::get_many(&ids, &**pool, &redis).await?;
+        let ids: Vec<VersionId> = filter_authorized_versions(versions, &user_option, &pool)
+            .await?
+            .into_iter()
+            .map(|x| x.id)
+            .collect::<Vec<_>>();
+
+        for id in &ids {
+            hm.insert(to_base62(id.0), HashMap::new());
+        }
+        // Get the playtimes
+        analytics_queue
+            .fetch_playtimes(
+                None,
+                Some(ids),
+                start_date,
+                end_date,
+                clickhouse.into_inner(),
+            )
+            .await?
+    } else {
+        // unreachable
+        return Err(ApiError::InvalidInput(
+            "Exactly one of 'project_ids' or 'version_ids' must be used.".to_string(),
+        ));
+    };
+
+    for playtime in playtimes {
+        let id_string = to_base62(playtime.id);
+        if let Some(hm) = hm.get_mut(&id_string) {
+            hm.insert(
+                playtime.day.to_string(),
+                FetchedPlaytime {
+                    day: playtime.day,
+                    total_seconds: playtime.total_seconds,
+                    loader_seconds: playtime
+                        .loader_seconds
+                        .into_iter()
+                        .collect::<HashMap<_, _>>(),
+                    game_version_seconds: playtime
+                        .game_version_seconds
+                        .into_iter()
+                        .collect::<HashMap<_, _>>(),
+                    parent_seconds: playtime
+                        .parent_seconds
+                        .into_iter()
+                        .map(|(k, v)| (VersionId(k), v))
+                        .collect::<HashMap<_, _>>(),
+                },
+            );
+        }
+    }
+
+    Ok(HttpResponse::Ok().json(hm))
+}

src/routes/v2/mod.rs

@@ -1,4 +1,5 @@
 mod admin;
+mod analytics_get;
 mod moderation;
 mod notifications;
 pub(crate) mod project_creation;
@@ -21,6 +22,7 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
         actix_web::web::scope("v2")
             .wrap(default_cors())
             .configure(admin::config)
+            .configure(analytics_get::config)
             .configure(crate::auth::session::config)
             .configure(crate::auth::flows::config)
             .configure(crate::auth::pats::config)