From d5c0e4aa94fb608f397f48bc795d97145825bf35 Mon Sep 17 00:00:00 2001 From: GreenAppers Date: Thu, 14 Nov 2024 14:37:36 -0800 Subject: [PATCH] Begin optimizing analytics --- src/components/Analytics.tsx | 313 +++++++++++++++++++---------------- src/constants.ts | 72 +++++++- src/main.ts | 17 +- src/preload.ts | 20 +-- src/types.d.ts | 8 + src/utils/auth.ts | 78 +-------- src/utils/gamelog.ts | 20 ++- src/utils/timeseries.ts | 89 ++++++---- 8 files changed, 330 insertions(+), 287 deletions(-) diff --git a/src/components/Analytics.tsx b/src/components/Analytics.tsx index de501eb..b1f26da 100644 --- a/src/components/Analytics.tsx +++ b/src/components/Analytics.tsx @@ -16,8 +16,8 @@ import { } from '@chakra-ui/icons' import React, { useEffect, useState } from 'react' -import type { TimeSeries } from '../types' -import { addSampleToTimeseries } from '../utils/timeseries' +import type { TimeSeries, TimeValue } from '../types' +import { addSamplesToTimeseries } from '../utils/timeseries' import TimeseriesChart from './TimeseriesChart' import { QUERY_KEYS, STORE_KEYS } from '../constants' import { useQuery } from '@tanstack/react-query' @@ -36,76 +36,112 @@ interface GameAnalytics { timeSeries: Record } +interface TimeseriesUpdates { + sources: Set + values: Record +} + +type PlayersTimeseriesUpdates = Record + const soldContainer = /Successfully sold a container worth: \$([,\d]+.\d+)!/ const soldContainer2 = /Sold \d+ item\(s\) for \$([,\d]+.\d+)!/ const playerKilledPVPLegacy = /(\S+) has been killed by (\S+) with ([.\d]+) health left./ -function topUpAnalyticsTimeSeries(analytics: Record) { - const now = new Date() - const result = { ...analytics } - for (const key in analytics) { - const userAnalytics = analytics[key] - for (const seriesName in userAnalytics.timeSeries) { - const timeseries = userAnalytics.timeSeries[seriesName] - result[key].timeSeries[seriesName] = addSampleToTimeseries( - 0, - now, - timeseries, - now - ) - } - } - return result +const formatPlayerKey = (userName: string, serverName: string) => + `${userName}@${serverName}` + +const parsePlayerKey = (key: string) => { + const [userName, serverName] = key.split('@') + return { userName, serverName } } -function updateAnalyticsTimeSeries( - analytics: Record, - window: AnalyticsWindow, +const ensureAnalyticsTimeSeriesUpdate = ( + updates: PlayersTimeseriesUpdates, + userName: string, + serverName: string +) => { + const key = formatPlayerKey(userName, serverName) + return updates[key] || (updates[key] = { sources: new Set(), values: {} }) +} + +const addAnalyticsTimeSeriesUpdate = ( + updates: PlayersTimeseriesUpdates, userName: string, serverName: string, seriesName: string, - value: number, timestamp: Date, - source?: string + value: number +) => { + const userUpdates = ensureAnalyticsTimeSeriesUpdate( + updates, + userName, + serverName + ) + const userSeriesUpdates = + userUpdates.values[seriesName] || (userUpdates.values[seriesName] = []) + userSeriesUpdates.push({ time: timestamp.getTime(), value }) +} + +function updateAnalyticsTimeSeries( + analytics: Record, + window: AnalyticsWindow, + updates: PlayersTimeseriesUpdates ) { - const key = `${userName}@${serverName}` - let userAnalytics = analytics[key] - if (!userAnalytics) { - userAnalytics = { + const result = { ...analytics } + for (const [key, seriesUpdates] of Object.entries(updates)) { + const { userName, serverName } = parsePlayerKey(key) + const userAnalytics = analytics[key] || { gamelogs: [], userName: userName, serverName: serverName, timeSeries: {}, } + const updatedUserAnalytics = (result[key] = { + ...userAnalytics, + gamelogs: Array.from( + new Set([...userAnalytics.gamelogs, ...seriesUpdates.sources]) + ), + timeSeries: { ...userAnalytics.timeSeries }, + }) + for (const [seriesName, values] of Object.entries(seriesUpdates.values)) { + const timeseries = userAnalytics.timeSeries[seriesName] || { + duration: window.duration, + samples: window.samples, + buckets: [], + } + updatedUserAnalytics.timeSeries[seriesName] = addSamplesToTimeseries( + timeseries, + values + ) + } } - let timeseries = userAnalytics.timeSeries[seriesName] - if (!timeseries) { - timeseries = { - duration: window.duration, - samples: window.samples, - buckets: [], + const players = Object.keys(result) + for (const player of players) { + const userAnalytics = result[player] + if ( + (!userAnalytics.userName || !userAnalytics.serverName) && + !Object.keys(userAnalytics.timeSeries).length + ) { + delete result[player] } } - timeseries = addSampleToTimeseries(value, timestamp, timeseries, timestamp) - const result = { - ...analytics, - [key]: { - ...userAnalytics, - gamelogs: [ - ...userAnalytics.gamelogs, - ...(!source || userAnalytics.gamelogs.find((x) => x === source) - ? [] - : [source]), - ], - timeSeries: { - ...userAnalytics.timeSeries, - [seriesName]: timeseries, - }, - }, + return result +} + +function topUpAnalyticsTimeSeries(analytics: Record) { + const now = new Date() + const result = { ...analytics } + for (const key in analytics) { + const userAnalytics = analytics[key] + for (const seriesName in userAnalytics.timeSeries) { + const timeseries = userAnalytics.timeSeries[seriesName] + result[key].timeSeries[seriesName] = addSamplesToTimeseries(timeseries, [ + { time: now.getTime(), value: 0 }, + ]) + } } - // console.log('updateAnalyticsTimeSeries result', result) return result } @@ -138,108 +174,103 @@ export function Analytics() { const handle = window.api.readGameLogs( gameLogDirectories.data, - ( - userName: string, - serverName: string, - content: string, - timestamp: Date, - source: string - ) => { - if (!earliestTimestamp) earliestTimestamp = timestamp + (lines) => { + const updates: PlayersTimeseriesUpdates = {} + for (const line of lines) { + if (!earliestTimestamp) earliestTimestamp = line.timestamp + ensureAnalyticsTimeSeriesUpdate( + updates, + line.userName, + line.serverName + ).sources.add(line.source) - const playerKilledPVPLegacyMatch = content.match(playerKilledPVPLegacy) - if (playerKilledPVPLegacyMatch) { - let userNameMatch = false - if (playerKilledPVPLegacyMatch[1] === userName) { - setAnalytics((analytics) => - updateAnalyticsTimeSeries( - analytics, - analyticsWindow, - userName, - serverName, + const playerKilledPVPLegacyMatch = line.content.match( + playerKilledPVPLegacy + ) + if (playerKilledPVPLegacyMatch) { + let userNameMatch = false + if (playerKilledPVPLegacyMatch[1] === line.userName) { + addAnalyticsTimeSeriesUpdate( + updates, + line.userName, + line.serverName, 'deaths', - 1, - timestamp, - source + line.timestamp, + 1 ) - ) - userNameMatch = true - } - if (playerKilledPVPLegacyMatch[2] === userName) { - setAnalytics((analytics) => - updateAnalyticsTimeSeries( - analytics, - analyticsWindow, - userName, - serverName, + userNameMatch = true + } + if (playerKilledPVPLegacyMatch[2] === line.userName) { + addAnalyticsTimeSeriesUpdate( + updates, + line.userName, + line.serverName, 'kills', - 1, - timestamp, - source + line.timestamp, + 1 ) + userNameMatch = true + } + if (userNameMatch) + console.log( + 'PVP kill', + playerKilledPVPLegacyMatch[1], + 'was killed by', + playerKilledPVPLegacyMatch[2], + 'with', + playerKilledPVPLegacyMatch[3], + 'health left', + line.source + ) + continue + } + + let soldContainerValue = 0 + const soldContainerMatch = line.content.match(soldContainer) + if (soldContainerMatch) { + soldContainerValue = parseFloat( + soldContainerMatch[1].replace(/,/g, '') ) - userNameMatch = true } - if (userNameMatch) - console.log( - 'PVP kill', - playerKilledPVPLegacyMatch[1], - 'was killed by', - playerKilledPVPLegacyMatch[2], - 'with', - playerKilledPVPLegacyMatch[3], - 'health left', - source + const soldContainerMatch2 = line.content.match(soldContainer2) + if (soldContainerMatch2) { + soldContainerValue = parseFloat( + soldContainerMatch2[1].replace(/,/g, '') ) - return - } - - let soldContainerValue = 0 - const soldContainerMatch = content.match(soldContainer) - if (soldContainerMatch) { - soldContainerValue = parseFloat( - soldContainerMatch[1].replace(/,/g, '') - ) - } - const soldContainerMatch2 = content.match(soldContainer2) - if (soldContainerMatch2) { - soldContainerValue = parseFloat( - soldContainerMatch2[1].replace(/,/g, '') - ) - } + } - if (soldContainerValue) { - total += soldContainerValue - setAnalytics((analytics) => - updateAnalyticsTimeSeries( - analytics, - analyticsWindow, - userName, - serverName, + if (soldContainerValue) { + total += soldContainerValue + addAnalyticsTimeSeriesUpdate( + updates, + line.userName, + line.serverName, 'sold', + line.timestamp, + soldContainerValue + ) + const totalSeconds = + (line.timestamp.getTime() - earliestTimestamp.getTime()) / 1000 + const ratePerMinute = (total * 60) / totalSeconds + console.log( + `${line.userName}@${line.serverName} Sold container value`, soldContainerValue, - timestamp, - source + 'Total', + total, + 'Rate', + ratePerMinute.toFixed(2), + 'per minute', + (ratePerMinute * 60).toFixed(2), + 'per hour', + line.timestamp, + line.source ) - ) - const totalSeconds = - (timestamp.getTime() - earliestTimestamp.getTime()) / 1000 - const ratePerMinute = (total * 60) / totalSeconds - console.log( - `${userName}@${serverName} Sold container value`, - soldContainerValue, - 'Total', - total, - 'Rate', - ratePerMinute.toFixed(2), - 'per minute', - (ratePerMinute * 60).toFixed(2), - 'per hour', - timestamp, - source - ) - return + continue + } } + setAnalytics((analytics) => + updateAnalyticsTimeSeries(analytics, analyticsWindow, updates) + ) }, analyticsWindow.beginDate, analyticsWindow.endDate @@ -276,7 +307,7 @@ export function Analytics() { onChange={(event) => setAnalyticsProfile(event.target.value)} > {Object.keys(analytics).map((profile) => ( - + ))} @@ -287,7 +318,7 @@ export function Analytics() { : } + icon={showGameLogFiles ? : } onClick={() => setShowGameLogFiles((x) => !x)} /> diff --git a/src/constants.ts b/src/constants.ts index ae59a57..27d59e0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,15 +1,42 @@ import { z } from 'zod' -import { - minecraftLoginResponse, - minecraftProfile, - xstsAuthorizeResponse, -} from './utils/auth' export enum ModLoaderName { Fabric = 'Fabric', None = 'None', } +export const minecraftLoginResponse = z.object({ + username: z.string(), + roles: z.array(z.string()), + access_token: z.string(), + token_type: z.string(), + expires_in: z.number(), +}) + +export const minecraftProfileState = z.enum(['ACTIVE', 'INACTIVE']) +export const minecraftSkinVariant = z.enum(['SLIM', 'CLASSIC']) + +export const minecraftSkin = z.object({ + id: z.string(), + state: minecraftProfileState, + url: z.string(), + variant: minecraftSkinVariant, +}) + +export const minecraftCape = z.object({ + id: z.string(), + state: minecraftProfileState, + url: z.string(), + alias: z.string(), +}) + +export const minecraftProfile = z.object({ + id: z.string(), + name: z.string(), + skins: z.array(minecraftSkin), + capes: z.array(minecraftCape), +}) + export const mojangVersionManifest = z.object({ id: z.string(), type: z.string(), @@ -118,6 +145,37 @@ export const fabricVersionDetails = z.object({ ), }) +export const xboxLiveProfile = z.object({ + profileUsers: z.array( + z.object({ + id: z.string(), + hostId: z.optional(z.string()).nullable(), + settings: z.array( + z.object({ + id: z.string(), + value: z.string(), + }) + ), + isSponsoredUser: z.boolean(), + }) + ), +}) + +export const xstsAuthorizeResponse = z.object({ + IssueInstant: z.string(), + NotAfter: z.string(), + Token: z.string(), + DisplayClaims: z.object({ + xui: z.array( + z.object({ + gtg: z.optional(z.string()), + uhs: z.string(), + xid: z.optional(z.string()), + }) + ), + }), +}) + export const gameAccount = z.object({ active: z.boolean(), profile: minecraftProfile, @@ -136,12 +194,16 @@ export const gameInstall = z.object({ mods: z.optional(z.array(z.string())), }) +export type MinecraftLoginResponse = z.infer< + typeof minecraftLoginResponse +> export type MojangLibrary = z.infer export type MojangRule = z.infer export type MojangVersionDetails = z.infer export type MojangVersionManifest = z.infer export type MojangVersionManifests = z.infer export type MojangStringsTemplate = z.infer +export type XSTSAuthorizeResponse = z.infer export type GameAccount = z.infer export type GameInstall = z.infer diff --git a/src/main.ts b/src/main.ts index 17a1c66..e100617 100644 --- a/src/main.ts +++ b/src/main.ts @@ -146,20 +146,9 @@ app.on('ready', () => { startDate, endDate ) - readGameLogs( - gameLogContext, - logFiles, - (context, content, timestamp, source) => { - mainWindow?.webContents.send( - CHANNELS.readGameLogs, - context.userName, - context.serverName, - content, - timestamp, - source - ) - } - ).catch((error) => log.error('readGameLogChannel error', error)) + readGameLogs(gameLogContext, logFiles, (lines) => { + mainWindow?.webContents.send(CHANNELS.readGameLogs, lines) + }).catch((error) => log.error('readGameLogChannel error', error)) return logFiles } ) diff --git a/src/preload.ts b/src/preload.ts index 2fb7fb6..3fd15ef 100644 --- a/src/preload.ts +++ b/src/preload.ts @@ -4,6 +4,7 @@ import { contextBridge, ipcRenderer } from 'electron/renderer' import { CHANNELS, GameInstall, LAUNCH_CHANNEL } from './constants' +import type { GameLogLine } from './types' export type Listener = ( event: Electron.IpcRendererEvent, @@ -51,27 +52,14 @@ export const api = { }, readGameLogs: ( gameLogDirectories: string[], - callback: ( - userName: string, - serverName: string, - content: string, - timestamp: Date, - source: string - ) => void, + callback: (lines: GameLogLine[]) => void, beginDate?: Date, endDate?: Date - ): ListenHandle<[string, string, string, Date, string]> => { + ): ListenHandle<[GameLogLine[]]> => { ipcRenderer .invoke(CHANNELS.readGameLogs, gameLogDirectories, beginDate, endDate) .catch((error) => console.log(`${CHANNELS.readGameLogs} error`, error)) - const listener: Listener<[string, string, string, Date, string]> = ( - _event, - userName, - serverName, - content, - timestamp, - source - ) => callback(userName, serverName, content, timestamp, source) + const listener: Listener<[GameLogLine[]]> = (_event, lines) => callback(lines) ipcRenderer.on(CHANNELS.readGameLogs, listener) return { channel: CHANNELS.readGameLogs, listener } }, diff --git a/src/types.d.ts b/src/types.d.ts index 8a515b4..47bd740 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -17,6 +17,14 @@ interface GameLog { mtimeMs: number } +interface GameLogLine { + userName: string + serverName: string + content: string + timestamp: Date + source: string +} + interface PieRaySample { chunk: Chunk direction?: ChunkDirection diff --git a/src/utils/auth.ts b/src/utils/auth.ts index 5931ff0..1494cd2 100644 --- a/src/utils/auth.ts +++ b/src/utils/auth.ts @@ -1,7 +1,13 @@ import axios from 'axios' import Store from 'electron-store' -import { z } from 'zod' -import type { GameAccount, StoreSchema } from '../constants' +import { + GameAccount, + minecraftLoginResponse, + minecraftProfile, + StoreSchema, + xboxLiveProfile, + xstsAuthorizeResponse, +} from '../constants' import { updateGameAccount } from '../store' // References: @@ -9,74 +15,6 @@ import { updateGameAccount } from '../store' // - https://minecraft-launcher-lib.readthedocs.io/en/stable/tutorial/microsoft_login.html // - https://learn.microsoft.com/en-us/gaming/gdk/_content/gc/commerce/service-to-service/xstore-requesting-a-userstoreid-from-services -export const minecraftLoginResponse = z.object({ - username: z.string(), - roles: z.array(z.string()), - access_token: z.string(), - token_type: z.string(), - expires_in: z.number(), -}) - -export const minecraftProfileState = z.enum(['ACTIVE', 'INACTIVE']) -export const minecraftSkinVariant = z.enum(['SLIM', 'CLASSIC']) - -export const minecraftSkin = z.object({ - id: z.string(), - state: minecraftProfileState, - url: z.string(), - variant: minecraftSkinVariant, -}) - -export const minecraftCape = z.object({ - id: z.string(), - state: minecraftProfileState, - url: z.string(), - alias: z.string(), -}) - -export const minecraftProfile = z.object({ - id: z.string(), - name: z.string(), - skins: z.array(minecraftSkin), - capes: z.array(minecraftCape), -}) - -export const xboxLiveProfile = z.object({ - profileUsers: z.array( - z.object({ - id: z.string(), - hostId: z.optional(z.string()).nullable(), - settings: z.array( - z.object({ - id: z.string(), - value: z.string(), - }) - ), - isSponsoredUser: z.boolean(), - }) - ), -}) - -export const xstsAuthorizeResponse = z.object({ - IssueInstant: z.string(), - NotAfter: z.string(), - Token: z.string(), - DisplayClaims: z.object({ - xui: z.array( - z.object({ - gtg: z.optional(z.string()), - uhs: z.string(), - xid: z.optional(z.string()), - }) - ), - }), -}) - -export type MinecraftLauncherLoginResponse = z.infer< - typeof minecraftLoginResponse -> -export type XSTSAuthorizeResponse = z.infer - export const formatXSTSToken = (uhs: string, token: string) => `XBL3.0 x=${uhs};${token}` diff --git a/src/utils/gamelog.ts b/src/utils/gamelog.ts index bf1a8e2..d3ba17f 100644 --- a/src/utils/gamelog.ts +++ b/src/utils/gamelog.ts @@ -5,7 +5,7 @@ import split2 from 'split2' import stream from 'stream' import { Tail } from 'tail' import zlib from 'zlib' -import type { GameLog } from '../types' +import type { GameLog, GameLogLine } from '../types' export interface GameLogContext { serverName: string @@ -106,16 +106,20 @@ export async function findGameLogFiles( export async function readGameLogs( context: GameLogContext, gamelogs: GameLog[], - callback: ( - context: GameLogContext, - content: string, - timestamp: Date, - path: string - ) => void + callback: (lines: GameLogLine[]) => void ) { const handleGameLogLine = (line: string, path: string) => { const parsed = parseGameLogLine(context, line, path) - if (parsed) callback(context, parsed.content, parsed.timestamp, path) + if (!parsed) return + callback([ + { + userName: parsed.context.userName, + serverName: parsed.context.serverName, + content: parsed.content, + timestamp: parsed.timestamp, + source: path, + }, + ]) } resetGameLogContext(context, gamelogs) diff --git a/src/utils/timeseries.ts b/src/utils/timeseries.ts index 00a64c8..e1c6a91 100644 --- a/src/utils/timeseries.ts +++ b/src/utils/timeseries.ts @@ -1,51 +1,74 @@ -import type { TimeSeries } from '../types' +import type { TimeSeries, TimeValue } from '../types' -export function addSampleToTimeseries( - value: number, - timestamp: Date, +export function addSamplesToTimeseries( timeseries: TimeSeries, + values: TimeValue[], adjustWindowNow?: Date | undefined ) { - let changed = false const buckets = [...timeseries.buckets] + let changed = false if (adjustWindowNow) { - const minLastBucket = - adjustWindowNow.getTime() - (adjustWindowNow.getTime() % timeseries.duration) - const minFirstBucket = - minLastBucket - (timeseries.samples - 1) * timeseries.duration - - while (buckets.length && buckets[0].time < minFirstBucket) { - buckets.shift() + if ( + adjustTimeseriesBucketsWindow( + timeseries, + buckets, + adjustWindowNow.getTime() + ) + ) changed = true - } + } - if (!buckets.length) { - buckets.push({ time: minFirstBucket, value: 0 }) - changed = true + for (const value of values) { + if (!adjustWindowNow) { + if (adjustTimeseriesBucketsWindow(timeseries, buckets, value.time)) + changed = true } - - while (buckets.length < timeseries.samples) { - buckets.push({ - time: buckets[buckets.length - 1].time + timeseries.duration, - value: 0, - }) + const firstBucket = buckets[0].time + const lastBucket = buckets[buckets.length - 1].time + if ( + value.time >= firstBucket && + value.time < lastBucket + timeseries.duration + ) { + const bucketIndex = Math.floor( + (value.time - firstBucket) / timeseries.duration + ) + buckets[bucketIndex].value += value.value changed = true } } - const firstBucket = buckets[0].time - const lastBucket = buckets[buckets.length - 1].time - if ( - timestamp.getTime() >= firstBucket && - timestamp.getTime() < lastBucket + timeseries.duration - ) { - const bucketIndex = Math.floor( - (timestamp.getTime() - firstBucket) / timeseries.duration - ) - buckets[bucketIndex].value += value + return changed ? { ...timeseries, buckets } : timeseries +} + +export function adjustTimeseriesBucketsWindow( + timeseries: TimeSeries, + buckets: TimeValue[], + adjustWindowNow: number +) { + const minLastBucket = + adjustWindowNow - (adjustWindowNow % timeseries.duration) + const minFirstBucket = + minLastBucket - (timeseries.samples - 1) * timeseries.duration + let changed = false + + while (buckets.length && buckets[0].time < minFirstBucket) { + buckets.shift() changed = true } - return changed ? { ...timeseries, buckets } : timeseries + if (!buckets.length) { + buckets.push({ time: minFirstBucket, value: 0 }) + changed = true + } + + while (buckets.length < timeseries.samples) { + buckets.push({ + time: buckets[buckets.length - 1].time + timeseries.duration, + value: 0, + }) + changed = true + } + + return changed }