From b7b71e7e88986bbd7f76f7d6cb55ee570b1e53a3 Mon Sep 17 00:00:00 2001 From: tudor <7089284+tudddorrr@users.noreply.github.com> Date: Tue, 17 Jun 2025 08:58:52 +0100 Subject: [PATCH] add game stat metrics to index endpoint --- src/entities/game-stat.ts | 143 ++++++++++++++++++++++ src/services/api/game-stat-api.service.ts | 136 +------------------- src/services/game-stat.service.ts | 7 ++ tests/services/game-stat/index.test.ts | 139 +++++++++++++++++++++ 4 files changed, 295 insertions(+), 130 deletions(-) diff --git a/src/entities/game-stat.ts b/src/entities/game-stat.ts index 6f8f67c3..e00504b6 100644 --- a/src/entities/game-stat.ts +++ b/src/entities/game-stat.ts @@ -2,6 +2,25 @@ import { EntityManager, Entity, ManyToOne, PrimaryKey, Property, Collection, One import { Request, Required, ValidationCondition } from 'koa-clay' import Game from './game' import PlayerGameStat from './player-game-stat' +import { ClickHouseClient } from '@clickhouse/client' +import Player from './player' +import { formatDateForClickHouse } from '../lib/clickhouse/formatDateTime' +import { endOfDay } from 'date-fns' + +type GlobalValueMetrics = { + minValue: number + maxValue: number + medianValue: number + averageValue: number + averageChange: number +} + +type PlayerValueMetrics = { + minValue: number + maxValue: number + medianValue: number + averageValue: number +} @Entity() export default class GameStat { @@ -93,6 +112,8 @@ export default class GameStat { @Property({ onUpdate: () => new Date() }) updatedAt: Date = new Date() + metrics?: { globalCount: number, globalValue: GlobalValueMetrics, playerValue: PlayerValueMetrics } + constructor(game: Game) { this.game = game } @@ -109,6 +130,127 @@ export default class GameStat { .reduce((acc, curr) => acc += curr.value, 0) } + async buildMetricsWhereConditions(startDate?: string, endDate?: string, player?: Player): Promise { + let whereConditions = `WHERE game_stat_id = ${this.id}` + + if (startDate) { + whereConditions += ` AND created_at >= '${formatDateForClickHouse(new Date(startDate))}'` + } + if (endDate) { + // when using YYYY-MM-DD, use the end of the day + const end = endDate.length === 10 ? endOfDay(new Date(endDate)) : new Date(endDate) + whereConditions += ` AND created_at <= '${formatDateForClickHouse(end)}'` + } + if (player) { + await player.aliases.loadItems() + const aliasIds = player.aliases.getIdentifiers() + whereConditions += ` AND player_alias_id IN (${aliasIds.join(', ')})` + } + + return whereConditions + } + + async loadMetrics(clickhouse: ClickHouseClient, metricsStartDate?: string, metricsEndDate?: string): Promise { + const whereConditions = await this.buildMetricsWhereConditions(metricsStartDate, metricsEndDate) + + const [globalCount, globalValue] = await this.getGlobalValueMetrics(clickhouse, whereConditions) + const playerValue = await this.getPlayerValueMetrics(clickhouse, whereConditions) + + this.metrics = { + globalCount, + globalValue, + playerValue + } + } + + async getGlobalValueMetrics( + clickhouse: ClickHouseClient, + whereConditions: string + ): Promise<[number, GlobalValueMetrics]> { + const query = ` + SELECT + count() as rawCount, + min(global_value) as minValue, + max(global_value) as maxValue, + median(global_value) as medianValue, + avg(global_value) as averageValue, + avg(change) as averageChange + FROM player_game_stat_snapshots + ${whereConditions} + ` + + const res = await clickhouse.query({ + query: query, + format: 'JSONEachRow' + }).then((res) => res.json<{ + rawCount: string | number + minValue: number + maxValue: number + medianValue: number | null + averageValue: number | null + averageChange: number | null + }>()) + + const { + rawCount, + minValue, + maxValue, + medianValue, + averageValue, + averageChange + } = res[0] + + return [ + Number(rawCount), + { + minValue: minValue || this.defaultValue, + maxValue: maxValue || this.defaultValue, + medianValue: medianValue ?? this.defaultValue, + averageValue: averageValue ?? this.defaultValue, + averageChange: averageChange ?? 0 + } + ] + } + + async getPlayerValueMetrics( + clickhouse: ClickHouseClient, + whereConditions: string + ): Promise { + const query = ` + SELECT + min(value) as minValue, + max(value) as maxValue, + median(value) as medianValue, + avg(value) as averageValue + FROM player_game_stat_snapshots + ${whereConditions} + ` + + const res = await clickhouse.query({ + query: query, + format: 'JSONEachRow' + }).then((res) => res.json<{ + minValue: number + maxValue: number + medianValue: number | null + averageValue: number | null + }>()) + + const { + minValue, + maxValue, + medianValue, + averageValue + } = res[0] + + return { + minValue: minValue || this.defaultValue, + maxValue: maxValue || this.defaultValue, + medianValue: medianValue ?? this.defaultValue, + averageValue: averageValue ?? this.defaultValue + } + } + toJSON() { return { id: this.id, @@ -116,6 +258,7 @@ export default class GameStat { name: this.name, global: this.global, globalValue: this.hydratedGlobalValue ?? this.globalValue, + metrics: this.metrics, defaultValue: this.defaultValue, maxChange: this.maxChange, minValue: this.minValue, diff --git a/src/services/api/game-stat-api.service.ts b/src/services/api/game-stat-api.service.ts index b7f495ee..e43b8166 100644 --- a/src/services/api/game-stat-api.service.ts +++ b/src/services/api/game-stat-api.service.ts @@ -1,5 +1,5 @@ import { EntityManager } from '@mikro-orm/mysql' -import { differenceInSeconds, endOfDay } from 'date-fns' +import { differenceInSeconds } from 'date-fns' import { HasPermission, Request, Response, Route, Validate } from 'koa-clay' import GameStatAPIDocs from '../../docs/game-stat-api.docs' import GameStat from '../../entities/game-stat' @@ -11,115 +11,9 @@ import { ClickHouseClient } from '@clickhouse/client' import PlayerGameStatSnapshot, { ClickHousePlayerGameStatSnapshot } from '../../entities/player-game-stat-snapshot' import Player from '../../entities/player' import { buildDateValidationSchema } from '../../lib/dates/dateValidationSchema' -import { formatDateForClickHouse } from '../../lib/clickhouse/formatDateTime' import PlayerAlias from '../../entities/player-alias' import { TraceService } from '../../lib/tracing/trace-service' -type GlobalValueMetrics = { - minValue: number - maxValue: number - medianValue: number - averageValue: number - averageChange: number -} - -type PlayerValueMetrics = { - minValue: number - maxValue: number - medianValue: number - averageValue: number -} - -async function getGlobalValueMetrics( - clickhouse: ClickHouseClient, - stat: GameStat, - whereConditions: string -): Promise<[number, GlobalValueMetrics]> { - const query = ` - SELECT - count() as rawCount, - min(global_value) as minValue, - max(global_value) as maxValue, - median(global_value) as medianValue, - avg(global_value) as averageValue, - avg(change) as averageChange - FROM player_game_stat_snapshots - ${whereConditions} - ` - - const res = await clickhouse.query({ - query: query, - format: 'JSONEachRow' - }).then((res) => res.json<{ - rawCount: string | number - minValue: number - maxValue: number - medianValue: number | null - averageValue: number | null - averageChange: number | null - }>()) - - const { - rawCount, - minValue, - maxValue, - medianValue, - averageValue, - averageChange - } = res[0] - - return [ - Number(rawCount), - { - minValue: minValue || stat.defaultValue, - maxValue: maxValue || stat.defaultValue, - medianValue: medianValue ?? stat.defaultValue, - averageValue: averageValue ?? stat.defaultValue, - averageChange: averageChange ?? 0 - } - ] -} - -async function getPlayerValueMetrics( - clickhouse: ClickHouseClient, - stat: GameStat, - whereConditions: string -): Promise { - const query = ` - SELECT - min(value) as minValue, - max(value) as maxValue, - median(value) as medianValue, - avg(value) as averageValue - FROM player_game_stat_snapshots - ${whereConditions} - ` - - const res = await clickhouse.query({ - query: query, - format: 'JSONEachRow' - }).then((res) => res.json<{ - minValue: number - maxValue: number - medianValue: number | null - averageValue: number | null - }>()) - - const { - minValue, - maxValue, - medianValue, - averageValue - } = res[0] - - return { - minValue: minValue || stat.defaultValue, - maxValue: maxValue || stat.defaultValue, - medianValue: medianValue ?? stat.defaultValue, - averageValue: averageValue ?? stat.defaultValue - } -} - @TraceService() export default class GameStatAPIService extends APIService { @Route({ @@ -261,18 +155,7 @@ export default class GameStatAPIService extends APIService { const stat: GameStat = req.ctx.state.stat const player: Player = req.ctx.state.player - await player.aliases.loadItems() - const aliasIds = player.aliases.getIdentifiers() - - let whereConditions = `WHERE game_stat_id = ${stat.id} AND player_alias_id IN (${aliasIds.join(', ')})` - if (startDate) { - whereConditions += ` AND created_at >= '${formatDateForClickHouse(new Date(startDate))}'` - } - if (endDate) { - // when using YYYY-MM-DD, use the end of the day - const end = endDate.length === 10 ? endOfDay(new Date(endDate)) : new Date(endDate) - whereConditions += ` AND created_at <= '${formatDateForClickHouse(end)}'` - } + const whereConditions = await stat.buildMetricsWhereConditions(startDate, endDate, player) const query = ` WITH (SELECT count() FROM player_game_stat_snapshots ${whereConditions}) AS count @@ -328,15 +211,8 @@ export default class GameStatAPIService extends APIService { req.ctx.throw(400, 'This stat is not globally available') } - let whereConditions = `WHERE game_stat_id = ${stat.id}` - if (startDate) { - whereConditions += ` AND created_at >= '${formatDateForClickHouse(new Date(startDate))}'` - } - if (endDate) { - // when using YYYY-MM-DD, use the end of the day - const end = endDate.length === 10 ? endOfDay(new Date(endDate)) : new Date(endDate) - whereConditions += ` AND created_at <= '${formatDateForClickHouse(end)}'` - } + let whereConditions = await stat.buildMetricsWhereConditions(startDate, endDate) + if (playerId) { try { const player = await em.repo(Player).findOneOrFail({ @@ -363,8 +239,8 @@ export default class GameStatAPIService extends APIService { }).then((res) => res.json()) const history = await Promise.all(snapshots.map((snapshot) => new PlayerGameStatSnapshot().hydrate(em, snapshot))) - const [count, globalValue] = await getGlobalValueMetrics(clickhouse, stat, whereConditions) - const playerValue = await getPlayerValueMetrics(clickhouse, stat, whereConditions) + const [count, globalValue] = await stat.getGlobalValueMetrics(clickhouse, whereConditions) + const playerValue = await stat.getPlayerValueMetrics(clickhouse, whereConditions) return { status: 200, diff --git a/src/services/game-stat.service.ts b/src/services/game-stat.service.ts index bd8a115b..2422c034 100644 --- a/src/services/game-stat.service.ts +++ b/src/services/game-stat.service.ts @@ -9,20 +9,27 @@ import PlayerGameStat from '../entities/player-game-stat' import triggerIntegrations from '../lib/integrations/triggerIntegrations' import updateAllowedKeys from '../lib/entities/updateAllowedKeys' import { TraceService } from '../lib/tracing/trace-service' +import { buildDateValidationSchema } from '../lib/dates/dateValidationSchema' @TraceService() export default class GameStatService extends Service { @Route({ method: 'GET' }) + @Validate({ + query: buildDateValidationSchema(false, false) + }) @HasPermission(GameStatPolicy, 'index') async index(req: Request): Promise { + const { metricsStartDate, metricsEndDate } = req.query + const em: EntityManager = req.ctx.em const stats = await em.getRepository(GameStat).find({ game: req.ctx.state.game }) for (const stat of stats) { if (stat.global) { await stat.recalculateGlobalValue(req.ctx.state.includeDevData) + await stat.loadMetrics(req.ctx.clickhouse, metricsStartDate, metricsEndDate) } } diff --git a/tests/services/game-stat/index.test.ts b/tests/services/game-stat/index.test.ts index 2fc7bcc8..48f704c3 100644 --- a/tests/services/game-stat/index.test.ts +++ b/tests/services/game-stat/index.test.ts @@ -4,6 +4,7 @@ import PlayerFactory from '../../fixtures/PlayerFactory' import PlayerGameStatFactory from '../../fixtures/PlayerGameStatFactory' import createUserAndToken from '../../utils/createUserAndToken' import createOrganisationAndGame from '../../utils/createOrganisationAndGame' +import PlayerGameStatSnapshot from '../../../src/entities/player-game-stat-snapshot' describe('Game stat service - index', () => { it('should return a list of game stats', async () => { @@ -73,4 +74,142 @@ describe('Game stat service - index', () => { expect(res.body.stats[0].globalValue).toBe(50) }) + + it('should load metrics filtered by startDate', async () => { + const [organisation, game] = await createOrganisationAndGame() + const [token] = await createUserAndToken({}, organisation) + + const stat = await new GameStatFactory([game]).global().state(() => ({ globalValue: 0 })).one() + const player = await new PlayerFactory([game]).one() + await em.persistAndFlush([stat, player]) + + const values: [Date, number][] = [ + [new Date('2025-06-09T09:00:00.000Z'), 1], + [new Date('2025-06-10T09:00:00.000Z'), 5], + [new Date('2025-06-11T09:00:00.000Z'), 7] + ] + + await clickhouse.insert({ + table: 'player_game_stat_snapshots', + values: await Promise.all(values.map(async ([date, value]) => { + const playerStat = await new PlayerGameStatFactory().construct(player, stat).state(() => ({ value })).one() + await em.persistAndFlush(playerStat) + stat.globalValue += value + + const snapshot = new PlayerGameStatSnapshot() + snapshot.construct(playerStat.player.aliases[0], playerStat) + snapshot.change = value + snapshot.createdAt = date + + return snapshot.toInsertable() + })), + format: 'JSONEachRow' + }) + + const res = await request(app) + .get(`/games/${game.id}/game-stats`) + .query({ metricsStartDate: values[1][0].toISOString() }) + .auth(token, { type: 'bearer' }) + .expect(200) + + // only changes from the last 2 dates + expect(res.body.stats[0].metrics.globalCount).toBe(2) + expect(res.body.stats[0].metrics.globalValue.minValue).toBe(5) + expect(res.body.stats[0].metrics.globalValue.maxValue).toBe(12) + expect(res.body.stats[0].metrics.playerValue.minValue).toBe(5) + expect(res.body.stats[0].metrics.playerValue.maxValue).toBe(7) + }) + + it('should load metrics filtered by endDate', async () => { + const [organisation, game] = await createOrganisationAndGame() + const [token] = await createUserAndToken({}, organisation) + + const stat = await new GameStatFactory([game]).global().state(() => ({ globalValue: 0 })).one() + const player = await new PlayerFactory([game]).one() + await em.persistAndFlush([stat, player]) + + const values: [Date, number][] = [ + [new Date('2025-06-09T09:00:00.000Z'), 1], + [new Date('2025-06-10T09:00:00.000Z'), 5], + [new Date('2025-06-11T09:00:00.000Z'), 7] + ] + + await clickhouse.insert({ + table: 'player_game_stat_snapshots', + values: await Promise.all(values.map(async ([date, value]) => { + const playerStat = await new PlayerGameStatFactory().construct(player, stat).state(() => ({ value })).one() + await em.persistAndFlush(playerStat) + stat.globalValue += value + + const snapshot = new PlayerGameStatSnapshot() + snapshot.construct(playerStat.player.aliases[0], playerStat) + snapshot.change = value + snapshot.createdAt = date + + return snapshot.toInsertable() + })), + format: 'JSONEachRow' + }) + + const res = await request(app) + .get(`/games/${game.id}/game-stats`) + .query({ metricsEndDate: values[1][0].toISOString() }) + .auth(token, { type: 'bearer' }) + .expect(200) + + // only changes from the first 2 dates + expect(res.body.stats[0].metrics.globalCount).toBe(2) + expect(res.body.stats[0].metrics.globalValue.minValue).toBe(5) + expect(res.body.stats[0].metrics.globalValue.maxValue).toBe(13) + expect(res.body.stats[0].metrics.playerValue.minValue).toBe(1) + expect(res.body.stats[0].metrics.playerValue.maxValue).toBe(5) + }) + + it('should load metrics filtered by both startDate and endDate', async () => { + const [organisation, game] = await createOrganisationAndGame() + const [token] = await createUserAndToken({}, organisation) + + const stat = await new GameStatFactory([game]).global().state(() => ({ globalValue: 0 })).one() + const player = await new PlayerFactory([game]).one() + await em.persistAndFlush([stat, player]) + + const values: [Date, number][] = [ + [new Date('2025-06-09T09:00:00.000Z'), 1], + [new Date('2025-06-10T09:00:00.000Z'), 5], + [new Date('2025-06-11T09:00:00.000Z'), 7] + ] + + await clickhouse.insert({ + table: 'player_game_stat_snapshots', + values: await Promise.all(values.map(async ([date, value]) => { + const playerStat = await new PlayerGameStatFactory().construct(player, stat).state(() => ({ value })).one() + await em.persistAndFlush(playerStat) + stat.globalValue += value + + const snapshot = new PlayerGameStatSnapshot() + snapshot.construct(playerStat.player.aliases[0], playerStat) + snapshot.change = value + snapshot.createdAt = date + + return snapshot.toInsertable() + })), + format: 'JSONEachRow' + }) + + const res = await request(app) + .get(`/games/${game.id}/game-stats`) + .query({ + metricsStartDate: '2025-06-10', + metricsEndDate: '2025-06-10' + }) + .auth(token, { type: 'bearer' }) + .expect(200) + + // only changes from the middle date + expect(res.body.stats[0].metrics.globalCount).toBe(1) + expect(res.body.stats[0].metrics.globalValue.minValue).toBe(5) + expect(res.body.stats[0].metrics.globalValue.maxValue).toBe(5) + expect(res.body.stats[0].metrics.playerValue.minValue).toBe(5) + expect(res.body.stats[0].metrics.playerValue.maxValue).toBe(5) + }) })