Release 0.76.0

package.json

@@ -1,6 +1,6 @@
 {
   "name": "game-services",
-  "version": "0.75.0",
+  "version": "0.76.0",
   "description": "",
   "main": "src/index.ts",
   "scripts": {

src/entities/game-stat.ts

@@ -2,6 +2,25 @@ import { EntityManager, Entity, ManyToOne, PrimaryKey, Property, Collection, One
 import { Request, Required, ValidationCondition } from 'koa-clay'
 import Game from './game'
 import PlayerGameStat from './player-game-stat'
+import { ClickHouseClient } from '@clickhouse/client'
+import Player from './player'
+import { formatDateForClickHouse } from '../lib/clickhouse/formatDateTime'
+import { endOfDay } from 'date-fns'
+
+type GlobalValueMetrics = {
+  minValue: number
+  maxValue: number
+  medianValue: number
+  averageValue: number
+  averageChange: number
+}
+
+type PlayerValueMetrics = {
+  minValue: number
+  maxValue: number
+  medianValue: number
+  averageValue: number
+}
 
 @Entity()
 export default class GameStat {
@@ -93,6 +112,8 @@ export default class GameStat {
   @Property({ onUpdate: () => new Date() })
   updatedAt: Date = new Date()
 
+  metrics?: { globalCount: number, globalValue: GlobalValueMetrics, playerValue: PlayerValueMetrics }
+
   constructor(game: Game) {
     this.game = game
   }
@@ -109,13 +130,135 @@ export default class GameStat {
       .reduce((acc, curr) => acc += curr.value, 0)
   }
 
+  async buildMetricsWhereConditions(startDate?: string, endDate?: string, player?: Player): Promise<string> {
+    let whereConditions = `WHERE game_stat_id = ${this.id}`
+
+    if (startDate) {
+      whereConditions += ` AND created_at >= '${formatDateForClickHouse(new Date(startDate))}'`
+    }
+    if (endDate) {
+      // when using YYYY-MM-DD, use the end of the day
+      const end = endDate.length === 10 ? endOfDay(new Date(endDate)) : new Date(endDate)
+      whereConditions += ` AND created_at <= '${formatDateForClickHouse(end)}'`
+    }
+    if (player) {
+      await player.aliases.loadItems()
+      const aliasIds = player.aliases.getIdentifiers()
+      whereConditions += ` AND player_alias_id IN (${aliasIds.join(', ')})`
+    }
+
+    return whereConditions
+  }
+
+  async loadMetrics(clickhouse: ClickHouseClient, metricsStartDate?: string, metricsEndDate?: string): Promise<void> {
+    const whereConditions = await this.buildMetricsWhereConditions(metricsStartDate, metricsEndDate)
+
+    const [globalCount, globalValue] = await this.getGlobalValueMetrics(clickhouse, whereConditions)
+    const playerValue = await this.getPlayerValueMetrics(clickhouse, whereConditions)
+
+    this.metrics = {
+      globalCount,
+      globalValue,
+      playerValue
+    }
+  }
+
+  async getGlobalValueMetrics(
+    clickhouse: ClickHouseClient,
+    whereConditions: string
+  ): Promise<[number, GlobalValueMetrics]> {
+    const query = `
+      SELECT
+        count() as rawCount,
+        min(global_value) as minValue,
+        max(global_value) as maxValue,
+        median(global_value) as medianValue,
+        avg(global_value) as averageValue,
+        avg(change) as averageChange
+      FROM player_game_stat_snapshots
+      ${whereConditions}
+    `
+
+    const res = await clickhouse.query({
+      query: query,
+      format: 'JSONEachRow'
+    }).then((res) => res.json<{
+      rawCount: string | number
+      minValue: number
+      maxValue: number
+      medianValue: number | null
+      averageValue: number | null
+      averageChange: number | null
+    }>())
+
+    const {
+      rawCount,
+      minValue,
+      maxValue,
+      medianValue,
+      averageValue,
+      averageChange
+    } = res[0]
+
+    return [
+      Number(rawCount),
+      {
+        minValue: minValue || this.defaultValue,
+        maxValue: maxValue || this.defaultValue,
+        medianValue: medianValue ?? this.defaultValue,
+        averageValue: averageValue ?? this.defaultValue,
+        averageChange: averageChange ?? 0
+      }
+    ]
+  }
+
+  async getPlayerValueMetrics(
+    clickhouse: ClickHouseClient,
+    whereConditions: string
+  ): Promise<PlayerValueMetrics> {
+    const query = `
+      SELECT
+        min(value) as minValue,
+        max(value) as maxValue,
+        median(value) as medianValue,
+        avg(value) as averageValue
+      FROM player_game_stat_snapshots
+      ${whereConditions}
+    `
+
+    const res = await clickhouse.query({
+      query: query,
+      format: 'JSONEachRow'
+    }).then((res) => res.json<{
+      minValue: number
+      maxValue: number
+      medianValue: number | null
+      averageValue: number | null
+    }>())
+
+    const {
+      minValue,
+      maxValue,
+      medianValue,
+      averageValue
+    } = res[0]
+
+    return {
+      minValue: minValue || this.defaultValue,
+      maxValue: maxValue || this.defaultValue,
+      medianValue: medianValue ?? this.defaultValue,
+      averageValue: averageValue ?? this.defaultValue
+    }
+  }
+
   toJSON() {
     return {
       id: this.id,
       internalName: this.internalName,
       name: this.name,
       global: this.global,
       globalValue: this.hydratedGlobalValue ?? this.globalValue,
+      metrics: this.metrics,
       defaultValue: this.defaultValue,
       maxChange: this.maxChange,
       minValue: this.minValue,

src/index.ts

@@ -1,5 +1,6 @@
 import 'dotenv/config'
 import './lib/tracing/sentry-instrument'
+import './lib/tracing/enable-tracing'
 import Koa from 'koa'
 import loggerMiddleware from './middleware/logger-middleware'
 import bodyParser from 'koa-bodyparser'
@@ -15,10 +16,8 @@ import requestContextMiddleware from './middleware/request-context-middleware'
 import helmetMiddleware from './middleware/helmet-middleware'
 import { createServer } from 'http'
 import Socket from './socket'
-import { enableTracing } from './lib/tracing/enable-tracing'
 
 const isTest = process.env.NODE_ENV === 'test'
-enableTracing(isTest)
 
 export default async function init(): Promise<Koa> {
   const app = new Koa()

src/lib/tracing/enable-tracing.ts

@@ -1,14 +1,11 @@
 import { init as initHyperDX } from '@hyperdx/node-opentelemetry'
 
-export function enableTracing(isTest: boolean) {
-  if (isTest || typeof process.env.HYPERDX_API_KEY !== 'string') {
-    return
-  }
-
+if (process.env.NODE_ENV !== 'test' && typeof process.env.HYPERDX_API_KEY === 'string') {
   initHyperDX({
     service: 'talo',
     instrumentations: {
       '@opentelemetry/instrumentation-http': {
+        // todo: advanced network capture doesn't work because this overrides the config
         ignoreOutgoingRequestHook: (req) => req.hostname === process.env.CLICKHOUSE_HOST
       }
     }

src/lib/tracing/sentry-instrument.ts

@@ -1,8 +1,10 @@
 import { init as initSentry } from '@sentry/node'
 
-initSentry({
-  dsn: process.env.SENTRY_DSN,
-  environment: process.env.SENTRY_ENV,
-  maxValueLength: 4096,
-  skipOpenTelemetrySetup: true
-})
+if (process.env.NODE_ENV !== 'test') {
+  initSentry({
+    dsn: process.env.SENTRY_DSN,
+    environment: process.env.SENTRY_ENV,
+    maxValueLength: 4096,
+    skipOpenTelemetrySetup: true
+  })
+}

src/middleware/error-middleware.ts

@@ -22,8 +22,7 @@ export default async function errorMiddleware(ctx: Context, next: Next) {
 
         Sentry.withScope((scope) => {
           scope.addEventProcessor((event) => {
-            const headers = Object.entries(ctx.request.headers).reduce((acc, curr) => {
-              const [key, value] = curr
+            const headers = Object.entries(ctx.request.headers).reduce((acc, [key, value]) => {
               if (typeof value === 'string') {
                 acc[key] = value
               }

src/middleware/logger-middleware.ts

@@ -1,12 +1,24 @@
 import { setTraceAttributes } from '@hyperdx/node-opentelemetry'
+import { IncomingHttpHeaders, OutgoingHttpHeaders } from 'http'
 import { Context, Next } from 'koa'
 
+// todo: unneeded when advanced network capture works
+function buildHeaders(prefix: 'req' | 'res', headers: IncomingHttpHeaders | OutgoingHttpHeaders) {
+  return Object.entries(headers).reduce((acc, [key, value]) => {
+    return {
+      ...acc,
+      [`http.headers.${prefix}.${key.toLowerCase()}`]: value
+    }
+  }, {})
+}
+
 export default async function loggerMiddleware(ctx: Context, next: Next) {
   const startTime = Date.now()
 
   setTraceAttributes({
     'http.method': ctx.method,
-    'http.route': ctx.path
+    'http.route': ctx.path,
+    ...buildHeaders('req', ctx.request.headers)
   })
   console.info(`--> ${ctx.method} ${ctx.path}`)
 
@@ -20,6 +32,7 @@ export default async function loggerMiddleware(ctx: Context, next: Next) {
       'http.route': ctx.path,
       'http.status': status,
       'http.time_taken_ms': timeMs,
+      ...buildHeaders('res', ctx.response.headers),
       'http.response_size': ctx.response.length,
       'clay.matched_route': ctx.state.matchedRoute,
       'clay.matched_key': ctx.state.matchedServiceKey,