✨ Add stats on conversations (#828)

* ✨ Add stats on conversations

* 🩹 Fix conversation stats compute

* ⚡️ Slightly better index

* 🔊

* 🩹 Some fixes

* 🐛 Fix aggregation query

* 🐛 Fix $merge stage of query

* 🐛 Fix query on sessionId

* ✨ Compute weekly/monthly unique users

* 🩹 Final tweaks

* ⚡️ Split aggregations for better perf

No need to recompute DAUs / WAUs and so on

* ♻️ Deprecate PARQUET_EXPORT_SECRET & fix sec vuln

* Add ADMIN_API_SECRET to CD action

---------

Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>

Author: coyotte508

.env

@@ -109,7 +109,9 @@ PUBLIC_ANNOUNCEMENT_BANNERS=`[
 
 PARQUET_EXPORT_DATASET=
 PARQUET_EXPORT_HF_TOKEN=
-PARQUET_EXPORT_SECRET=
+ADMIN_API_SECRET=# secret to admin API calls, like computing usage stats or exporting parquet data
+
+PARQUET_EXPORT_SECRET=#DEPRECATED, use ADMIN_API_SECRET instead
 
 RATE_LIMIT= # requests per minute
 MESSAGES_BEFORE_LOGIN=# how many messages a user can send in a conversation before having to login. set to 0 to force login right away

.github/workflows/deploy-release.yml

@@ -26,6 +26,7 @@ jobs:
           MONGODB_URL: ${{ secrets.MONGODB_URL }}
           HF_DEPLOYMENT_TOKEN: ${{ secrets.HF_DEPLOYMENT_TOKEN }}
           WEBHOOK_URL_REPORT_ASSISTANT: ${{ secrets.WEBHOOK_URL_REPORT_ASSISTANT }}
+          ADMIN_API_SECRET: ${{ secrets.ADMIN_API_SECRET }}
         run: npm run updateProdEnv
   sync-to-hub:
     runs-on: ubuntu-latest

package-lock.json

@@ -7,6 +7,7 @@ const OPENID_CONFIG = process.env.OPENID_CONFIG;
 const MONGODB_URL = process.env.MONGODB_URL;
 const HF_TOKEN = process.env.HF_TOKEN ?? process.env.HF_ACCESS_TOKEN; // token used for API requests in prod
 const WEBHOOK_URL_REPORT_ASSISTANT = process.env.WEBHOOK_URL_REPORT_ASSISTANT; // slack webhook url used to get "report assistant" events
+const ADMIN_API_SECRET = process.env.ADMIN_API_SECRET;
 
 // Read the content of the file .env.template
 const PUBLIC_CONFIG = fs.readFileSync(".env.template", "utf8");
@@ -18,6 +19,7 @@ OPENID_CONFIG=${OPENID_CONFIG}
 SERPER_API_KEY=${SERPER_API_KEY}
 HF_TOKEN=${HF_TOKEN}
 WEBHOOK_URL_REPORT_ASSISTANT=${WEBHOOK_URL_REPORT_ASSISTANT}
+ADMIN_API_SECRET=${ADMIN_API_SECRET}
 `;
 
 // Make an HTTP POST request to add the space secrets

scripts/updateProdEnv.ts

@@ -1,4 +1,10 @@
-import { COOKIE_NAME, EXPOSE_API, MESSAGES_BEFORE_LOGIN } from "$env/static/private";
+import {
+	ADMIN_API_SECRET,
+	COOKIE_NAME,
+	EXPOSE_API,
+	MESSAGES_BEFORE_LOGIN,
+	PARQUET_EXPORT_SECRET,
+} from "$env/static/private";
 import type { Handle } from "@sveltejs/kit";
 import {
 	PUBLIC_GOOGLE_ANALYTICS_ID,
@@ -29,6 +35,18 @@ export const handle: Handle = async ({ event, resolve }) => {
 		});
 	}
 
+	if (event.url.pathname.startsWith(`${base}/admin/`) || event.url.pathname === `${base}/admin`) {
+		const ADMIN_SECRET = ADMIN_API_SECRET || PARQUET_EXPORT_SECRET;
+
+		if (!ADMIN_SECRET) {
+			return errorResponse(500, "Admin API is not configured");
+		}
+
+		if (event.request.headers.get("Authorization") !== `Bearer ${ADMIN_SECRET}`) {
+			return errorResponse(401, "Unauthorized");
+		}
+	}
+
 	const token = event.cookies.get(COOKIE_NAME);
 
 	let secretSessionId: string;

src/hooks.server.ts

@@ -9,6 +9,7 @@ import type { MessageEvent } from "$lib/types/MessageEvent";
 import type { Session } from "$lib/types/Session";
 import type { Assistant } from "$lib/types/Assistant";
 import type { Report } from "$lib/types/Report";
+import type { ConversationStats } from "$lib/types/ConversationStats";
 
 if (!MONGODB_URL) {
 	throw new Error(
@@ -24,7 +25,10 @@ export const connectPromise = client.connect().catch(console.error);
 
 const db = client.db(MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : ""));
 
+export const CONVERSATION_STATS_COLLECTION = "conversations.stats";
+
 const conversations = db.collection<Conversation>("conversations");
+const conversationStats = db.collection<ConversationStats>(CONVERSATION_STATS_COLLECTION);
 const assistants = db.collection<Assistant>("assistants");
 const reports = db.collection<Report>("reports");
 const sharedConversations = db.collection<SharedConversation>("sharedConversations");
@@ -38,6 +42,7 @@ const bucket = new GridFSBucket(db, { bucketName: "files" });
 export { client, db };
 export const collections = {
 	conversations,
+	conversationStats,
 	assistants,
 	reports,
 	sharedConversations,
@@ -68,6 +73,33 @@ client.on("open", () => {
 			{ partialFilterExpression: { userId: { $exists: true } } }
 		)
 		.catch(console.error);
+	// To do stats on conversations
+	conversations.createIndex({ updatedAt: 1 }).catch(console.error);
+	// Not strictly necessary, could use _id, but more convenient. Also for stats
+	conversations.createIndex({ createdAt: 1 }).catch(console.error);
+	// To do stats on conversation messages
+	conversations.createIndex({ "messages.createdAt": 1 }, { sparse: true }).catch(console.error);
+	// Unique index for stats
+	conversationStats
+		.createIndex(
+			{
+				type: 1,
+				"date.field": 1,
+				"date.span": 1,
+				"date.at": 1,
+				distinct: 1,
+			},
+			{ unique: true }
+		)
+		.catch(console.error);
+	// Allow easy check of last computed stat for given type/dateField
+	conversationStats
+		.createIndex({
+			type: 1,
+			"date.field": 1,
+			"date.at": 1,
+		})
+		.catch(console.error);
 	abortedGenerations.createIndex({ updatedAt: 1 }, { expireAfterSeconds: 30 }).catch(console.error);
 	abortedGenerations.createIndex({ conversationId: 1 }, { unique: true }).catch(console.error);
 	sharedConversations.createIndex({ hash: 1 }, { unique: true }).catch(console.error);

src/lib/server/database.ts

@@ -0,0 +1,13 @@
+import type { Timestamps } from "./Timestamps";
+
+export interface ConversationStats extends Timestamps {
+	date: {
+		at: Date;
+		span: "day" | "week" | "month";
+		field: "updatedAt" | "createdAt";
+	};
+	type: "conversation" | "message";
+	/**  _id => number of conversations/messages in the month */
+	distinct: "sessionId" | "userId" | "userOrSessionId" | "_id";
+	count: number;
+}

src/lib/types/ConversationStats.ts

@@ -1,8 +1,4 @@
-import {
-	PARQUET_EXPORT_DATASET,
-	PARQUET_EXPORT_HF_TOKEN,
-	PARQUET_EXPORT_SECRET,
-} from "$env/static/private";
+import { PARQUET_EXPORT_DATASET, PARQUET_EXPORT_HF_TOKEN } from "$env/static/private";
 import { collections } from "$lib/server/database";
 import type { Message } from "$lib/types/Message";
 import { error } from "@sveltejs/kit";
@@ -13,17 +9,13 @@ import parquet from "parquetjs";
 import { z } from "zod";
 
 // Triger like this:
-// curl -X POST "http://localhost:5173/chat/admin/export" -H "Authorization: Bearer <PARQUET_EXPORT_SECRET>" -H "Content-Type: application/json" -d '{"model": "OpenAssistant/oasst-sft-6-llama-30b-xor"}'
+// curl -X POST "http://localhost:5173/chat/admin/export" -H "Authorization: Bearer <ADMIN_API_SECRET>" -H "Content-Type: application/json" -d '{"model": "OpenAssistant/oasst-sft-6-llama-30b-xor"}'
 
 export async function POST({ request }) {
-	if (!PARQUET_EXPORT_SECRET || !PARQUET_EXPORT_DATASET || !PARQUET_EXPORT_HF_TOKEN) {
+	if (!PARQUET_EXPORT_DATASET || !PARQUET_EXPORT_HF_TOKEN) {
 		throw error(500, "Parquet export is not configured.");
 	}
 
-	if (request.headers.get("Authorization") !== `Bearer ${PARQUET_EXPORT_SECRET}`) {
-		throw error(403);
-	}
-
 	const { model } = z
 		.object({
 			model: z.string(),