Added score normalization and list of scored apps (#692)

* Created empty route with function structure

* Added query for retrieving all hacker apps

* Filtered overqualified out of retrieve

* Implemented function to get mean and std

* Implemented normalizing function

* Removed unnecessary check

* returned new dict for normalized scores

* Created bulk write function and tests

* Implemented bulk update for normalized scores

* Added skeleton for normalization page

* Added normalized_scores to summary fetching

* Added normalized-scores to list of hackers

* Added email and resume to returned list

* Created table and csv download

* Filtered out overqualified

* Added normalize scores button

* Reordered imports

* Removed test json

Author: IanWearsHat

apps/api/src/admin/score_normalizing_handler.py

@@ -0,0 +1,114 @@
+from collections import defaultdict
+from statistics import mean, pstdev
+from typing import Any
+
+from pymongo import UpdateOne
+
+from models.user_record import Role
+from services import mongodb_handler
+from services.mongodb_handler import Collection
+
+GLOBAL_FIELDS = {"resume", "hackathon_experience"}
+
+
+async def add_normalized_scores_to_all_hacker_applicants() -> None:
+    """Calculates normalized scores and adds them to all hacker apps"""
+    all_apps = await get_all_hacker_apps()
+    reviewer_stats = get_reviewer_stats(all_apps)
+
+    normalized_scores = get_normalized_scores_for_hacker_applicants(
+        all_apps, reviewer_stats
+    )
+    await update_hacker_applicants_in_collection(normalized_scores)
+
+
+async def get_all_hacker_apps() -> list[dict[str, object]]:
+    return await mongodb_handler.retrieve(
+        Collection.USERS,
+        {
+            "roles": Role.HACKER,
+            "application_data.global_field_scores.resume": {"$gte": 0},
+            "application_data.global_field_scores.hackathon_experience": {"$gte": 0},
+        },
+        [
+            "_id",
+            "status",
+            "application_data.review_breakdown",
+            "application_data.global_field_scores",
+        ],
+    )
+
+
+def get_reviewer_stats(all_apps: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
+    """Compute mean and std for each reviewer across all applications."""
+    reviewer_totals: dict[str, list[float]] = defaultdict(list)
+
+    for app in all_apps:
+        breakdown = app.get("application_data", {}).get("review_breakdown", {})
+        for reviewer, scores_dict in breakdown.items():
+            total_score = sum(
+                [
+                    score
+                    for field, score in scores_dict.items()
+                    if field not in GLOBAL_FIELDS
+                ]
+            )
+            reviewer_totals[reviewer].append(total_score)
+
+    reviewer_stats = {
+        reviewer: {
+            "mean": mean(scores),
+            "std": pstdev(scores) or 1.0,  # avoid divide-by-zero if all same
+        }
+        for reviewer, scores in reviewer_totals.items()
+    }
+
+    return reviewer_stats
+
+
+def get_normalized_scores_for_hacker_applicants(
+    all_apps: list[dict[str, Any]], reviewer_stats: dict[str, dict[str, float]]
+) -> dict[str, dict[str, float]]:
+    """
+    Compute normalized scores for each applicant and return a dict in the format:
+    {
+        "app1": {"ian": 0.5, "bob": -0.3},
+        "app2": {"ian": 1.2}
+    }
+
+    - all_apps: list of applicant dicts
+    - reviewer_stats: dict of reviewer mean/std
+    """
+    result: dict[str, dict[str, float]] = {}
+
+    for app in all_apps:
+        app_id = app["_id"]
+        breakdown = app.get("application_data", {}).get("review_breakdown", {})
+        normalized_scores: dict[str, float] = {}
+
+        for reviewer, scores_dict in breakdown.items():
+            total_score = sum(
+                score
+                for field, score in scores_dict.items()
+                if field not in GLOBAL_FIELDS  # exclude global fields if needed
+            )
+            stats = reviewer_stats.get(reviewer, {"mean": 0, "std": 1})
+            normalized = (total_score - stats["mean"]) / stats["std"]
+            normalized_scores[reviewer] = normalized
+
+        result[app_id] = normalized_scores
+
+    return result
+
+
+async def update_hacker_applicants_in_collection(
+    normalized_scores: dict[str, dict[str, float]]
+) -> None:
+    operations = [
+        UpdateOne(
+            {"_id": app_id}, {"$set": {"application_data.normalized_scores": scores}}
+        )
+        for app_id, scores in normalized_scores.items()
+    ]
+
+    await mongodb_handler.bulk_update(Collection.USERS, operations)

apps/api/src/routers/admin.py

@@ -9,6 +9,9 @@
 
 from admin import applicant_review_processor, participant_manager, summary_handler
 from admin.participant_manager import Participant
+from admin.score_normalizing_handler import (
+    add_normalized_scores_to_all_hacker_applicants,
+)
 from auth.authorization import require_role
 from auth.user_identity import User, utc_now
 from models.ApplicationData import Decision, Review
@@ -56,6 +59,9 @@ class ApplicationDataSummary(BaseModel):
 class ZotHacksApplicationDataSummary(BaseModel):
     school_year: str
     submission_time: Any
+    normalized_scores: Optional[dict[str, float]] = None
+    email: str
+    resume_url: str
 
 
 class ApplicantSummary(BaseRecord):
@@ -182,7 +188,6 @@ async def hacker_applicants(
 
     try:
         return TypeAdapter(list[HackerApplicantSummary]).validate_python(records)
-
     except ValidationError:
         raise RuntimeError("Could not parse applicant data.")
 
@@ -440,6 +445,18 @@ async def subevent_checkin(
     await participant_manager.subevent_checkin(event, uid, organizer)
 
 
+@router.get(
+    "/normalize-detailed-scores",
+    dependencies=[Depends(require_role({Role.DIRECTOR, Role.LEAD}))],
+)
+async def normalize_detailed_scores_for_all_hacker_apps() -> None:
+    try:
+        await add_normalized_scores_to_all_hacker_applicants()
+    except RuntimeError:
+        log.error("Could not update/add normalized scores to hacker applicants")
+        raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
 async def retrieve_thresholds() -> Optional[dict[str, Any]]:
     return await mongodb_handler.retrieve_one(
         Collection.SETTINGS, {"_id": "hacker_score_thresholds"}, ["accept", "waitlist"]

apps/api/src/services/mongodb_handler.py

@@ -2,12 +2,13 @@
 import os
 from enum import Enum
 from logging import getLogger
-from typing import Any, Mapping, Optional, Union
+from typing import Any, Mapping, Optional, Sequence, Union
 
 from bson import CodecOptions
 from motor.core import AgnosticClient, AgnosticDatabase
 from motor.motor_asyncio import AsyncIOMotorClient
 from pydantic import BaseModel, ConfigDict, Field
+from pymongo import UpdateMany, UpdateOne
 
 from utils.hackathon_context import hackathon_name_ctx, HackathonName
 
@@ -153,3 +154,30 @@ async def update(
         raise RuntimeError("Could not update documents in MongoDB collection")
 
     return result.modified_count > 0
+
+
+async def bulk_update(
+    collection: Collection,
+    operations: Sequence[Union[UpdateOne, UpdateMany]],
+) -> bool:
+    """
+    Perform multiple updates in bulk on a collection.
+
+    operations should be a list of pymongo UpdateOne or UpdateMany objects.
+    Returns True if at least one document was modified.
+    """
+    if not operations:
+        log.warning("No operations provided to bulk_update")
+        return False
+
+    DB = get_database()
+    COLLECTION = DB[collection.value]
+
+    result = await COLLECTION.bulk_write(operations)
+
+    if not result.acknowledged:
+        log.error("MongoDB bulk write was not acknowledged")
+        raise RuntimeError("Could not perform bulk write in MongoDB collection")
+
+    log.info(f"Bulk write completed: {result.modified_count} documents modified")
+    return result.modified_count > 0

apps/api/tests/test_mongodb_handler.py

@@ -1,5 +1,6 @@
 from unittest.mock import AsyncMock, MagicMock, Mock, patch
 
+from pymongo import UpdateOne
 import pytest
 from pymongo.results import InsertOneResult, UpdateResult
 
@@ -218,3 +219,60 @@ async def test_retrieve_documents_sorted_descending(mock_DB: MagicMock) -> None:
     mock_collection.find.assert_called_once_with(query, [])
     mock_collection.find.return_value.sort.assert_called_once_with(sort)
     assert result == SAMPLE_DOCUMENTS
+
+
+@patch("services.mongodb_handler.get_database")
+async def test_bulk_update_success(mock_DB: MagicMock) -> None:
+    """Test that bulk_update returns True if at least one document modified"""
+    mock_collection = AsyncMock()
+    mock_result = MagicMock()
+    mock_result.acknowledged = True
+    mock_result.modified_count = 2
+    mock_collection.bulk_write.return_value = mock_result
+
+    mock_db_instance = MagicMock()
+    mock_db_instance.__getitem__.return_value = mock_collection
+    mock_DB.return_value = mock_db_instance
+
+    operations = [
+        UpdateOne({"_id": "app1"}, {"$set": {"score": 1}}),
+        UpdateOne({"_id": "app2"}, {"$set": {"score": 2}}),
+    ]
+
+    result = await mongodb_handler.bulk_update(Collection.TESTING, operations)
+
+    mock_collection.bulk_write.assert_awaited_once_with(operations)
+    assert result is True
+
+
+@patch("services.mongodb_handler.get_database")
+async def test_bulk_update_no_acknowledgement(mock_DB: MagicMock) -> None:
+    """Test that bulk_update raises RuntimeError if not acknowledged"""
+    mock_collection = AsyncMock()
+    mock_result = MagicMock()
+    mock_result.acknowledged = False
+    mock_result.modified_count = 0
+    mock_collection.bulk_write.return_value = mock_result
+
+    mock_db_instance = MagicMock()
+    mock_db_instance.__getitem__.return_value = mock_collection
+    mock_DB.return_value = mock_db_instance
+
+    operations = [UpdateOne({"_id": "app1"}, {"$set": {"score": 1}})]
+
+    with pytest.raises(RuntimeError):
+        await mongodb_handler.bulk_update(Collection.TESTING, operations)
+
+
+@patch("services.mongodb_handler.get_database")
+async def test_bulk_update_empty_operations(mock_DB: MagicMock) -> None:
+    """Test that bulk_update returns False if no operations are provided"""
+    mock_collection = AsyncMock()
+    mock_db_instance = MagicMock()
+    mock_db_instance.__getitem__.return_value = mock_collection
+    mock_DB.return_value = mock_db_instance
+
+    result = await mongodb_handler.bulk_update(Collection.TESTING, [])
+
+    assert result is False
+    mock_collection.bulk_write.assert_not_called()

apps/site/src/app/admin/layout/AdminSidebar.tsx

@@ -10,6 +10,7 @@ import {
 	isApplicationManager,
 	isHackerReviewer,
 	isDirector,
+	isLead,
 } from "@/lib/admin/authorization";
 
 import UserContext from "@/lib/admin/UserContext";
@@ -56,6 +57,14 @@ function AdminSidebar() {
 	// 	});
 	// }
 
+	if (isLead(roles) || isDirector(roles)) {
+		navigationItems.splice(1, 0, {
+			type: "link",
+			text: "Scores",
+			href: "/admin/scores",
+		});
+	}
+
 	if (isApplicationManager(roles)) {
 		navigationItems.splice(1, 0, {
 			type: "link-group",

apps/site/src/app/admin/layout/Breadcrumbs.tsx

@@ -21,6 +21,7 @@ const pathTitles: PathTitles = {
 	organizers: "Organizers",
 	"email-sender": "Email Sender",
 	"zothacks-hackers": "ZotHacks Hacker Applications",
+	scores: "Scores",
 };
 
 const DEFAULT_ITEMS = [{ text: "Admin Dashboard", href: BASE_PATH }];