feat: add named exercise groups to web-evals

apps/web-evals/src/actions/exercise-groups.ts

@@ -0,0 +1,12 @@
+"use server"
+
+import exerciseGroups from "@/lib/exercise-groups.json"
+
+export interface ExerciseGroup {
+	name: string
+	exercises: string[]
+}
+
+export const getExerciseGroups = async (): Promise<ExerciseGroup[]> => {
+	return exerciseGroups.groups
+}

apps/web-evals/src/app/runs/new/new-run.tsx

@@ -8,12 +8,13 @@ import { useForm, FormProvider } from "react-hook-form"
 import { zodResolver } from "@hookform/resolvers/zod"
 import fuzzysort from "fuzzysort"
 import { toast } from "sonner"
-import { X, Rocket, Check, ChevronsUpDown, SlidersHorizontal, Book, CircleCheck } from "lucide-react"
+import { X, Rocket, Check, ChevronsUpDown, SlidersHorizontal, Book, CircleCheck, Users } from "lucide-react"
 
 import { globalSettingsSchema, providerSettingsSchema, EVALS_SETTINGS, getModelId } from "@roo-code/types"
 
 import { createRun } from "@/actions/runs"
 import { getExercises } from "@/actions/exercises"
+import { getExerciseGroups } from "@/actions/exercise-groups"
 import {
 	createRunSchema,
 	type CreateRun,
@@ -70,6 +71,7 @@ export function NewRun() {
 
 	const models = useOpenRouterModels()
 	const exercises = useQuery({ queryKey: ["getExercises"], queryFn: () => getExercises() })
+	const exerciseGroups = useQuery({ queryKey: ["getExerciseGroups"], queryFn: () => getExerciseGroups() })
 
 	const form = useForm<CreateRun>({
 		resolver: zodResolver(createRunSchema),
@@ -178,6 +180,18 @@ export function NewRun() {
 		[clearErrors, setValue],
 	)
 
+	const onSelectExerciseGroup = useCallback(
+		(groupName: string) => {
+			const group = exerciseGroups.data?.find((g) => g.name === groupName)
+			if (group) {
+				setValue("suite", "partial")
+				setValue("exercises", group.exercises)
+				toast.success(`Selected "${groupName}" exercise group with ${group.exercises.length} exercises`)
+			}
+		},
+		[exerciseGroups.data, setValue],
+	)
+
 	return (
 		<>
 			<FormProvider {...form}>
@@ -309,13 +323,37 @@ export function NewRun() {
 									</TabsList>
 								</Tabs>
 								{suite === "partial" && (
-									<MultiSelect
-										options={exercises.data?.map((path) => ({ value: path, label: path })) || []}
-										onValueChange={(value) => setValue("exercises", value)}
-										placeholder="Select"
-										variant="inverted"
-										maxCount={4}
-									/>
+									<>
+										{exerciseGroups.data && exerciseGroups.data.length > 0 && (
+											<div className="flex flex-wrap gap-2 mt-2 mb-2">
+												<div className="text-sm text-muted-foreground flex items-center gap-1">
+													<Users className="size-4" />
+													Predefined groups:
+												</div>
+												{exerciseGroups.data.map((group) => (
+													<Button
+														key={group.name}
+														type="button"
+														variant="outline"
+														size="sm"
+														onClick={() => onSelectExerciseGroup(group.name)}
+														className="text-xs">
+														{group.name} ({group.exercises.length})
+													</Button>
+												))}
+											</div>
+										)}
+										<MultiSelect
+											options={
+												exercises.data?.map((path) => ({ value: path, label: path })) || []
+											}
+											onValueChange={(value) => setValue("exercises", value)}
+											placeholder="Select"
+											variant="inverted"
+											maxCount={4}
+											defaultValue={form.watch("exercises") || []}
+										/>
+									</>
 								)}
 								<FormMessage />
 							</FormItem>

apps/web-evals/src/lib/exercise-groups.json

@@ -0,0 +1,35 @@
+{
+	"groups": [
+		{
+			"name": "Hard",
+			"exercises": [
+				"go/connect",
+				"go/robot-simulator",
+				"go/react",
+				"go/forth",
+				"go/bowling",
+				"go/book-store",
+				"go/matrix",
+				"java/transpose",
+				"java/hangman",
+				"java/change",
+				"java/bowling",
+				"javascript/lodash",
+				"javascript/zebra-puzzle",
+				"javascript/connect",
+				"javascript/food-chain",
+				"javascript/go-counting",
+				"javascript/scale-generator",
+				"javascript/transpose",
+				"python/affine-cipher",
+				"python/forth",
+				"python/transpose",
+				"python/wordy",
+				"rust/ocr-numbers",
+				"rust/forth",
+				"rust/doubly-linked-list",
+				"rust/xorcism"
+			]
+		}
+	]
+}