add support for examples

Author: davidsbailey

API.md

@@ -18,6 +18,7 @@
 * `code`: The code to assess. Required.
 * `prompt`: The system prompt. Required.
 * `rubric`: The rubric, as a CSV. Required.
+* `examples`: Array of pairs of code (js) and openai response (tsv).
 * `remove-comments`: When `1`, attempts to strip comments out of the code before assessment. Default: 0
 * `num-responses`: The number of times it should ask the AI model. It votes on the final answer. Default: 1
 * `num-passing-grades`: The number of grades to consider 'passing'. Defaults: 2 (pass fail)

lib/assessment/assess.py

@@ -22,7 +22,7 @@
     compute_accuracy
 )
 
-def grade(code, prompt, rubric, api_key='', llm_model='gpt-4', num_responses=1, temperature=0.2, num_passing_grades=2, remove_comments=False):
+def grade(code, prompt, rubric, examples=[], api_key='', llm_model='gpt-4', num_responses=1, temperature=0.2, num_passing_grades=2, remove_comments=False):
   OPENAI_API_KEY = api_key
 
   # Set the key
@@ -36,7 +36,8 @@ def grade(code, prompt, rubric, api_key='', llm_model='gpt-4', num_responses=1,
 
   grade = Grade()
   return grade.grade_student_work(
-      prompt, rubric, code, "student", [],
+      prompt, rubric, code, "student",
+      examples=examples,
       use_cached=False,
       write_cached=False,
       num_responses=num_responses,

src/assessment.py

@@ -5,6 +5,7 @@
 
 import os
 import openai
+import json
 
 # Our assessment code
 from lib.assessment import assess
@@ -31,11 +32,14 @@ def post_assessment():
     if request.values.get("rubric", None) == None:
         return "`rubric` is required", 400
 
+    examples = json.loads(request.values.get("examples", "[]"))
+
     try:
         grades = assess.grade(
             code=request.values.get("code", ""),
             prompt=request.values.get("prompt", ""),
             rubric=request.values.get("rubric", ""),
+            examples=examples,
             api_key=request.values.get("api-key", openai.api_key),
             llm_model=request.values.get("model", "gpt-4"),
             remove_comments=(request.values.get("remove-comments", "0") != "0"),