Add schema validation metric (#1033)

neoxelox · web-flow · commit be25629b40cf · 2025-03-26T12:31:12.000+01:00
* Add schema validation metric

* rebase package json
diff --git a/apps/web/src/components/evaluations/ConfigurationForm.tsx b/apps/web/src/components/evaluations/ConfigurationForm.tsx
@@ -43,7 +43,7 @@ export default function ConfigurationForm<
       <SelectableSwitch
         selected={!(configuration.reverseScale ?? false)}
         name='reverseScale'
-        label='Orientation'
+        label='Scale orientation'
         trueLabel='Higher is better'
         falseLabel='Lower is better'
         description='Orientation of the metric scale when normalizing the score for internal operations and to display evaluation results'
diff --git a/apps/web/src/components/evaluations/rule/RegularExpression.tsx b/apps/web/src/components/evaluations/rule/RegularExpression.tsx
@@ -40,7 +40,7 @@ function ConfigurationForm({
       <Input
         value={configuration.pattern ?? ''}
         name='pattern'
-        label='Regex Pattern'
+        label='Regex pattern'
         description='The regex pattern to match against'
         placeholder='.*pattern.*'
         onChange={(e) =>
diff --git a/apps/web/src/components/evaluations/rule/SchemaValidation.tsx b/apps/web/src/components/evaluations/rule/SchemaValidation.tsx
@@ -0,0 +1,140 @@
+import {
+  EvaluationType,
+  RuleEvaluationMetric,
+  RuleEvaluationSchemaValidationSpecification,
+} from '@latitude-data/constants'
+import { IconName, Select, TextArea } from '@latitude-data/web-ui'
+import {
+  ChartConfigurationArgs,
+  ConfigurationFormProps,
+  ResultBadgeProps,
+  ResultPanelProps,
+  ResultRowCellsProps,
+  ResultRowHeadersProps,
+} from '../index'
+
+const specification = RuleEvaluationSchemaValidationSpecification
+export default {
+  ...specification,
+  icon: 'clipboardCheck' as IconName,
+  ConfigurationForm: ConfigurationForm,
+  ResultBadge: ResultBadge,
+  ResultRowHeaders: ResultRowHeaders,
+  ResultRowCells: ResultRowCells,
+  resultPanelTabs: [],
+  ResultPanelMetadata: ResultPanelMetadata,
+  ResultPanelContent: ResultPanelContent,
+  chartConfiguration: chartConfiguration,
+}
+
+const FORMAT_OPTIONS = specification.configuration.shape.format.options.map(
+  (option) => ({
+    label: option.toUpperCase().split('_').join(' '),
+    value: option,
+  }),
+)
+
+function ConfigurationForm({
+  configuration,
+  setConfiguration,
+  disabled,
+}: ConfigurationFormProps<
+  EvaluationType.Rule,
+  RuleEvaluationMetric.SchemaValidation
+>) {
+  return (
+    <>
+      <Select
+        value={configuration.format ?? ''}
+        name='format'
+        label='Schema format'
+        description='The format of the schema'
+        placeholder='Select a schema format'
+        options={FORMAT_OPTIONS}
+        onChange={(value) =>
+          setConfiguration({ ...configuration, format: value })
+        }
+        disabled={disabled}
+        required
+      />
+      <TextArea
+        value={configuration.schema ?? ''}
+        name='schema'
+        label={
+          configuration.format
+            ? `${configuration.format.toUpperCase().split('_').join(' ')} schema`
+            : 'Schema'
+        }
+        description='The schema to validate against'
+        placeholder='{ "type": "object" }'
+        onChange={(e) =>
+          setConfiguration({ ...configuration, schema: e.target.value })
+        }
+        minRows={3}
+        disabled={disabled}
+        required
+      />
+    </>
+  )
+}
+
+function ResultBadge({
+  result,
+}: ResultBadgeProps<
+  EvaluationType.Rule,
+  RuleEvaluationMetric.SchemaValidation
+>) {
+  return <>{result.score === 1 ? 'Valid' : 'Invalid'}</>
+}
+
+function ResultRowHeaders(
+  _props: ResultRowHeadersProps<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SchemaValidation
+  >,
+) {
+  return <></>
+}
+
+function ResultRowCells(
+  _props: ResultRowCellsProps<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SchemaValidation
+  >,
+) {
+  return <></>
+}
+
+function ResultPanelMetadata(
+  _props: ResultPanelProps<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SchemaValidation
+  >,
+) {
+  return <></>
+}
+
+function ResultPanelContent(
+  _props: ResultPanelProps<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SchemaValidation
+  >,
+) {
+  return <></>
+}
+
+function chartConfiguration(
+  _args: ChartConfigurationArgs<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SchemaValidation
+  >,
+) {
+  return {
+    min: 0,
+    max: 100,
+    thresholds: [50] as const,
+    scale: (point: number) => Math.min(Math.max(point * 100, 0), 100),
+    format: (point: number, short?: boolean) =>
+      short ? `${point.toFixed(0)}%` : `${point.toFixed(0)}% valid`,
+  }
+}
diff --git a/apps/web/src/components/evaluations/rule/index.tsx b/apps/web/src/components/evaluations/rule/index.tsx
@@ -15,13 +15,15 @@ import {
 } from '../index'
 import RuleEvaluationExactMatchSpecification from './ExactMatch'
 import RuleEvaluationRegularExpressionSpecification from './RegularExpression'
+import RuleEvaluationSchemaValidationSpecification from './SchemaValidation'
 
 // prettier-ignore
 const METRICS: {
   [M in RuleEvaluationMetric]: EvaluationMetricFrontendSpecification<EvaluationType.Rule, M>
 } = {
   [RuleEvaluationMetric.ExactMatch]: RuleEvaluationExactMatchSpecification,
   [RuleEvaluationMetric.RegularExpression]: RuleEvaluationRegularExpressionSpecification,
+  [RuleEvaluationMetric.SchemaValidation]: RuleEvaluationSchemaValidationSpecification,
   [RuleEvaluationMetric.LengthCount]:  undefined as any, // TODO: Implement
   [RuleEvaluationMetric.LexicalOverlap]:  undefined as any, // TODO: Implement
   [RuleEvaluationMetric.SemanticSimilarity]:  undefined as any, // TODO: Implement
diff --git a/packages/constants/src/evaluations/rule.ts b/packages/constants/src/evaluations/rule.ts
@@ -73,6 +73,39 @@ export type RuleEvaluationRegularExpressionResultError = z.infer<
   typeof RuleEvaluationRegularExpressionSpecification.resultError
 >
 
+// SCHEMA VALIDATION
+
+const ruleEvaluationSchemaValidationConfiguration =
+  ruleEvaluationConfiguration.extend({
+    format: z.enum(['json']),
+    schema: z.string(),
+  })
+const ruleEvaluationSchemaValidationResultMetadata =
+  ruleEvaluationResultMetadata.extend({
+    configuration: ruleEvaluationSchemaValidationConfiguration,
+  })
+const ruleEvaluationSchemaValidationResultError =
+  ruleEvaluationResultError.extend({})
+export const RuleEvaluationSchemaValidationSpecification = {
+  name: 'Schema Validation',
+  description: 'Checks if the response follows the schema',
+  configuration: ruleEvaluationSchemaValidationConfiguration,
+  resultMetadata: ruleEvaluationSchemaValidationResultMetadata,
+  resultError: ruleEvaluationSchemaValidationResultError,
+  requiresExpectedOutput: false,
+  supportsLiveEvaluation: true,
+  supportsBatchEvaluation: true,
+}
+export type RuleEvaluationSchemaValidationConfiguration = z.infer<
+  typeof RuleEvaluationSchemaValidationSpecification.configuration
+>
+export type RuleEvaluationSchemaValidationResultMetadata = z.infer<
+  typeof RuleEvaluationSchemaValidationSpecification.resultMetadata
+>
+export type RuleEvaluationSchemaValidationResultError = z.infer<
+  typeof RuleEvaluationSchemaValidationSpecification.resultError
+>
+
 // LENGTH COUNT
 
 const ruleEvaluationLengthCountConfiguration =
@@ -188,6 +221,7 @@ export type RuleEvaluationSemanticSimilarityResultError = z.infer<
 export enum RuleEvaluationMetric {
   ExactMatch = 'exact_match',
   RegularExpression = 'regular_expression',
+  SchemaValidation = 'schema_validation',
   LengthCount = 'length_count',
   LexicalOverlap = 'lexical_overlap',
   SemanticSimilarity = 'semantic_similarity',
@@ -197,6 +231,7 @@ export enum RuleEvaluationMetric {
 export type RuleEvaluationConfiguration<M extends RuleEvaluationMetric = RuleEvaluationMetric> = 
   M extends RuleEvaluationMetric.ExactMatch ? RuleEvaluationExactMatchConfiguration :
   M extends RuleEvaluationMetric.RegularExpression ? RuleEvaluationRegularExpressionConfiguration :
+  M extends RuleEvaluationMetric.SchemaValidation ? RuleEvaluationSchemaValidationConfiguration :
   M extends RuleEvaluationMetric.LengthCount ? RuleEvaluationLengthCountConfiguration :
   M extends RuleEvaluationMetric.LexicalOverlap ? RuleEvaluationLexicalOverlapConfiguration :
   M extends RuleEvaluationMetric.SemanticSimilarity ? RuleEvaluationSemanticSimilarityConfiguration :
@@ -206,6 +241,7 @@ export type RuleEvaluationConfiguration<M extends RuleEvaluationMetric = RuleEva
 export type RuleEvaluationResultMetadata<M extends RuleEvaluationMetric = RuleEvaluationMetric> = 
   M extends RuleEvaluationMetric.ExactMatch ? RuleEvaluationExactMatchResultMetadata :
   M extends RuleEvaluationMetric.RegularExpression ? RuleEvaluationRegularExpressionResultMetadata :
+  M extends RuleEvaluationMetric.SchemaValidation ? RuleEvaluationSchemaValidationResultMetadata :
   M extends RuleEvaluationMetric.LengthCount ? RuleEvaluationLengthCountResultMetadata :
   M extends RuleEvaluationMetric.LexicalOverlap ? RuleEvaluationLexicalOverlapResultMetadata :
   M extends RuleEvaluationMetric.SemanticSimilarity ? RuleEvaluationSemanticSimilarityResultMetadata :
@@ -215,6 +251,7 @@ export type RuleEvaluationResultMetadata<M extends RuleEvaluationMetric = RuleEv
 export type RuleEvaluationResultError<M extends RuleEvaluationMetric = RuleEvaluationMetric> = 
   M extends RuleEvaluationMetric.ExactMatch ? RuleEvaluationExactMatchResultError :
   M extends RuleEvaluationMetric.RegularExpression ? RuleEvaluationRegularExpressionResultError :
+  M extends RuleEvaluationMetric.SchemaValidation ? RuleEvaluationSchemaValidationResultError :
   M extends RuleEvaluationMetric.LengthCount ? RuleEvaluationLengthCountResultError :
   M extends RuleEvaluationMetric.LexicalOverlap ? RuleEvaluationLexicalOverlapResultError :
   M extends RuleEvaluationMetric.SemanticSimilarity ? RuleEvaluationSemanticSimilarityResultError :
@@ -230,6 +267,7 @@ export const RuleEvaluationSpecification = {
   metrics: {
     [RuleEvaluationMetric.ExactMatch]: RuleEvaluationExactMatchSpecification,
     [RuleEvaluationMetric.RegularExpression]: RuleEvaluationRegularExpressionSpecification,
+    [RuleEvaluationMetric.SchemaValidation]: RuleEvaluationSchemaValidationSpecification,
     [RuleEvaluationMetric.LengthCount]: RuleEvaluationLengthCountSpecification,
     [RuleEvaluationMetric.LexicalOverlap]: RuleEvaluationLexicalOverlapSpecification,
     [RuleEvaluationMetric.SemanticSimilarity]: RuleEvaluationSemanticSimilaritySpecification,
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -161,6 +161,7 @@
     "@codesandbox/sdk": "^0.6.2",
     "@modelcontextprotocol/sdk": "^1.6.0",
     "@tavily/core": "^0.3.1",
+    "ajv": "^8.17.1",
     "date-fns": "^3.6.0",
     "diff-match-patch": "^1.0.5",
     "js-yaml": "^4.1.0",
diff --git a/packages/core/src/services/evaluationsV2/rule/index.ts b/packages/core/src/services/evaluationsV2/rule/index.ts
@@ -13,13 +13,15 @@ import {
 } from '../shared'
 import RuleEvaluationExactMatchSpecification from './exactMatch'
 import RuleEvaluationRegularExpressionSpecification from './regularExpression'
+import RuleEvaluationSchemaValidationSpecification from './schemaValidation'
 
 // prettier-ignore
 const METRICS: {
   [M in RuleEvaluationMetric]: EvaluationMetricBackendSpecification<EvaluationType.Rule, M>
 } = {
   [RuleEvaluationMetric.ExactMatch]: RuleEvaluationExactMatchSpecification,
   [RuleEvaluationMetric.RegularExpression]: RuleEvaluationRegularExpressionSpecification,
+  [RuleEvaluationMetric.SchemaValidation]: RuleEvaluationSchemaValidationSpecification,
   [RuleEvaluationMetric.LengthCount]:  undefined as any, // TODO: Implement
   [RuleEvaluationMetric.LexicalOverlap]:  undefined as any, // TODO: Implement
   [RuleEvaluationMetric.SemanticSimilarity]:  undefined as any, // TODO: Implement
diff --git a/packages/core/src/services/evaluationsV2/rule/schemaValidation.ts b/packages/core/src/services/evaluationsV2/rule/schemaValidation.ts
diff --git a/packages/web-ui/src/ds/atoms/Icons/index.tsx b/packages/web-ui/src/ds/atoms/Icons/index.tsx
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
