Add semantic similarity metric (#1039)

neoxelox · web-flow · commit cdb173843795 · 2025-03-27T13:04:31.000+01:00
diff --git a/apps/web/src/components/evaluations/rule/SemanticSimilarity.tsx b/apps/web/src/components/evaluations/rule/SemanticSimilarity.tsx
@@ -0,0 +1,159 @@
+import {
+  EvaluationType,
+  RuleEvaluationMetric,
+  RuleEvaluationSemanticSimilaritySpecification,
+} from '@latitude-data/constants'
+import { IconName, NumberInput, Select } from '@latitude-data/web-ui'
+import {
+  ChartConfigurationArgs,
+  ConfigurationFormProps,
+  ResultBadgeProps,
+  ResultPanelProps,
+  ResultRowCellsProps,
+  ResultRowHeadersProps,
+} from '../index'
+
+const specification = RuleEvaluationSemanticSimilaritySpecification
+export default {
+  ...specification,
+  icon: 'equalApproximately' as IconName,
+  ConfigurationForm: ConfigurationForm,
+  ResultBadge: ResultBadge,
+  ResultRowHeaders: ResultRowHeaders,
+  ResultRowCells: ResultRowCells,
+  resultPanelTabs: [],
+  ResultPanelMetadata: ResultPanelMetadata,
+  ResultPanelContent: ResultPanelContent,
+  chartConfiguration: chartConfiguration,
+}
+
+const ALGORITHM_OPTIONS =
+  specification.configuration.shape.algorithm.options.map((option) => ({
+    label: option.toUpperCase().split('_').join(' '),
+    value: option,
+  }))
+
+function ConfigurationForm({
+  configuration,
+  setConfiguration,
+  disabled,
+}: ConfigurationFormProps<
+  EvaluationType.Rule,
+  RuleEvaluationMetric.SemanticSimilarity
+>) {
+  return (
+    <>
+      <Select
+        value={configuration.algorithm ?? ''}
+        name='algorithm'
+        label='Algorithm'
+        description='How to measure percentage of similarity'
+        placeholder='Select an algorithm'
+        options={ALGORITHM_OPTIONS}
+        onChange={(value) =>
+          setConfiguration({ ...configuration, algorithm: value })
+        }
+        disabled={disabled}
+        required
+      />
+      <NumberInput
+        value={configuration.minSimilarity ?? undefined}
+        name='minSimilarity'
+        label='Minimum similarity'
+        description='The minimum percentage of similarity of the response'
+        placeholder='No minimum'
+        min={0}
+        max={100}
+        onChange={(value) =>
+          setConfiguration({ ...configuration, minSimilarity: value })
+        }
+        className='w-full'
+        disabled={disabled}
+        required
+      />
+      <NumberInput
+        value={configuration.maxSimilarity ?? undefined}
+        name='maxSimilarity'
+        label='Maximum similarity'
+        description='The maximum percentage of similarity of the response'
+        placeholder='No maximum'
+        min={0}
+        max={100}
+        onChange={(value) =>
+          setConfiguration({ ...configuration, maxSimilarity: value })
+        }
+        className='w-full'
+        disabled={disabled}
+        required
+      />
+    </>
+  )
+}
+
+function ResultBadge({
+  result,
+}: ResultBadgeProps<
+  EvaluationType.Rule,
+  RuleEvaluationMetric.SemanticSimilarity
+>) {
+  return <>{result.score!.toFixed(0)}% similar</>
+}
+
+function ResultRowHeaders(
+  _props: ResultRowHeadersProps<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SemanticSimilarity
+  >,
+) {
+  return <></>
+}
+
+function ResultRowCells(
+  _props: ResultRowCellsProps<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SemanticSimilarity
+  >,
+) {
+  return <></>
+}
+
+function ResultPanelMetadata(
+  _props: ResultPanelProps<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SemanticSimilarity
+  >,
+) {
+  return <></>
+}
+
+function ResultPanelContent(
+  _props: ResultPanelProps<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SemanticSimilarity
+  >,
+) {
+  return <></>
+}
+
+function chartConfiguration({
+  evaluation,
+}: ChartConfigurationArgs<
+  EvaluationType.Rule,
+  RuleEvaluationMetric.SemanticSimilarity
+>) {
+  return {
+    min: 0,
+    max: 100,
+    thresholds: [
+      ...(evaluation.configuration.minSimilarity
+        ? [evaluation.configuration.minSimilarity]
+        : []),
+      ...(evaluation.configuration.maxSimilarity
+        ? [evaluation.configuration.maxSimilarity]
+        : []),
+    ] as const,
+    scale: (point: number) => point,
+    format: (point: number, short?: boolean) =>
+      short ? `${point.toFixed(0)}%` : `${point.toFixed(0)}% similar`,
+  }
+}
diff --git a/apps/web/src/components/evaluations/rule/index.tsx b/apps/web/src/components/evaluations/rule/index.tsx
@@ -18,6 +18,7 @@ import RuleEvaluationLengthCountSpecification from './LengthCount'
 import RuleEvaluationLexicalOverlapSpecification from './LexicalOverlap'
 import RuleEvaluationRegularExpressionSpecification from './RegularExpression'
 import RuleEvaluationSchemaValidationSpecification from './SchemaValidation'
+import RuleEvaluationSemanticSimilaritySpecification from './SemanticSimilarity'
 
 // prettier-ignore
 const METRICS: {
@@ -28,7 +29,7 @@ const METRICS: {
   [RuleEvaluationMetric.SchemaValidation]: RuleEvaluationSchemaValidationSpecification,
   [RuleEvaluationMetric.LengthCount]: RuleEvaluationLengthCountSpecification,
   [RuleEvaluationMetric.LexicalOverlap]: RuleEvaluationLexicalOverlapSpecification,
-  [RuleEvaluationMetric.SemanticSimilarity]:  undefined as any, // TODO: Implement
+  [RuleEvaluationMetric.SemanticSimilarity]: RuleEvaluationSemanticSimilaritySpecification,
 }
 
 const specification = RuleEvaluationSpecification
diff --git a/packages/constants/src/evaluations/rule.ts b/packages/constants/src/evaluations/rule.ts
@@ -179,7 +179,7 @@ export type RuleEvaluationLexicalOverlapResultError = z.infer<
 
 const ruleEvaluationSemanticSimilarityConfiguration =
   ruleEvaluationConfiguration.extend({
-    algorithm: z.literal('cosine_similarity'),
+    algorithm: z.enum(['cosine_distance']),
     minSimilarity: z.number().optional(), // Percentage of similarity
     maxSimilarity: z.number().optional(), // Percentage of similarity
   })
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -162,6 +162,7 @@
     "@modelcontextprotocol/sdk": "^1.6.0",
     "@tavily/core": "^0.3.1",
     "ajv": "^8.17.1",
+    "compute-cosine-similarity": "^1.1.0",
     "date-fns": "^3.6.0",
     "diff-match-patch": "^1.0.5",
     "fastest-levenshtein": "^1.0.16",
diff --git a/packages/core/src/repositories/evaluationResultsV2Repository.ts b/packages/core/src/repositories/evaluationResultsV2Repository.ts
@@ -244,7 +244,10 @@ export class EvaluationResultsV2Repository extends Repository<EvaluationResultV2
           : sql`0`.mapWith(Number),
     }
 
-    const filter = this.listByEvaluationFilter({ evaluationUuid, params })
+    const filter = and(
+      this.listByEvaluationFilter({ evaluationUuid, params }),
+      isNull(evaluationResultsV2.error),
+    )
 
     const totalStats = await this.db
       .select(stats)
diff --git a/packages/core/src/services/evaluationsV2/rule/index.ts b/packages/core/src/services/evaluationsV2/rule/index.ts
@@ -16,6 +16,7 @@ import RuleEvaluationLengthCountSpecification from './lengthCount'
 import RuleEvaluationLexicalOverlapSpecification from './lexicalOverlap'
 import RuleEvaluationRegularExpressionSpecification from './regularExpression'
 import RuleEvaluationSchemaValidationSpecification from './schemaValidation'
+import RuleEvaluationSemanticSimilaritySpecification from './semanticSimilarity'
 
 // prettier-ignore
 const METRICS: {
@@ -26,7 +27,7 @@ const METRICS: {
   [RuleEvaluationMetric.SchemaValidation]: RuleEvaluationSchemaValidationSpecification,
   [RuleEvaluationMetric.LengthCount]: RuleEvaluationLengthCountSpecification,
   [RuleEvaluationMetric.LexicalOverlap]: RuleEvaluationLexicalOverlapSpecification,
-  [RuleEvaluationMetric.SemanticSimilarity]: undefined as any, // TODO: Implement
+  [RuleEvaluationMetric.SemanticSimilarity]: RuleEvaluationSemanticSimilaritySpecification,
 }
 
 const specification = RuleEvaluationSpecification
diff --git a/packages/core/src/services/evaluationsV2/rule/semanticSimilarity.ts b/packages/core/src/services/evaluationsV2/rule/semanticSimilarity.ts
@@ -0,0 +1,142 @@
+import { createOpenAI } from '@ai-sdk/openai'
+import { env } from '@latitude-data/env'
+import { embedMany } from 'ai'
+import similarity from 'compute-cosine-similarity'
+import {
+  EvaluationType,
+  RuleEvaluationMetric,
+  RuleEvaluationSemanticSimilaritySpecification,
+} from '../../../browser'
+import { database, Database } from '../../../client'
+import { BadRequestError, Result } from '../../../lib'
+import {
+  EvaluationMetricRunArgs,
+  EvaluationMetricValidateArgs,
+  normalizeScore,
+} from '../shared'
+
+const specification = RuleEvaluationSemanticSimilaritySpecification
+export default {
+  ...specification,
+  validate: validate,
+  run: run,
+}
+
+async function validate(
+  {
+    configuration,
+  }: EvaluationMetricValidateArgs<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SemanticSimilarity
+  >,
+  _: Database = database,
+) {
+  if (
+    configuration.minSimilarity !== undefined &&
+    (configuration.minSimilarity < 0 || configuration.minSimilarity > 100)
+  ) {
+    return Result.error(
+      new BadRequestError(
+        'Minimum similarity must be a number between 0 and 100',
+      ),
+    )
+  }
+
+  if (
+    configuration.maxSimilarity !== undefined &&
+    (configuration.maxSimilarity < 0 || configuration.maxSimilarity > 100)
+  ) {
+    return Result.error(
+      new BadRequestError(
+        'Maximum similarity must be a number between 0 and 100',
+      ),
+    )
+  }
+
+  if (
+    configuration.minSimilarity !== undefined &&
+    configuration.maxSimilarity !== undefined &&
+    configuration.minSimilarity >= configuration.maxSimilarity
+  ) {
+    return Result.error(
+      new BadRequestError(
+        'Minimum similarity must be less than maximum similarity',
+      ),
+    )
+  }
+
+  // Note: all settings are explicitly returned to ensure we don't
+  // carry dangling fields from the original settings object
+  return Result.ok({
+    reverseScale: configuration.reverseScale,
+    algorithm: configuration.algorithm,
+    minSimilarity: configuration.minSimilarity,
+    maxSimilarity: configuration.maxSimilarity,
+  })
+}
+
+async function run(
+  {
+    evaluation,
+    actualOutput,
+    expectedOutput,
+  }: EvaluationMetricRunArgs<
+    EvaluationType.Rule,
+    RuleEvaluationMetric.SemanticSimilarity
+  >,
+  _: Database = database,
+) {
+  try {
+    let metadata = {
+      configuration: evaluation.configuration,
+      actualOutput: actualOutput,
+      expectedOutput: expectedOutput,
+    }
+
+    if (!metadata.expectedOutput) {
+      throw new BadRequestError('Expected output is required')
+    }
+
+    if (!env.OPENAI_API_KEY) {
+      throw new BadRequestError('Internal OPENAI_API_KEY is not set')
+    }
+
+    const {
+      embeddings: [actualEmbedding, expectedEmbedding],
+    } = await embedMany({
+      model: createOpenAI({
+        apiKey: env.OPENAI_API_KEY,
+        compatibility: 'strict',
+      }).textEmbeddingModel('text-embedding-3-small'),
+      values: [metadata.actualOutput, metadata.expectedOutput],
+    })
+
+    let score = 0
+
+    switch (metadata.configuration.algorithm) {
+      case 'cosine_distance':
+        {
+          score = (similarity(actualEmbedding!, expectedEmbedding!) ?? 0) * 100
+        }
+        break
+      default:
+        throw new Error('Invalid similarity algorithm')
+    }
+
+    score = Math.min(Math.max(Number(score.toFixed(0)), 0), 100)
+
+    const minSimilarity = metadata.configuration.minSimilarity ?? 0
+    const maxSimilarity = metadata.configuration.maxSimilarity ?? 100
+
+    let normalizedScore = normalizeScore(score, minSimilarity, maxSimilarity)
+    if (metadata.configuration.reverseScale) {
+      normalizedScore = normalizeScore(score, maxSimilarity, minSimilarity)
+    }
+
+    const hasPassed = score >= minSimilarity && score <= maxSimilarity
+
+    return { score, normalizedScore, metadata, hasPassed }
+  } catch (error) {
+    return { error: { message: (error as Error).message } }
+  }
+}
diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts
@@ -188,6 +188,7 @@ export const env = createEnv({
     CODESANDBOX_API_KEY: z.string().optional(),
     TAVILY_API_KEY: z.string().optional(),
     HANDINGER_API_KEY: z.string().optional(),
+    OPENAI_API_KEY: z.string().optional(),
 
     // Mail settings
     FROM_MAILER_EMAIL: z.string(),
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
