From bebbfceab6e22a6bdeda3c3e168aceaf52994abd Mon Sep 17 00:00:00 2001 From: dperezcabrera Date: Sat, 13 Apr 2024 11:05:48 +0200 Subject: [PATCH] Refactor PgVercorStore filter template to use JSONB field access Changed the filter template in PgVercorStore to replace JSONPath expressions with JSONB field access, using standard SQL operators such as '=' instead of '==', 'AND' instead of '&&', and 'OR' instead of '||'. This adjustment addresses compatibility issues with the 'IN' operator, which previously returned parse errors. Also updated PgVectorFilterExpressionConverter and corresponding tests to align with these changes. - Replaced `metadata::jsonb @@ '$.key == "value"'::jsonpath` with `metadata::jsonb->>'$.key' = 'value'` - Fixed parsing issues with `metadata::jsonb @@ '$.key in ["value"]'::jsonpath` by using JSONB field access. - Updated logical operators IN filter expressions to standard SQL syntax. These changes improve the clarity, execution reliability, and compatibility of filter expressions in the database. --- .../PgVectorFilterExpressionConverter.java | 40 ++++++++++++++++--- ...gVectorFilterExpressionConverterTests.java | 21 +++++----- .../ai/vectorstore/PgVectorStore.java | 2 +- .../ai/vectorstore/PgVectorStoreIT.java | 5 +++ 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/filter/converter/PgVectorFilterExpressionConverter.java b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/filter/converter/PgVectorFilterExpressionConverter.java index fc81c218ce5..e88e9d5ea25 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/filter/converter/PgVectorFilterExpressionConverter.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/filter/converter/PgVectorFilterExpressionConverter.java @@ -15,6 +15,7 @@ */ package org.springframework.ai.vectorstore.filter.converter; +import org.springframework.ai.vectorstore.filter.Filter; import org.springframework.ai.vectorstore.filter.Filter.Expression; import org.springframework.ai.vectorstore.filter.Filter.Group; import org.springframework.ai.vectorstore.filter.Filter.Key; @@ -37,11 +38,11 @@ protected void doExpression(Expression expression, StringBuilder context) { private String getOperationSymbol(Expression exp) { switch (exp.type()) { case AND: - return " && "; + return " AND "; case OR: - return " || "; + return " OR "; case EQ: - return " == "; + return " = "; case NE: return " != "; case LT: @@ -53,9 +54,9 @@ private String getOperationSymbol(Expression exp) { case GTE: return " >= "; case IN: - return " in "; + return " IN "; case NIN: - return " nin "; + return " NOT IN "; default: throw new RuntimeException("Not supported expression type: " + exp.type()); } @@ -63,7 +64,24 @@ private String getOperationSymbol(Expression exp) { @Override protected void doKey(Key key, StringBuilder context) { - context.append("$." + key.key()); + context.append("metadata::jsonb->>'"); + if (hasOuterQuotes(key.key())) { + context.append(removeOuterQuotes(key.key())); + } + else { + context.append(key.key()); + } + context.append('\''); + } + + @Override + protected void doSingleValue(Object value, StringBuilder context) { + if (value instanceof String) { + context.append(String.format("\'%s\'", value)); + } + else { + context.append(value); + } } @Override @@ -76,4 +94,14 @@ protected void doEndGroup(Group group, StringBuilder context) { context.append(")"); } + @Override + protected void doStartValueRange(Filter.Value listValue, StringBuilder context) { + context.append("("); + } + + @Override + protected void doEndValueRange(Filter.Value listValue, StringBuilder context) { + context.append(")"); + } + } \ No newline at end of file diff --git a/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/filter/converter/PgVectorFilterExpressionConverterTests.java b/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/filter/converter/PgVectorFilterExpressionConverterTests.java index b6856d12b15..9463ac08a16 100644 --- a/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/filter/converter/PgVectorFilterExpressionConverterTests.java +++ b/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/filter/converter/PgVectorFilterExpressionConverterTests.java @@ -46,7 +46,7 @@ public class PgVectorFilterExpressionConverterTests { public void testEQ() { // country == "BG" String vectorExpr = converter.convertExpression(new Expression(EQ, new Key("country"), new Value("BG"))); - assertThat(vectorExpr).isEqualTo("$.country == \"BG\""); + assertThat(vectorExpr).isEqualTo("metadata::jsonb->>'country' = 'BG'"); } @Test @@ -55,7 +55,7 @@ public void tesEqAndGte() { String vectorExpr = converter .convertExpression(new Expression(AND, new Expression(EQ, new Key("genre"), new Value("drama")), new Expression(GTE, new Key("year"), new Value(2020)))); - assertThat(vectorExpr).isEqualTo("$.genre == \"drama\" && $.year >= 2020"); + assertThat(vectorExpr).isEqualTo("metadata::jsonb->>'genre' = 'drama' AND metadata::jsonb->>'year' >= 2020"); } @Test @@ -63,7 +63,7 @@ public void tesIn() { // genre in ["comedy", "documentary", "drama"] String vectorExpr = converter.convertExpression( new Expression(IN, new Key("genre"), new Value(List.of("comedy", "documentary", "drama")))); - assertThat(vectorExpr).isEqualTo("$.genre in [\"comedy\",\"documentary\",\"drama\"]"); + assertThat(vectorExpr).isEqualTo("metadata::jsonb->>'genre' IN ('comedy','documentary','drama')"); } @Test @@ -73,7 +73,8 @@ public void testNe() { .convertExpression(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)), new Expression(AND, new Expression(EQ, new Key("country"), new Value("BG")), new Expression(NE, new Key("city"), new Value("Sofia"))))); - assertThat(vectorExpr).isEqualTo("$.year >= 2020 || $.country == \"BG\" && $.city != \"Sofia\""); + assertThat(vectorExpr).isEqualTo( + "metadata::jsonb->>'year' >= 2020 OR metadata::jsonb->>'country' = 'BG' AND metadata::jsonb->>'city' != 'Sofia'"); } @Test @@ -83,8 +84,8 @@ public void testGroup() { new Group(new Expression(OR, new Expression(GTE, new Key("year"), new Value(2020)), new Expression(EQ, new Key("country"), new Value("BG")))), new Expression(NIN, new Key("city"), new Value(List.of("Sofia", "Plovdiv"))))); - assertThat(vectorExpr) - .isEqualTo("($.year >= 2020 || $.country == \"BG\") && $.city nin [\"Sofia\",\"Plovdiv\"]"); + assertThat(vectorExpr).isEqualTo( + "(metadata::jsonb->>'year' >= 2020 OR metadata::jsonb->>'country' = 'BG') AND metadata::jsonb->>'city' NOT IN ('Sofia','Plovdiv')"); } @Test @@ -95,7 +96,8 @@ public void tesBoolean() { new Expression(GTE, new Key("year"), new Value(2020))), new Expression(IN, new Key("country"), new Value(List.of("BG", "NL", "US"))))); - assertThat(vectorExpr).isEqualTo("$.isOpen == true && $.year >= 2020 && $.country in [\"BG\",\"NL\",\"US\"]"); + assertThat(vectorExpr).isEqualTo( + "metadata::jsonb->>'isOpen' = true AND metadata::jsonb->>'year' >= 2020 AND metadata::jsonb->>'country' IN ('BG','NL','US')"); } @Test @@ -105,14 +107,15 @@ public void testDecimal() { .convertExpression(new Expression(AND, new Expression(GTE, new Key("temperature"), new Value(-15.6)), new Expression(LTE, new Key("temperature"), new Value(20.13)))); - assertThat(vectorExpr).isEqualTo("$.temperature >= -15.6 && $.temperature <= 20.13"); + assertThat(vectorExpr) + .isEqualTo("metadata::jsonb->>'temperature' >= -15.6 AND metadata::jsonb->>'temperature' <= 20.13"); } @Test public void testComplexIdentifiers() { String vectorExpr = converter .convertExpression(new Expression(EQ, new Key("\"country 1 2 3\""), new Value("BG"))); - assertThat(vectorExpr).isEqualTo("$.\"country 1 2 3\" == \"BG\""); + assertThat(vectorExpr).isEqualTo("metadata::jsonb->>'country 1 2 3' = 'BG'"); } } diff --git a/vector-stores/spring-ai-pgvector-store/src/main/java/org/springframework/ai/vectorstore/PgVectorStore.java b/vector-stores/spring-ai-pgvector-store/src/main/java/org/springframework/ai/vectorstore/PgVectorStore.java index 1d85ee361df..b7efe776d3a 100644 --- a/vector-stores/spring-ai-pgvector-store/src/main/java/org/springframework/ai/vectorstore/PgVectorStore.java +++ b/vector-stores/spring-ai-pgvector-store/src/main/java/org/springframework/ai/vectorstore/PgVectorStore.java @@ -292,7 +292,7 @@ public List similaritySearch(SearchRequest request) { String jsonPathFilter = ""; if (StringUtils.hasText(nativeFilterExpression)) { - jsonPathFilter = " AND metadata::jsonb @@ '" + nativeFilterExpression + "'::jsonpath "; + jsonPathFilter = " AND (" + nativeFilterExpression + ") "; } double distance = 1 - request.getSimilarityThreshold(); diff --git a/vector-stores/spring-ai-pgvector-store/src/test/java/org/springframework/ai/vectorstore/PgVectorStoreIT.java b/vector-stores/spring-ai-pgvector-store/src/test/java/org/springframework/ai/vectorstore/PgVectorStoreIT.java index 9bfb63e567c..f8a252b636c 100644 --- a/vector-stores/spring-ai-pgvector-store/src/test/java/org/springframework/ai/vectorstore/PgVectorStoreIT.java +++ b/vector-stores/spring-ai-pgvector-store/src/test/java/org/springframework/ai/vectorstore/PgVectorStoreIT.java @@ -157,6 +157,11 @@ public void searchWithFilters(String distanceType) { assertThat(results).hasSize(1); assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId()); + results = vectorStore.similaritySearch(searchRequest.withFilterExpression("country in ['NL', 'SP']")); + + assertThat(results).hasSize(1); + assertThat(results.get(0).getId()).isEqualTo(nlDocument.getId()); + results = vectorStore.similaritySearch(searchRequest.withFilterExpression("country == 'BG'")); assertThat(results).hasSize(2);