From 8d76dea6178c107db9ccf9367b7a4f015f3c1d83 Mon Sep 17 00:00:00 2001 From: Anton Rubin Date: Thu, 1 May 2025 00:48:51 +0100 Subject: [PATCH 1/8] adding percolate query dsl docs Signed-off-by: Anton Rubin --- .../supported-field-types/percolator.md | 2 +- _query-dsl/specialized/percolate.md | 620 ++++++++++++++++++ 2 files changed, 621 insertions(+), 1 deletion(-) create mode 100644 _query-dsl/specialized/percolate.md diff --git a/_field-types/supported-field-types/percolator.md b/_field-types/supported-field-types/percolator.md index 2b067cf5951..2aba821b0cb 100644 --- a/_field-types/supported-field-types/percolator.md +++ b/_field-types/supported-field-types/percolator.md @@ -13,7 +13,7 @@ redirect_from: **Introduced 1.0** {: .label .label-purple } -A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. +A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For detailed example see [Perculate]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/) ## Example diff --git a/_query-dsl/specialized/percolate.md b/_query-dsl/specialized/percolate.md new file mode 100644 index 00000000000..9c7cc2215ac --- /dev/null +++ b/_query-dsl/specialized/percolate.md @@ -0,0 +1,620 @@ +--- +layout: default +title: Percolate +parent: Specialized queries +nav_order: 55 +--- + +# Percolate + +Use the `percolate` query to find stored queries that match a given document. This is the opposite of a regular search. Instead of finding documents that match a query, you find queries that match a document. Percolate queries are often used for alerting, notifications, and reverse search use cases. + +- You can percolate a document provided inline or fetch an existing document from an index. +- The document and the stored queries must use the same field names and types. +- You can combine percolation with filtering and scoring to build complex matching systems. +- Percolate queries are considered expensive and will only run if the cluster setting `search.allow_expensive_queries` is set to `true` (default). If this setting is `false`, percolate queries will be rejected. + +## How percolation works + +1. Saved queries are stored in a special [`percolator` field type]({{site.url}}{{site.baseurl}}/field-types/supported-field-types/percolator/). +2. Documents are compared against all saved queries. +3. Matching queries are returned with their `_id`. +4. If highlighting is enabled, matched text snippets are also returned. +5. If multiple documents are sent, `_percolator_document_slot` displays which document matched. + +## Create an index to store saved queries + +First, create an index with `mappings` configured with [`percolator` field type]({{site.url}}{{site.baseurl}}/field-types/supported-field-types/percolator/) to store the saved queries: + +```json +PUT /my_percolator_index +{ + "mappings": { + "properties": { + "query": { + "type": "percolator" + }, + "title": { + "type": "text" + } + } + } +} +``` +{% include copy-curl.html %} + +Add a query matching "apple" in `title` field: + +```json +POST /my_percolator_index/_doc/1 +{ + "query": { + "match": { + "title": "apple" + } + } +} +``` +{% include copy-curl.html %} + +Add a query matching "banana" in `title` field: + +```json +POST /my_percolator_index/_doc/2 +{ + "query": { + "match": { + "title": "banana" + } + } +} +``` +{% include copy-curl.html %} + +## Percolate an inline document + +You can test an inline document against saved queries: + +```json +POST /my_percolator_index/_search +{ + "query": { + "percolate": { + "field": "query", + "document": { + "title": "Fresh Apple Harvest" + } + } + } +} +``` +{% include copy-curl.html %} + +This will match the saved query looking for "apple": + +```json +{ + ... + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 0.13076457, + "hits": [ + { + "_index": "my_percolator_index", + "_id": "1", + "_score": 0.13076457, + "_source": { + "query": { + "match": { + "title": "apple" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 0 + ] + } + } + ] + } +} +``` + +## Percolate an existing indexed document + +Create a separate index for your documents: + +```json +PUT /products +{ + "mappings": { + "properties": { + "title": { + "type": "text" + } + } + } +} +``` +{% include copy-curl.html %} + +Add a document: + +```json +POST /products/_doc/1 +{ + "title": "Banana Smoothie Special" +} +``` +{% include copy-curl.html %} + +Check if the stored queries match the indexed document: + +```json +POST /my_percolator_index/_search +{ + "query": { + "percolate": { + "field": "query", + "index": "products", + "id": "1" + } + } +} +``` +{% include copy-curl.html %} + +The corresponding query is returned: + +```json +{ + ... + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 0.13076457, + "hits": [ + { + "_index": "my_percolator_index", + "_id": "2", + "_score": 0.13076457, + "_source": { + "query": { + "match": { + "title": "banana" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 0 + ] + } + } + ] + } +} +``` + +## Batch percolation (multiple documents) + +You can check multiple documents in one request: + +```json +POST /my_percolator_index/_search +{ + "query": { + "percolate": { + "field": "query", + "documents": [ + { "title": "Apple event coming soon" }, + { "title": "Banana farms expand" }, + { "title": "Cherry season starts" } + ] + } + } +} +``` +{% include copy-curl.html %} + +Each match indicates which document slot matched using `_percolator_document_slot`: + +```json +{ + ... + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0.46484798, + "hits": [ + { + "_index": "my_percolator_index", + "_id": "2", + "_score": 0.46484798, + "_source": { + "query": { + "match": { + "title": "banana" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 1 + ] + } + }, + { + "_index": "my_percolator_index", + "_id": "1", + "_score": 0.41211313, + "_source": { + "query": { + "match": { + "title": "apple" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 0 + ] + } + } + ] + } +} +``` + +## Multi-query percolation using named query + +You can percolate different documents inside a named query: + +```json +GET /my_percolator_index/_search +{ + "query": { + "bool": { + "should": [ + { + "percolate": { + "field": "query", + "document": { + "title": "Apple pie recipe" + }, + "name": "apple_doc" + } + }, + { + "percolate": { + "field": "query", + "document": { + "title": "Banana bread instructions" + }, + "name": "banana_doc" + } + } + ] + } + } +} +``` +{% include copy-curl.html %} + +The `name` parameter is appended to `_percolator_document_slot` to distinguish which query was matched: + +```json +{ + ... + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0.13076457, + "hits": [ + { + "_index": "my_percolator_index", + "_id": "1", + "_score": 0.13076457, + "_source": { + "query": { + "match": { + "title": "apple" + } + } + }, + "fields": { + "_percolator_document_slot_apple_doc": [ + 0 + ] + } + }, + { + "_index": "my_percolator_index", + "_id": "2", + "_score": 0.13076457, + "_source": { + "query": { + "match": { + "title": "banana" + } + } + }, + "fields": { + "_percolator_document_slot_banana_doc": [ + 0 + ] + } + } + ] + } +} +``` + +This approach enables you to configure more custom query logic for individual documents. In the following example you can see how a field `title` is queried in the first document and `description` in the second, with added `boost` parameter: + +```json +GET /my_percolator_index/_search +{ + "query": { + "bool": { + "should": [ + { + "constant_score": { + "filter": { + "percolate": { + "field": "query", + "document": { + "title": "Apple pie recipe" + }, + "name": "apple_doc" + } + }, + "boost": 1.0 + } + }, + { + "constant_score": { + "filter": { + "percolate": { + "field": "query", + "document": { + "description": "Banana bread with honey" + }, + "name": "banana_doc" + } + }, + "boost": 3.0 + } + } + ] + } + } +} +``` +{% include copy-curl.html %} + + +## Batch vs named percolation + +Both `documents` and `bool` with `name` approaches can be used to percolate multiple documents, but they differ in how results are labeled, interpreted, and controlled. They provide functionally similar results but with important structural differences. See following comparison: + +| Feature | `documents` (Batch) | Named (`bool` + `percolate` + `name`) | +|-------------------------------|------------------------------------------------|--------------------------------------------------| +| Input format | One percolate clause, array of documents | Multiple percolate clauses, one per document | +| Traceability per document | By slot index (0, 1, ...) | By name (`apple_doc`, `banana_doc`) | +| Response field for match slot | `_percolator_document_slot: [0]` | `_percolator_document_slot_: [0]` | +| Highlight prefix | `0_title`, `1_title` | `apple_doc_title`, `banana_doc_title` | +| Custom control per doc | Not possible | Can customize each clause | +| Supports boosts / filters | No | Yes (per clause) | +| Performance | Best for large batches | Slightly slower with many clauses | +| Use case | Bulk matching jobs, large event streams | Per-document tracing, testing, custom control | + + +## Highlighting matches + +Percolate queries handle highlighting differently from regular queries: + +- In a regular query, the document is stored in the index, and the search query is used to highlight the matching terms. +- In a percolate query, the roles are reversed, the saved queries (in the percolator index) are used to highlight the document. + +This means the document provided via `document` or `documents` is the target for highlighting and the percolator queries determine which section is highlighted. + +### Single document highlighting + +Using previously defined searches in `my_percolator_index`, use the following command to highlight matches in the `title` field: + +```json +POST /my_percolator_index/_search +{ + "query": { + "percolate": { + "field": "query", + "document": { + "title": "Apple banana smoothie" + } + } + }, + "highlight": { + "fields": { + "title": {} + } + } +} +``` +{% include copy-curl.html %} + +The matches are highlighted depending on the query that was matched: + +```json +{ + ... + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0.13076457, + "hits": [ + { + "_index": "my_percolator_index", + "_id": "1", + "_score": 0.13076457, + "_source": { + "query": { + "match": { + "title": "apple" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 0 + ] + }, + "highlight": { + "title": [ + "Apple banana smoothie" + ] + } + }, + { + "_index": "my_percolator_index", + "_id": "2", + "_score": 0.13076457, + "_source": { + "query": { + "match": { + "title": "banana" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 0 + ] + }, + "highlight": { + "title": [ + "Apple banana smoothie" + ] + } + } + ] + } +} +``` + +### Multiple documents highlighting + +When percolating multiple documents using the `documents` array, a slot index is assigned to each document. The highlight keys then take the following form, where `` is the index of the document in your `documents` array: + +```json +"_": [ ... ] +``` + +Use the following command to percolate two documents with highlighting: + +```json +POST /my_percolator_index/_search +{ + "query": { + "percolate": { + "field": "query", + "documents": [ + { "title": "Apple pie recipe" }, + { "title": "Banana smoothie ideas" } + ] + } + }, + "highlight": { + "fields": { + "title": {} + } + } +} +``` +{% include copy-curl.html %} + +The response will contain highlighting fields prefixed with document slots, such as `0_title` and `1_title`: + +```json +{ + ... + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0.31506687, + "hits": [ + { + "_index": "my_percolator_index", + "_id": "1", + "_score": 0.31506687, + "_source": { + "query": { + "match": { + "title": "apple" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 0 + ] + }, + "highlight": { + "0_title": [ + "Apple pie recipe" + ] + } + }, + { + "_index": "my_percolator_index", + "_id": "2", + "_score": 0.31506687, + "_source": { + "query": { + "match": { + "title": "banana" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 1 + ] + }, + "highlight": { + "1_title": [ + "Banana smoothie ideas" + ] + } + } + ] + } +} +``` + +## Parameters + +| Parameter | Required/Optional | Description | +|-----------|-------------------|-------------| +| `field` | Required | Field containing the stored percolator queries. | +| `document` | Optional | Single inline document to match against saved queries. | +| `documents` | Optional | Array of multiple inline documents to match. | +| `index` | Optional | Index containing the document you want to match. | +| `id` | Optional | ID of the document to fetch from the index. | +| `routing` | Optional | Routing value to use when fetching the document. | +| `preference` | Optional | Preference for the shard routing when fetching the document. | +| `name` | Optional | Name assigned to a percolate clause, helpful when using multiple percolates in a `bool` query. | From e8802d2feb2bbea711157616150f3d1d7754d81e Mon Sep 17 00:00:00 2001 From: Anton Rubin Date: Thu, 1 May 2025 00:54:32 +0100 Subject: [PATCH 2/8] fixing vale errors Signed-off-by: Anton Rubin --- _field-types/supported-field-types/percolator.md | 2 +- _query-dsl/specialized/percolate.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/_field-types/supported-field-types/percolator.md b/_field-types/supported-field-types/percolator.md index 2aba821b0cb..8724377db27 100644 --- a/_field-types/supported-field-types/percolator.md +++ b/_field-types/supported-field-types/percolator.md @@ -13,7 +13,7 @@ redirect_from: **Introduced 1.0** {: .label .label-purple } -A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For detailed example see [Perculate]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/) +A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For detailed example see [Percolate]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/) ## Example diff --git a/_query-dsl/specialized/percolate.md b/_query-dsl/specialized/percolate.md index 9c7cc2215ac..be52364c41b 100644 --- a/_query-dsl/specialized/percolate.md +++ b/_query-dsl/specialized/percolate.md @@ -404,7 +404,7 @@ GET /my_percolator_index/_search {% include copy-curl.html %} -## Batch vs named percolation +## Batch compared to named percolation Both `documents` and `bool` with `name` approaches can be used to percolate multiple documents, but they differ in how results are labeled, interpreted, and controlled. They provide functionally similar results but with important structural differences. See following comparison: @@ -415,7 +415,7 @@ Both `documents` and `bool` with `name` approaches can be used to percolate mult | Response field for match slot | `_percolator_document_slot: [0]` | `_percolator_document_slot_: [0]` | | Highlight prefix | `0_title`, `1_title` | `apple_doc_title`, `banana_doc_title` | | Custom control per doc | Not possible | Can customize each clause | -| Supports boosts / filters | No | Yes (per clause) | +| Supports boosts and filters | No | Yes (per clause) | | Performance | Best for large batches | Slightly slower with many clauses | | Use case | Bulk matching jobs, large event streams | Per-document tracing, testing, custom control | From e749e625952caea15b22f9cf7f181b5d2f824d23 Mon Sep 17 00:00:00 2001 From: Anton Rubin Date: Thu, 3 Jul 2025 16:08:49 +0100 Subject: [PATCH 3/8] addressing the PR comments Signed-off-by: Anton Rubin --- _query-dsl/specialized/percolate.md | 84 +++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/_query-dsl/specialized/percolate.md b/_query-dsl/specialized/percolate.md index be52364c41b..1b0ca780b6d 100644 --- a/_query-dsl/specialized/percolate.md +++ b/_query-dsl/specialized/percolate.md @@ -14,6 +14,13 @@ Use the `percolate` query to find stored queries that match a given document. Th - You can combine percolation with filtering and scoring to build complex matching systems. - Percolate queries are considered expensive and will only run if the cluster setting `search.allow_expensive_queries` is set to `true` (default). If this setting is `false`, percolate queries will be rejected. +Percolate queries are useful in a variety of real-time matching scenarios. Some common use cases include: + +- **E-commerce notifications**: Customers can register interest in products, for example: “Notify me when new Apple laptops are in stock”. When new product documents are indexed, the system finds all users with matching saved queries and sends alerts. +- **Job alerts**: Job seekers save queries based on preferred job titles or locations, and new job postings are matched against these to trigger alerts. +- **Security and alerting systems**: Percolate incoming log or event data against saved rules or anomaly patterns. +- **News filtering**: Match incoming articles against saved topic profiles to categorize or deliver relevant content. + ## How percolation works 1. Saved queries are stored in a special [`percolator` field type]({{site.url}}{{site.baseurl}}/field-types/supported-field-types/percolator/). @@ -124,6 +131,80 @@ This will match the saved query looking for "apple": } ``` +## Percolate with multiple documents + +You can test multiple documents in the same query: + +```json +POST /my_percolator_index/_search +{ + "query": { + "percolate": { + "field": "query", + "documents": [ + { "title": "Banana flavoured ice-cream" }, + { "title": "Apple pie recipe" }, + { "title": "Banana bread instructions" }, + { "title": "Cherry tart" } + ] + } + } +} +``` +{% include copy-curl.html %} + +The `_percolator_document_slot` field helps you identify which document (by index) matched which saved query. + +```json +{ + ... + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "max_score": 0.54726034, + "hits": [ + { + "_index": "my_percolator_index", + "_id": "1", + "_score": 0.54726034, + "_source": { + "query": { + "match": { + "title": "apple" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 1 + ] + } + }, + { + "_index": "my_percolator_index", + "_id": "2", + "_score": 0.31506687, + "_source": { + "query": { + "match": { + "title": "banana" + } + } + }, + "fields": { + "_percolator_document_slot": [ + 0, + 2 + ] + } + } + ] + } +} +``` + ## Percolate an existing indexed document Create a separate index for your documents: @@ -168,6 +249,9 @@ POST /my_percolator_index/_search ``` {% include copy-curl.html %} +You must provide both `index` and `id` when using an already stored document. +{: .note} + The corresponding query is returned: ```json From 1a73d488740c1c0276f8c60515680ad6057dd621 Mon Sep 17 00:00:00 2001 From: AntonEliatra Date: Tue, 8 Jul 2025 08:18:45 +0100 Subject: [PATCH 4/8] Apply suggestions from code review Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Signed-off-by: AntonEliatra --- .../supported-field-types/percolator.md | 2 +- _query-dsl/specialized/percolate.md | 66 ++++++++++--------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/_field-types/supported-field-types/percolator.md b/_field-types/supported-field-types/percolator.md index 8724377db27..dbe497c719c 100644 --- a/_field-types/supported-field-types/percolator.md +++ b/_field-types/supported-field-types/percolator.md @@ -13,7 +13,7 @@ redirect_from: **Introduced 1.0** {: .label .label-purple } -A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For detailed example see [Percolate]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/) +A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For a detailed example see [Percolate]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/) ## Example diff --git a/_query-dsl/specialized/percolate.md b/_query-dsl/specialized/percolate.md index 1b0ca780b6d..02285c66eb6 100644 --- a/_query-dsl/specialized/percolate.md +++ b/_query-dsl/specialized/percolate.md @@ -7,7 +7,7 @@ nav_order: 55 # Percolate -Use the `percolate` query to find stored queries that match a given document. This is the opposite of a regular search. Instead of finding documents that match a query, you find queries that match a document. Percolate queries are often used for alerting, notifications, and reverse search use cases. +Use the `percolate` query to find stored queries that match a given document. This operation is the opposite of a regular search: instead of finding documents that match a query, you find queries that match a document. Percolate queries are often used for alerting, notifications, and reverse search use cases. - You can percolate a document provided inline or fetch an existing document from an index. - The document and the stored queries must use the same field names and types. @@ -16,7 +16,7 @@ Use the `percolate` query to find stored queries that match a given document. Th Percolate queries are useful in a variety of real-time matching scenarios. Some common use cases include: -- **E-commerce notifications**: Customers can register interest in products, for example: “Notify me when new Apple laptops are in stock”. When new product documents are indexed, the system finds all users with matching saved queries and sends alerts. +- **E-commerce notifications**: Users can register interest in products, for example: “Notify me when new Apple laptops are in stock”. When new product documents are indexed, the system finds all users with matching saved queries and sends alerts. - **Job alerts**: Job seekers save queries based on preferred job titles or locations, and new job postings are matched against these to trigger alerts. - **Security and alerting systems**: Percolate incoming log or event data against saved rules or anomaly patterns. - **News filtering**: Match incoming articles against saved topic profiles to categorize or deliver relevant content. @@ -25,13 +25,13 @@ Percolate queries are useful in a variety of real-time matching scenarios. Some 1. Saved queries are stored in a special [`percolator` field type]({{site.url}}{{site.baseurl}}/field-types/supported-field-types/percolator/). 2. Documents are compared against all saved queries. -3. Matching queries are returned with their `_id`. +3. Each matching query is returned with its `_id`. 4. If highlighting is enabled, matched text snippets are also returned. -5. If multiple documents are sent, `_percolator_document_slot` displays which document matched. +5. If multiple documents are sent, `_percolator_document_slot` displays the matching document. ## Create an index to store saved queries -First, create an index with `mappings` configured with [`percolator` field type]({{site.url}}{{site.baseurl}}/field-types/supported-field-types/percolator/) to store the saved queries: +First, create an index and configure its `mappings` with a [`percolator` field type]({{site.url}}{{site.baseurl}}/field-types/supported-field-types/percolator/) to store the saved queries: ```json PUT /my_percolator_index @@ -50,7 +50,7 @@ PUT /my_percolator_index ``` {% include copy-curl.html %} -Add a query matching "apple" in `title` field: +Add a query matching "apple" in the `title` field: ```json POST /my_percolator_index/_doc/1 @@ -64,7 +64,7 @@ POST /my_percolator_index/_doc/1 ``` {% include copy-curl.html %} -Add a query matching "banana" in `title` field: +Add a query matching "banana" in the `title` field: ```json POST /my_percolator_index/_doc/2 @@ -80,7 +80,7 @@ POST /my_percolator_index/_doc/2 ## Percolate an inline document -You can test an inline document against saved queries: +Test an inline document against the saved queries: ```json POST /my_percolator_index/_search @@ -133,7 +133,7 @@ This will match the saved query looking for "apple": ## Percolate with multiple documents -You can test multiple documents in the same query: +To test multiple documents in the same query, use the following request: ```json POST /my_percolator_index/_search @@ -153,7 +153,7 @@ POST /my_percolator_index/_search ``` {% include copy-curl.html %} -The `_percolator_document_slot` field helps you identify which document (by index) matched which saved query. +The `_percolator_document_slot` field helps you identify each document (by index) matching each saved query: ```json { @@ -307,7 +307,7 @@ POST /my_percolator_index/_search ``` {% include copy-curl.html %} -Each match indicates which document slot matched using `_percolator_document_slot`: +Each match indicates the matching document in the `_percolator_document_slot` field: ```json { @@ -358,7 +358,7 @@ Each match indicates which document slot matched using `_percolator_document_slo } ``` -## Multi-query percolation using named query +## Multi-query percolation using a named query You can percolate different documents inside a named query: @@ -393,7 +393,7 @@ GET /my_percolator_index/_search ``` {% include copy-curl.html %} -The `name` parameter is appended to `_percolator_document_slot` to distinguish which query was matched: +The `name` parameter is appended to `_percolator_document_slot` to provide the matching query: ```json { @@ -444,7 +444,7 @@ The `name` parameter is appended to `_percolator_document_slot` to distinguish w } ``` -This approach enables you to configure more custom query logic for individual documents. In the following example you can see how a field `title` is queried in the first document and `description` in the second, with added `boost` parameter: +This approach enables you to configure more custom query logic for individual documents. In the following example, the `title` field is queried in the first document and the `description` field is queried in the second document. The `boost` parameter is also provided: ```json GET /my_percolator_index/_search @@ -488,19 +488,19 @@ GET /my_percolator_index/_search {% include copy-curl.html %} -## Batch compared to named percolation +## Batch percolation compared to named percolation -Both `documents` and `bool` with `name` approaches can be used to percolate multiple documents, but they differ in how results are labeled, interpreted, and controlled. They provide functionally similar results but with important structural differences. See following comparison: +Both batch percolation (using `documents`) and named percolation (using `bool` with `name`) can be used to percolate multiple documents, but they differ in how results are labeled, interpreted, and controlled. They provide functionally similar results but with important structural differences, described in the following table. -| Feature | `documents` (Batch) | Named (`bool` + `percolate` + `name`) | +| Feature | Batch (`documents`) | Named (`bool` + `percolate` + `name`) | |-------------------------------|------------------------------------------------|--------------------------------------------------| | Input format | One percolate clause, array of documents | Multiple percolate clauses, one per document | | Traceability per document | By slot index (0, 1, ...) | By name (`apple_doc`, `banana_doc`) | | Response field for match slot | `_percolator_document_slot: [0]` | `_percolator_document_slot_: [0]` | | Highlight prefix | `0_title`, `1_title` | `apple_doc_title`, `banana_doc_title` | -| Custom control per doc | Not possible | Can customize each clause | +| Custom control per doc | Not supported | Can customize each clause | | Supports boosts and filters | No | Yes (per clause) | -| Performance | Best for large batches | Slightly slower with many clauses | +| Performance | Best for large batches | Slightly slower when there are many clauses | | Use case | Bulk matching jobs, large event streams | Per-document tracing, testing, custom control | @@ -509,13 +509,13 @@ Both `documents` and `bool` with `name` approaches can be used to percolate mult Percolate queries handle highlighting differently from regular queries: - In a regular query, the document is stored in the index, and the search query is used to highlight the matching terms. -- In a percolate query, the roles are reversed, the saved queries (in the percolator index) are used to highlight the document. +- In a percolate query, the roles are reversed: the saved queries (in the percolator index) are used to highlight the document. -This means the document provided via `document` or `documents` is the target for highlighting and the percolator queries determine which section is highlighted. +This means the document provided in `document` or `documents` is the target for highlighting and the percolator queries determine the sections that are highlighted. ### Single document highlighting -Using previously defined searches in `my_percolator_index`, use the following command to highlight matches in the `title` field: +This example uses the previously defined searches in `my_percolator_index`. Use the following request to highlight matches in the `title` field: ```json POST /my_percolator_index/_search @@ -598,7 +598,7 @@ The matches are highlighted depending on the query that was matched: } ``` -### Multiple documents highlighting +### Multiple document highlighting When percolating multiple documents using the `documents` array, a slot index is assigned to each document. The highlight keys then take the following form, where `` is the index of the document in your `documents` array: @@ -629,7 +629,7 @@ POST /my_percolator_index/_search ``` {% include copy-curl.html %} -The response will contain highlighting fields prefixed with document slots, such as `0_title` and `1_title`: +The response contains highlighting fields prefixed with document slots, such as `0_title` and `1_title`: ```json { @@ -692,13 +692,15 @@ The response will contain highlighting fields prefixed with document slots, such ## Parameters +The `percolate` query supports the following parameters. + | Parameter | Required/Optional | Description | |-----------|-------------------|-------------| -| `field` | Required | Field containing the stored percolator queries. | -| `document` | Optional | Single inline document to match against saved queries. | -| `documents` | Optional | Array of multiple inline documents to match. | -| `index` | Optional | Index containing the document you want to match. | -| `id` | Optional | ID of the document to fetch from the index. | -| `routing` | Optional | Routing value to use when fetching the document. | -| `preference` | Optional | Preference for the shard routing when fetching the document. | -| `name` | Optional | Name assigned to a percolate clause, helpful when using multiple percolates in a `bool` query. | +| `field` | Required | The field containing the stored percolator queries. | +| `document` | Optional | A single inline document to match against saved queries. | +| `documents` | Optional | An array of multiple inline documents to match against saved queries. | +| `index` | Optional | An index containing the document you want to match. | +| `id` | Optional | The ID of the document to fetch from the index. | +| `routing` | Optional | The routing value to use when fetching the document. | +| `preference` | Optional | The preference for the shard routing when fetching the document. | +| `name` | Optional | The name assigned to a `percolate` clause. Helpful when using multiple `percolate` clauses in a `bool` query. | From 43453fd7b772fca7fabdf1710a8a6c6e36af0e3e Mon Sep 17 00:00:00 2001 From: Anton Rubin Date: Tue, 8 Jul 2025 08:54:54 +0100 Subject: [PATCH 5/8] addressing the PR comments Signed-off-by: Anton Rubin --- _query-dsl/specialized/percolate.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/_query-dsl/specialized/percolate.md b/_query-dsl/specialized/percolate.md index 02285c66eb6..1427f9dd22d 100644 --- a/_query-dsl/specialized/percolate.md +++ b/_query-dsl/specialized/percolate.md @@ -9,10 +9,12 @@ nav_order: 55 Use the `percolate` query to find stored queries that match a given document. This operation is the opposite of a regular search: instead of finding documents that match a query, you find queries that match a document. Percolate queries are often used for alerting, notifications, and reverse search use cases. +When working with `percolate` queries, consider the following key points: + - You can percolate a document provided inline or fetch an existing document from an index. - The document and the stored queries must use the same field names and types. - You can combine percolation with filtering and scoring to build complex matching systems. -- Percolate queries are considered expensive and will only run if the cluster setting `search.allow_expensive_queries` is set to `true` (default). If this setting is `false`, percolate queries will be rejected. +- Percolate queries are considered [expensive queries]({{site.url}}{{site.baseurl}}/query-dsl/#expensive-queries) and will only run if the cluster setting `search.allow_expensive_queries` is set to `true` (default). If this setting is `false`, percolate queries will be rejected. Percolate queries are useful in a variety of real-time matching scenarios. Some common use cases include: @@ -29,7 +31,11 @@ Percolate queries are useful in a variety of real-time matching scenarios. Some 4. If highlighting is enabled, matched text snippets are also returned. 5. If multiple documents are sent, `_percolator_document_slot` displays the matching document. -## Create an index to store saved queries +## Example + +The following examples demonstrate how to store percolator queries and test documents against them using different methods. + +### Create an index to store saved queries First, create an index and configure its `mappings` with a [`percolator` field type]({{site.url}}{{site.baseurl}}/field-types/supported-field-types/percolator/) to store the saved queries: @@ -78,7 +84,7 @@ POST /my_percolator_index/_doc/2 ``` {% include copy-curl.html %} -## Percolate an inline document +### Percolate an inline document Test an inline document against the saved queries: @@ -97,7 +103,7 @@ POST /my_percolator_index/_search ``` {% include copy-curl.html %} -This will match the saved query looking for "apple": +The response provides the stored percolator query that searches for documents containing the word “apple” in the `title` field, identified by `_id`: `1` ```json { @@ -131,7 +137,7 @@ This will match the saved query looking for "apple": } ``` -## Percolate with multiple documents +### Percolate with multiple documents To test multiple documents in the same query, use the following request: @@ -205,7 +211,9 @@ The `_percolator_document_slot` field helps you identify each document (by index } ``` -## Percolate an existing indexed document +### Percolate an existing indexed document + +You can reference an existing document already stored in another index to check for matching percolator queries. Create a separate index for your documents: @@ -286,7 +294,7 @@ The corresponding query is returned: } ``` -## Batch percolation (multiple documents) +### Batch percolation (multiple documents) You can check multiple documents in one request: @@ -358,7 +366,7 @@ Each match indicates the matching document in the `_percolator_document_slot` fi } ``` -## Multi-query percolation using a named query +### Multi-query percolation using a named query You can percolate different documents inside a named query: From 042fde12c902c650904aba5b24ac1aa3bf075e72 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:24:24 -0400 Subject: [PATCH 6/8] Update _field-types/supported-field-types/percolator.md Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> --- _field-types/supported-field-types/percolator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_field-types/supported-field-types/percolator.md b/_field-types/supported-field-types/percolator.md index dbe497c719c..866cad856f7 100644 --- a/_field-types/supported-field-types/percolator.md +++ b/_field-types/supported-field-types/percolator.md @@ -13,7 +13,7 @@ redirect_from: **Introduced 1.0** {: .label .label-purple } -A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For a detailed example see [Percolate]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/) +A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For a detailed example, see [Percolate]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/) ## Example From ebd889373c2afd929fadbcd979547afb7a36edbf Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:24:58 -0400 Subject: [PATCH 7/8] Update _field-types/supported-field-types/percolator.md Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> --- _field-types/supported-field-types/percolator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_field-types/supported-field-types/percolator.md b/_field-types/supported-field-types/percolator.md index 866cad856f7..ab53e46a3ed 100644 --- a/_field-types/supported-field-types/percolator.md +++ b/_field-types/supported-field-types/percolator.md @@ -13,7 +13,7 @@ redirect_from: **Introduced 1.0** {: .label .label-purple } -A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For a detailed example, see [Percolate]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/) +A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For a detailed example, see [Percolate query]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/). ## Example From 773b91f559ec740af09d771d73617102e8fb12cd Mon Sep 17 00:00:00 2001 From: Nathan Bower Date: Tue, 8 Jul 2025 10:30:06 -0400 Subject: [PATCH 8/8] Apply suggestions from code review Signed-off-by: Nathan Bower --- .../supported-field-types/percolator.md | 2 +- _query-dsl/specialized/percolate.md | 32 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/_field-types/supported-field-types/percolator.md b/_field-types/supported-field-types/percolator.md index ab53e46a3ed..229677bfd99 100644 --- a/_field-types/supported-field-types/percolator.md +++ b/_field-types/supported-field-types/percolator.md @@ -13,7 +13,7 @@ redirect_from: **Introduced 1.0** {: .label .label-purple } -A percolator field type specifies to treat this field as a query. Any JSON object field can be marked as a percolator field. Normally, documents are indexed and searches are run against them. When you use a percolator field, you store a search, and later the percolate query matches documents to that search. For a detailed example, see [Percolate query]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/). +A `percolator` field type specifies to treat this field as a query. Any JSON object field can be marked as a `percolator` field. Normally, documents are indexed and searches are run against them. When you use a `percolator` field, you store a search, and the `percolate` query later matches documents to that search. For a detailed example, see [Percolate query]({{site.url}}{{site.baseurl}}/query-dsl/specialized/percolate/). ## Example diff --git a/_query-dsl/specialized/percolate.md b/_query-dsl/specialized/percolate.md index 1427f9dd22d..24c88c207b2 100644 --- a/_query-dsl/specialized/percolate.md +++ b/_query-dsl/specialized/percolate.md @@ -7,18 +7,18 @@ nav_order: 55 # Percolate -Use the `percolate` query to find stored queries that match a given document. This operation is the opposite of a regular search: instead of finding documents that match a query, you find queries that match a document. Percolate queries are often used for alerting, notifications, and reverse search use cases. +Use the `percolate` query to find stored queries that match a given document. This operation is the opposite of a regular search: instead of finding documents that match a query, you find queries that match a document. `percolate` queries are often used for alerting, notifications, and reverse search use cases. When working with `percolate` queries, consider the following key points: - You can percolate a document provided inline or fetch an existing document from an index. - The document and the stored queries must use the same field names and types. - You can combine percolation with filtering and scoring to build complex matching systems. -- Percolate queries are considered [expensive queries]({{site.url}}{{site.baseurl}}/query-dsl/#expensive-queries) and will only run if the cluster setting `search.allow_expensive_queries` is set to `true` (default). If this setting is `false`, percolate queries will be rejected. +- `percolate` queries are considered [expensive queries]({{site.url}}{{site.baseurl}}/query-dsl/#expensive-queries) and will only run if the cluster setting `search.allow_expensive_queries` is set to `true` (default). If this setting is `false`, `percolate` queries will be rejected. -Percolate queries are useful in a variety of real-time matching scenarios. Some common use cases include: +`percolate` queries are useful in a variety of real-time matching scenarios. Some common use cases include: -- **E-commerce notifications**: Users can register interest in products, for example: “Notify me when new Apple laptops are in stock”. When new product documents are indexed, the system finds all users with matching saved queries and sends alerts. +- **E-commerce notifications**: Users can register interest in products, for example, "Notify me when new Apple laptops are in stock". When new product documents are indexed, the system finds all users with matching saved queries and sends alerts. - **Job alerts**: Job seekers save queries based on preferred job titles or locations, and new job postings are matched against these to trigger alerts. - **Security and alerting systems**: Percolate incoming log or event data against saved rules or anomaly patterns. - **News filtering**: Match incoming articles against saved topic profiles to categorize or deliver relevant content. @@ -33,9 +33,9 @@ Percolate queries are useful in a variety of real-time matching scenarios. Some ## Example -The following examples demonstrate how to store percolator queries and test documents against them using different methods. +The following examples demonstrate how to store `percolate` queries and test documents against them using different methods. -### Create an index to store saved queries +### Create an index for storing saved queries First, create an index and configure its `mappings` with a [`percolator` field type]({{site.url}}{{site.baseurl}}/field-types/supported-field-types/percolator/) to store the saved queries: @@ -103,7 +103,7 @@ POST /my_percolator_index/_search ``` {% include copy-curl.html %} -The response provides the stored percolator query that searches for documents containing the word “apple” in the `title` field, identified by `_id`: `1` +The response provides the stored `percolate` query that searches for documents containing the word "apple" in the `title` field, identified by `_id`: `1`: ```json { @@ -213,7 +213,7 @@ The `_percolator_document_slot` field helps you identify each document (by index ### Percolate an existing indexed document -You can reference an existing document already stored in another index to check for matching percolator queries. +You can reference an existing document already stored in another index to check for matching `percolate` queries. Create a separate index for your documents: @@ -241,7 +241,7 @@ POST /products/_doc/1 ``` {% include copy-curl.html %} -Check if the stored queries match the indexed document: +Check whether the stored queries match the indexed document: ```json POST /my_percolator_index/_search @@ -257,7 +257,7 @@ POST /my_percolator_index/_search ``` {% include copy-curl.html %} -You must provide both `index` and `id` when using an already stored document. +You must provide both `index` and `id` when using a stored document. {: .note} The corresponding query is returned: @@ -514,14 +514,14 @@ Both batch percolation (using `documents`) and named percolation (using `bool` w ## Highlighting matches -Percolate queries handle highlighting differently from regular queries: +`percolate` queries handle highlighting differently from regular queries: - In a regular query, the document is stored in the index, and the search query is used to highlight the matching terms. -- In a percolate query, the roles are reversed: the saved queries (in the percolator index) are used to highlight the document. +- In a `percolate` query, the roles are reversed: the saved queries (in the percolator index) are used to highlight the document. -This means the document provided in `document` or `documents` is the target for highlighting and the percolator queries determine the sections that are highlighted. +This means that the document provided in `document` or `documents` is the target for highlighting and that the `percolate` queries determine the sections to be highlighted. -### Single document highlighting +### Highlighting a single document This example uses the previously defined searches in `my_percolator_index`. Use the following request to highlight matches in the `title` field: @@ -606,7 +606,7 @@ The matches are highlighted depending on the query that was matched: } ``` -### Multiple document highlighting +### Highlighting multiple documents When percolating multiple documents using the `documents` array, a slot index is assigned to each document. The highlight keys then take the following form, where `` is the index of the document in your `documents` array: @@ -704,7 +704,7 @@ The `percolate` query supports the following parameters. | Parameter | Required/Optional | Description | |-----------|-------------------|-------------| -| `field` | Required | The field containing the stored percolator queries. | +| `field` | Required | The field containing the stored `percolate` queries. | | `document` | Optional | A single inline document to match against saved queries. | | `documents` | Optional | An array of multiple inline documents to match against saved queries. | | `index` | Optional | An index containing the document you want to match. |