From 61d0e6a41b988769b13487cc2b6646066b3e72b6 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Wed, 2 Jul 2025 09:19:16 +0100 Subject: [PATCH 1/5] Update source.query.json-string.json --- .../ROOT/examples/docker-data/source.query.json-string.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/examples/docker-data/source.query.json-string.json b/modules/ROOT/examples/docker-data/source.query.json-string.json index 2efc7cd..e99084b 100644 --- a/modules/ROOT/examples/docker-data/source.query.json-string.json +++ b/modules/ROOT/examples/docker-data/source.query.json-string.json @@ -16,6 +16,7 @@ "neo4j.query.streaming-property": "timestamp", "neo4j.query.topic": "test-source", "neo4j.query.polling-interval": "1s", - "neo4j.query.polling-duration": "5s" + "neo4j.query.polling-duration": "5s", + "neo4j.payload-mode": "COMPACT" } -} \ No newline at end of file +} From e3583d5ef01b37e34051544aacc9aa1b69d19950 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Wed, 2 Jul 2025 09:41:05 +0100 Subject: [PATCH 2/5] Explain payload mode in source query section --- .../docker-data/source.query.json-string.json | 3 +- modules/ROOT/pages/source/query.adoc | 93 ++++++++++++++++++- 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/modules/ROOT/examples/docker-data/source.query.json-string.json b/modules/ROOT/examples/docker-data/source.query.json-string.json index e99084b..4f230d0 100644 --- a/modules/ROOT/examples/docker-data/source.query.json-string.json +++ b/modules/ROOT/examples/docker-data/source.query.json-string.json @@ -16,7 +16,6 @@ "neo4j.query.streaming-property": "timestamp", "neo4j.query.topic": "test-source", "neo4j.query.polling-interval": "1s", - "neo4j.query.polling-duration": "5s", - "neo4j.payload-mode": "COMPACT" + "neo4j.query.polling-duration": "5s" } } diff --git a/modules/ROOT/pages/source/query.adoc b/modules/ROOT/pages/source/query.adoc index 025d96a..f0d0926 100644 --- a/modules/ROOT/pages/source/query.adoc +++ b/modules/ROOT/pages/source/query.adoc @@ -101,9 +101,98 @@ curl -X POST http://localhost:8083/connectors \ This will create a Kafka Connect source instance that will send change event messages derived by the provided query over to the `my-topic` topic, using your preferred serialization format. In Control Center, confirm that the Source connector has been created in the Connect tab, under connect-default. -Generated change event messages in this case will have the following structure: +Generated change event messages in this case will have the following structure, which wraps each field into a dedicated structure which is type safe: [source,json] ---- -{"name": , "surname": , "timestamp": } +{ + "name": { + "type": "S", + "B": null, + "I64": null, + "F64": null, + "S": "", + "BA": null, + "TLD": null, + "TLDT": null, + "TLT": null, + "TZDT": null, + "TOT": null, + "TD": null, + "SP": null, + "LB": null, + "LI64": null, + "LF64": null, + "LS": null, + "LTLD": null, + "LTLDT": null, + "LTLT": null, + "LZDT": null, + "LTOT": null, + "LTD": null, + "LSP": null + }, + "surname": { + "type": "S", + "B": null, + "I64": null, + "F64": null, + "S": "", + "BA": null, + "TLD": null, + "TLDT": null, + "TLT": null, + "TZDT": null, + "TOT": null, + "TD": null, + "SP": null, + "LB": null, + "LI64": null, + "LF64": null, + "LS": null, + "LTLD": null, + "LTLDT": null, + "LTLT": null, + "LZDT": null, + "LTOT": null, + "LTD": null, + "LSP": null + }, + "timestamp": { + "type": "I64", + "B": null, + "I64": , + "F64": null, + "S": null, + "BA": null, + "TLD": null, + "TLDT": null, + "TLT": null, + "TZDT": null, + "TOT": null, + "TD": null, + "SP": null, + "LB": null, + "LI64": null, + "LF64": null, + "LS": null, + "LTLD": null, + "LTLDT": null, + "LTLT": null, + "LZDT": null, + "LTOT": null, + "LTD": null, + "LSP": null + } +} +---- + +While the above serialized form of the change message is type safe, it might make the consumer side deserialization logic quite complex and not desired. +In this case, you can configure `neo4j.payload-mode` setting as `COMPACT` and change event messages will instead be structured as; + +[source,json] ---- +{"name": "", "surname": "", "timestamp": } +---- + +Refer to xref:source/payload-mode.adoc[payload mode] for more information about `neo4j.payload-mode` setting, including its limitations. \ No newline at end of file From f95bdccd3e6e6eb7b76288dab9dac8aa0f2f9305 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Wed, 2 Jul 2025 10:05:11 +0100 Subject: [PATCH 3/5] Update modules/ROOT/pages/source/query.adoc Co-authored-by: Florent Biville <445792+fbiville@users.noreply.github.com> --- modules/ROOT/pages/source/query.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/pages/source/query.adoc b/modules/ROOT/pages/source/query.adoc index f0d0926..04b95f1 100644 --- a/modules/ROOT/pages/source/query.adoc +++ b/modules/ROOT/pages/source/query.adoc @@ -187,7 +187,7 @@ Generated change event messages in this case will have the following structure, } ---- -While the above serialized form of the change message is type safe, it might make the consumer side deserialization logic quite complex and not desired. +While the above serialized form of the change message is forward compatible in terms of property type changes, it might make the consumer side deserialization logic quite complex and not desired. In this case, you can configure `neo4j.payload-mode` setting as `COMPACT` and change event messages will instead be structured as; [source,json] From 3bf1ac86a838b39819e780745a648e3fca71cb66 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Wed, 2 Jul 2025 10:37:40 +0100 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Nicola Vitucci --- modules/ROOT/pages/source/query.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/pages/source/query.adoc b/modules/ROOT/pages/source/query.adoc index 04b95f1..907794e 100644 --- a/modules/ROOT/pages/source/query.adoc +++ b/modules/ROOT/pages/source/query.adoc @@ -188,11 +188,11 @@ Generated change event messages in this case will have the following structure, ---- While the above serialized form of the change message is forward compatible in terms of property type changes, it might make the consumer side deserialization logic quite complex and not desired. -In this case, you can configure `neo4j.payload-mode` setting as `COMPACT` and change event messages will instead be structured as; +In this case, you can configure `neo4j.payload-mode` setting as `COMPACT` so that change event messages will instead be structured as follows: [source,json] ---- {"name": "", "surname": "", "timestamp": } ---- -Refer to xref:source/payload-mode.adoc[payload mode] for more information about `neo4j.payload-mode` setting, including its limitations. \ No newline at end of file +Refer to the xref:source/payload-mode.adoc[payload mode] page for more information about the `neo4j.payload-mode` setting, including its limitations. \ No newline at end of file From 90c9e41f1350ee665542b3af36ae114db10b6270 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Wed, 2 Jul 2025 10:41:06 +0100 Subject: [PATCH 5/5] Extract sample jsons into files --- .../examples/producer-data/query.compact.json | 5 ++ .../producer-data/query.extended.json | 80 ++++++++++++++++++ modules/ROOT/pages/source/query.adoc | 83 +------------------ 3 files changed, 87 insertions(+), 81 deletions(-) create mode 100644 modules/ROOT/examples/producer-data/query.compact.json create mode 100644 modules/ROOT/examples/producer-data/query.extended.json diff --git a/modules/ROOT/examples/producer-data/query.compact.json b/modules/ROOT/examples/producer-data/query.compact.json new file mode 100644 index 0000000..222dc12 --- /dev/null +++ b/modules/ROOT/examples/producer-data/query.compact.json @@ -0,0 +1,5 @@ +{ + "name": "", + "surname": "", + "timestamp": +} diff --git a/modules/ROOT/examples/producer-data/query.extended.json b/modules/ROOT/examples/producer-data/query.extended.json new file mode 100644 index 0000000..fa568cc --- /dev/null +++ b/modules/ROOT/examples/producer-data/query.extended.json @@ -0,0 +1,80 @@ +{ + "name": { + "type": "S", + "B": null, + "I64": null, + "F64": null, + "S": "", + "BA": null, + "TLD": null, + "TLDT": null, + "TLT": null, + "TZDT": null, + "TOT": null, + "TD": null, + "SP": null, + "LB": null, + "LI64": null, + "LF64": null, + "LS": null, + "LTLD": null, + "LTLDT": null, + "LTLT": null, + "LZDT": null, + "LTOT": null, + "LTD": null, + "LSP": null + }, + "surname": { + "type": "S", + "B": null, + "I64": null, + "F64": null, + "S": "", + "BA": null, + "TLD": null, + "TLDT": null, + "TLT": null, + "TZDT": null, + "TOT": null, + "TD": null, + "SP": null, + "LB": null, + "LI64": null, + "LF64": null, + "LS": null, + "LTLD": null, + "LTLDT": null, + "LTLT": null, + "LZDT": null, + "LTOT": null, + "LTD": null, + "LSP": null + }, + "timestamp": { + "type": "I64", + "B": null, + "I64": , + "F64": null, + "S": null, + "BA": null, + "TLD": null, + "TLDT": null, + "TLT": null, + "TZDT": null, + "TOT": null, + "TD": null, + "SP": null, + "LB": null, + "LI64": null, + "LF64": null, + "LS": null, + "LTLD": null, + "LTLDT": null, + "LTLT": null, + "LZDT": null, + "LTOT": null, + "LTD": null, + "LSP": null + } +} \ No newline at end of file diff --git a/modules/ROOT/pages/source/query.adoc b/modules/ROOT/pages/source/query.adoc index 907794e..e71b87e 100644 --- a/modules/ROOT/pages/source/query.adoc +++ b/modules/ROOT/pages/source/query.adoc @@ -105,86 +105,7 @@ Generated change event messages in this case will have the following structure, [source,json] ---- -{ - "name": { - "type": "S", - "B": null, - "I64": null, - "F64": null, - "S": "", - "BA": null, - "TLD": null, - "TLDT": null, - "TLT": null, - "TZDT": null, - "TOT": null, - "TD": null, - "SP": null, - "LB": null, - "LI64": null, - "LF64": null, - "LS": null, - "LTLD": null, - "LTLDT": null, - "LTLT": null, - "LZDT": null, - "LTOT": null, - "LTD": null, - "LSP": null - }, - "surname": { - "type": "S", - "B": null, - "I64": null, - "F64": null, - "S": "", - "BA": null, - "TLD": null, - "TLDT": null, - "TLT": null, - "TZDT": null, - "TOT": null, - "TD": null, - "SP": null, - "LB": null, - "LI64": null, - "LF64": null, - "LS": null, - "LTLD": null, - "LTLDT": null, - "LTLT": null, - "LZDT": null, - "LTOT": null, - "LTD": null, - "LSP": null - }, - "timestamp": { - "type": "I64", - "B": null, - "I64": , - "F64": null, - "S": null, - "BA": null, - "TLD": null, - "TLDT": null, - "TLT": null, - "TZDT": null, - "TOT": null, - "TD": null, - "SP": null, - "LB": null, - "LI64": null, - "LF64": null, - "LS": null, - "LTLD": null, - "LTLDT": null, - "LTLT": null, - "LZDT": null, - "LTOT": null, - "LTD": null, - "LSP": null - } -} +include::example$producer-data/query.extended.json[] ---- While the above serialized form of the change message is forward compatible in terms of property type changes, it might make the consumer side deserialization logic quite complex and not desired. @@ -192,7 +113,7 @@ In this case, you can configure `neo4j.payload-mode` setting as `COMPACT` so tha [source,json] ---- -{"name": "", "surname": "", "timestamp": } +include::example$producer-data/query.compact.json[] ---- Refer to the xref:source/payload-mode.adoc[payload mode] page for more information about the `neo4j.payload-mode` setting, including its limitations. \ No newline at end of file