Skip to content

Commit 5c35e96

Browse files
andy-stark-redisandy-stark-redis
authored
Merge pull request #1174 from redis/DOC-4863-update-rdi-config-example
DOC-4863 update config.yaml example
2 parents a88acdc + 55c569f commit 5c35e96

File tree

1 file changed

+111
-52
lines changed

1 file changed

+111
-52
lines changed

‎content/integrate/redis-data-integration/data-pipelines/data-pipelines.md

Lines changed: 111 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ form "`${name}`" refer to environment variables that you should set with the
7373
[`redis-di set-secret`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-set-secret" >}})
7474
command. In particular, you should normally use environment variables as shown to set the source
7575
and target username and password rather than storing them in plain text in this
76-
file (see [Set secrets](#set-secrets) below for more information).
76+
file (see [Set secrets]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets" >}}) for more information).
7777

7878
```yaml
7979
sources:
@@ -83,64 +83,123 @@ sources:
8383
level: info
8484
connection:
8585
type: mysql
86-
host: ${RDI_REDIS_HOST}
87-
port: 13000
86+
host: <DB_HOST> # e.g. localhost
87+
port: 3306
88+
# User and password are injected from the secrets.
8889
user: ${SOURCE_DB_USERNAME}
8990
password: ${SOURCE_DB_PASSWORD}
90-
# The names of the following properties should match the ones you used
91-
# when setting the TLS/mTLS secrets. Set only `cacert` if you are using
92-
# TLS, but set all of them if you are using mTLS:
93-
# key: ${SOURCE_DB_KEY}
94-
# cert: ${SOURCE_DB_CERT}
95-
# cacert: ${SOURCE_DB_CACERT}
96-
# key_password: ${SOURCE_DB_KEY_PASSWORD}
97-
tables:
98-
redislabscdc.emp:
99-
snapshot_sql: "SELECT * from redislabscdc.emp WHERE empno < 1000"
100-
columns:
101-
- empno
102-
- fname
103-
- lname
104-
# Advanced collector properties (optional):
105-
# advanced:
106-
# Sink collector properties - see the full list at https://debezium.io/documentation/reference/stable/operations/debezium-server.html#_redis_stream
107-
# sink:
108-
# redis.memory.limit.mb: 100
109-
# redis.memory.threshold.percentage: 85
110-
# Uncomment the lines below for production usage with high availability (HA). When writing data
111-
# to the state database or the target with HA enabled, RDI should wait briefly for an
112-
# acknowledgment from the replica database. It should also retry a write operation after a
113-
# certain delay if the original operation times out.
114-
# redis.wait.enabled:true
115-
# redis.wait.timeout.ms:2
116-
# redis.wait.retry.enabled:true
117-
# redis.wait.retry.delay.ms:
118-
# Source specific properties - see the full list at https://debezium.io/documentation/reference/stable/connectors/
119-
# source:
120-
# snapshot.mode: initial
121-
# Quarkus framework properties - see the full list at https://quarkus.io/guides/all-config
122-
# quarkus:
123-
# banner.enabled: "false"
91+
# Additional properties for the source collector:
92+
# List of databases to include (optional).
93+
# databases:
94+
# - database1
95+
# - database2
96+
97+
# List of tables to be synced (optional).
98+
# tables:
99+
# If only one database is specified in the databases property above,
100+
# then tables can be defined without the database prefix.
101+
# <DATABASE_NAME>.<TABLE_NAME>:
102+
# List of columns to be synced (optional).
103+
# columns:
104+
# - <COLUMN_NAME>
105+
# - <COLUMN_NAME>
106+
# List of columns to be used as keys (optional).
107+
# keys:
108+
# - <COLUMN_NAME>
109+
110+
# Example: Sync specific tables.
111+
# tables:
112+
# Sync a specific table with all its columns:
113+
# redislabscdc.account: {}
114+
# Sync a specific table with selected columns:
115+
# redislabscdc.emp:
116+
# columns:
117+
# - empno
118+
# - fname
119+
# - lname
120+
121+
# Advanced collector properties (optional):
122+
# advanced:
123+
# Sink collector properties - see the full list at
124+
# https://debezium.io/documentation/reference/stable/operations/debezium-server.html#_redis_stream
125+
# sink:
126+
# Optional hard limits on memory usage of RDI streams.
127+
# redis.memory.limit.mb: 300
128+
# redis.memory.threshold.percentage: 85
129+
130+
# Uncomment for production so RDI Collector will wait on replica
131+
# when writing entries.
132+
# redis.wait.enabled: true
133+
# redis.wait.timeout.ms: 1000
134+
# redis.wait.retry.enabled: true
135+
# redis.wait.retry.delay.ms: 1000
136+
137+
# Source specific properties - see the full list at
138+
# https://debezium.io/documentation/reference/stable/connectors/
139+
# source:
140+
# snapshot.mode: initial
141+
# Uncomment if you want a snapshot to include only a subset of the rows
142+
# in a table. This property affects snapshots only.
143+
# snapshot.select.statement.overrides: <DATABASE_NAME>.<TABLE_NAME>
144+
# The specified SELECT statement determines the subset of table rows to
145+
# include in the snapshot.
146+
# snapshot.select.statement.overrides.<DATABASE_NAME>.<TABLE_NAME>: <SELECT_STATEMENT>
147+
148+
# Example: Snapshot filtering by order status.
149+
# To include only orders with non-pending status from customers.orders
150+
# table:
151+
# snapshot.select.statement.overrides: customer.orders
152+
# snapshot.select.statement.overrides.customer.orders: SELECT * FROM customers.orders WHERE status != 'pending' ORDER BY order_id DESC
153+
154+
# Quarkus framework properties - see the full list at
155+
# https://quarkus.io/guides/all-config
156+
# quarkus:
157+
# banner.enabled: "false"
158+
124159
targets:
125-
my-redis:
160+
# Redis target database connections.
161+
# The default connection must be named 'target' and is used when no
162+
# connection is specified in jobs or no jobs
163+
# are deployed. However multiple connections can be defined here and used
164+
# in the job definition output blocks:
165+
# (e.g. target1, my-cloud-redis-db2, etc.)
166+
target:
126167
connection:
127168
type: redis
128-
host: localhost
129-
port: 12000
130-
user: ${TARGET_DB_USERNAME}
169+
# Host of the Redis database to which RDI will
170+
# write the processed data.
171+
host: <REDIS_TARGET_DB_HOST> # e.g. localhost
172+
# Port for the Redis database to which RDI will
173+
# write the processed data.
174+
port: <REDIS_TARGET_DB_PORT> # e.g. 12000
175+
# User of the Redis database to which RDI will write the processed data.
176+
# Uncomment if you are not using the default user.
177+
# user: ${TARGET_DB_USERNAME}
178+
# Password for Redis target database.
131179
password: ${TARGET_DB_PASSWORD}
132-
# The names of the following properties should match the ones you used
133-
# when setting the TLS/mTLS secrets. Set only `cacert` if you are using
134-
# TLS, but set all of them if you are using mTLS:
135-
# key: ${TARGET_DB_KEY}
136-
# cert: ${TARGET_DB_CERT}
137-
# cacert: ${TARGET_DB_CACERT}
138-
# key_password: ${TARGET_DB_KEY_PASSWORD}
180+
# SSL/TLS configuration: Uncomment to enable secure connections.
181+
# key: ${TARGET_DB_KEY}
182+
# key_password: ${TARGET_DB_KEY_PASSWORD}
183+
# cert: ${TARGET_DB_CERT}
184+
# cacert: ${TARGET_DB_CACERT}
139185
processors:
140-
# Enable Debezium LOB placeholders for tables that contain large objects.
141-
# Uncomment this property (and the `processors:` section) if your tables include
142-
# Oracle large objects (BLOB, CLOB, NCLOB).
143-
# debezium_lob_encoded_placeholder: X19kZWJleml1bV91bmF2YWlsYWJsZV92YWx1ZQ==
186+
# Interval (in seconds) on which to perform retry on failure.
187+
# on_failed_retry_interval: 5
188+
# The batch size for reading data from the source database.
189+
# read_batch_size: 2000
190+
# Time (in ms) after which data will be read from stream even if
191+
# read_batch_size was not reached.
192+
# duration: 100
193+
# The batch size for writing data to the target Redis database. Should be
194+
# less than or equal to the read_batch_size.
195+
# write_batch_size: 200
196+
# Enable deduplication mechanism (default: false).
197+
# dedup: <DEDUP_ENABLED>
198+
# Max size of the deduplication set (default: 1024).
199+
# dedup_max_size: <DEDUP_MAX_SIZE>
200+
# Error handling strategy: ignore - skip, dlq - store rejected messages
201+
# in a dead letter queue
202+
# error_handling: dlq
144203
```
145204

146205
The main sections of the file configure [`sources`](#sources) and [`targets`](#targets).

0 commit comments

Comments
 (0)