Skip to content

Commit 459cd71

Browse files
subkanthisubkanthi
authored
Merge pull request #1008 from Altinity/2.6.0
2.6.0
2 parents f35514c + 0c6ae97 commit 459cd71

File tree

153 files changed

+11754
-3602
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

153 files changed

+11754
-3602
lines changed

‎.github/workflows/docker-build.yml

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,9 @@ jobs:
9999

100100
- name: Build Docker image (Lightweight)
101101
run: |
102-
docker build . --file sink-connector-lightweight/Dockerfile --build-arg DOCKER_TAG=${{ env.IMAGE_TAG }} --tag altinityinfra/clickhouse-sink-connector:${{ env.IMAGE_TAG }}-lt
102+
docker buildx create --name graviton --platform linux/arm64,linux/amd64
103+
docker buildx build . --file sink-connector-lightweight/Dockerfile --build-arg DOCKER_TAG=${{ env.IMAGE_TAG }} --tag altinityinfra/clickhouse-sink-connector:${{ env.IMAGE_TAG }}-lt --push --builder graviton --platform linux/arm64,linux/amd64
104+
docker image pull altinityinfra/clickhouse-sink-connector:${{ env.IMAGE_TAG }}-lt
103105
docker save altinityinfra/clickhouse-sink-connector:${{ env.IMAGE_TAG }}-lt | gzip > clickhouse-sink-connector_${{ env.IMAGE_TAG }}-lt.tar.gz
104106
105107
- name: Upload Docker tar (Lightweight)
@@ -109,6 +111,6 @@ jobs:
109111
name: clickhouse-sink-connector_${{ env.IMAGE_TAG }}-lt.tar.gz
110112
path: clickhouse-sink-connector_${{ env.IMAGE_TAG }}-lt.tar.gz
111113

112-
- name: Upload Docker image (Lightweight)
113-
if: ${{ env.DOCKERHUB_USERNAME != '' }}
114-
run: docker image push altinityinfra/clickhouse-sink-connector:${{ env.IMAGE_TAG }}-lt
114+
#- name: Upload Docker image (Lightweight)
115+
# if: ${{ env.DOCKERHUB_USERNAME != '' }}
116+
# run: docker image push altinityinfra/clickhouse-sink-connector:${{ env.IMAGE_TAG }}-lt

‎.github/workflows/pull-request.yml

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,16 @@ jobs:
2222
secrets: inherit
2323
with:
2424
SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-kafka
25-
26-
testflows-lightweight:
25+
testflows-lightweight-x86:
2726
needs: [build-kafka-lightweight]
2827
uses: ./.github/workflows/testflows-sink-connector-lightweight.yml
2928
secrets: inherit
3029
with:
3130
SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-lt
31+
extra_args: ""
32+
artifact_suffix: ""
3233

33-
testflows-lightweight-hikari-pool:
34+
testflows-lightweight-x86-hikari-pool:
3435
needs: [build-kafka-lightweight]
3536
uses: ./.github/workflows/testflows-sink-connector-lightweight.yml
3637
secrets: inherit
@@ -39,12 +40,30 @@ jobs:
3940
extra_args: --hikari-pool
4041
artifact_suffix: hikari-pool
4142

43+
# testflows-lightweight-arm:
44+
# needs: [build-kafka-lightweight]
45+
# uses: ./.github/workflows/testflows-sink-connector-lightweight-arm.yml
46+
# secrets: inherit
47+
# with:
48+
# SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-lt
49+
# extra_args: ""
50+
# artifact_suffix: ""
51+
52+
# testflows-lightweight-arm-hikari-pool:
53+
# needs: [build-kafka-lightweight]
54+
# uses: ./.github/workflows/testflows-sink-connector-lightweight-arm.yml
55+
# secrets: inherit
56+
# with:
57+
# SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-lt
58+
# extra_args: --hikari-pool
59+
# artifact_suffix: hikari-pool
60+
4261
java-tests-kafka:
4362
needs: [build-kafka-lightweight]
4463
uses: ./.github/workflows/sink-connector-kafka-tests.yml
4564
with:
4665
SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-kafka
47-
66+
4867
java-tests-lightweight:
4968
needs: [build-kafka-lightweight]
5069
uses: ./.github/workflows/sink-connector-lightweight-tests.yml

‎.github/workflows/testflows-sink-connector-lightweight-arm.yml

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
name: Lightweight - TestFlows Tests | ARM
2+
run-name: ${{ inputs.custom_run_name || 'Lightweight - TestFlows Tests' }}
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
SINK_CONNECTOR_IMAGE:
8+
description: "Lightweight connector docker image"
9+
required: true
10+
type: string
11+
package:
12+
description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
13+
type: string
14+
default: docker://clickhouse/clickhouse-server:23.8
15+
output_format:
16+
description: "Testflows output style."
17+
type: string
18+
default: new-fails
19+
extra_args:
20+
description: "Extra arguments for the tests."
21+
required: false
22+
type: string
23+
default: ""
24+
artifact_suffix:
25+
description: "Artifact sufix"
26+
required: false
27+
type: string
28+
default: ""
29+
secrets:
30+
DOCKERHUB_USERNAME:
31+
required: false
32+
DOCKERHUB_TOKEN:
33+
required: false
34+
AWS_ACCESS_KEY_ID:
35+
required: false
36+
AWS_SECRET_ACCESS_KEY:
37+
required: false
38+
workflow_dispatch:
39+
inputs:
40+
SINK_CONNECTOR_IMAGE:
41+
description: "Lightweight connector docker image"
42+
required: true
43+
type: string
44+
package:
45+
description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
46+
type: string
47+
default: docker://clickhouse/clickhouse-server:23.8
48+
suite:
49+
description: "Specific Suite To Run (Default * to run everything)."
50+
required: false
51+
type: string
52+
extra_args:
53+
description: "Extra arguments for the tests."
54+
required: false
55+
type: string
56+
default: ""
57+
custom_run_name:
58+
description: 'Custom run name (optional)'
59+
required: false
60+
output_format:
61+
description: "Testflows output style."
62+
type: choice
63+
options:
64+
- new-fails
65+
- nice-new-fails
66+
- brisk-new-fails
67+
- plain-new-fails
68+
- pnice-new-fails
69+
- classic
70+
- nice
71+
- fails
72+
- slick
73+
- brisk
74+
- quiet
75+
- short
76+
- manual
77+
- dots
78+
- progress
79+
- raw
80+
81+
env:
82+
SINK_CONNECTOR_IMAGE: ${{ inputs.SINK_CONNECTOR_IMAGE }}
83+
84+
jobs:
85+
testflows-lightweight:
86+
runs-on: [self-hosted, on-demand, type-cax41, image-arm-app-docker-ce]
87+
88+
steps:
89+
- uses: actions/checkout@v2
90+
91+
- uses: actions/download-artifact@v4
92+
if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
93+
with:
94+
name: clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz
95+
96+
- name: Load Docker image
97+
if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
98+
run: |
99+
docker load < clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz
100+
docker image ls
101+
102+
- name: Runner ssh command
103+
working-directory: sink-connector/tests/integration
104+
run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
105+
106+
- name: Create a virtual environment
107+
run: |
108+
echo "Install Python modules..."
109+
sudo apt-get clean
110+
sudo apt-get update
111+
sudo apt-get install -y python3.12-venv
112+
113+
echo "Create and activate Python virtual environment..."
114+
python3 -m venv venv
115+
source venv/bin/activate
116+
echo PATH=$PATH >> $GITHUB_ENV
117+
118+
- name: Install all dependencies
119+
working-directory: sink-connector-lightweight/tests/integration
120+
run: pip3 install -r requirements.txt
121+
122+
- name: Get current date
123+
id: date
124+
run: echo "date=$(date +'%Y-%m-%d_%H%M%S')" >> $GITHUB_OUTPUT
125+
126+
- name: Add ~./local/bin to the PATH
127+
if: always()
128+
working-directory: sink-connector-lightweight/tests/integration
129+
run: echo ~/.local/bin >> $GITHUB_PATH
130+
131+
- name: Run testflows tests
132+
working-directory: sink-connector-lightweight/tests/integration
133+
run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.suite != '' && inputs.suite || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end ${{ inputs.extra_args != '' && inputs.extra_args || '' }} --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
134+
135+
- name: Create tfs results report
136+
if: always()
137+
working-directory: sink-connector-lightweight/tests/integration/logs
138+
run: cat raw.log | tfs report results | tfs document convert > report.html
139+
140+
- name: Create tfs coverage report
141+
if: always()
142+
working-directory: sink-connector-lightweight/tests/integration/logs
143+
run: cat raw.log | tfs report coverage ../requirements/requirements.py | tfs document convert > coverage.html
144+
145+
- name: Upload artifacts to Altinity Test Reports S3 bucket
146+
if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
147+
working-directory: sink-connector-lightweight/tests/integration/logs
148+
env:
149+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
150+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
151+
AWS_DEFAULT_REGION: 'eu-west-2'
152+
run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink_lw/ --recursive --exclude "*" --include "*.log" --include "*.html"
153+
154+
- uses: actions/upload-artifact@v4
155+
if: always()
156+
with:
157+
name: testflows-sink-connector-lightweight-artefacts-arm${{ inputs.artifact_suffix != '' && '-' || '' }}${{ inputs.artifact_suffix }}
158+
path: |
159+
sink-connector-lightweight/tests/integration/logs/*.log
160+
sink-connector-lightweight/tests/integration/env/auto/configs/*.yml
161+
if-no-files-found: error
162+
retention-days: 60
163+
testflows-lightweight-replicated:
164+
runs-on: [self-hosted, on-demand, type-cax41, image-arm-app-docker-ce]
165+
166+
steps:
167+
- uses: actions/checkout@v2
168+
169+
- uses: actions/download-artifact@v4
170+
if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
171+
with:
172+
name: clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz
173+
174+
- name: Load Docker image
175+
if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
176+
run: |
177+
docker load < clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz
178+
docker image ls
179+
180+
- name: Runner ssh command
181+
working-directory: sink-connector/tests/integration
182+
run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
183+
184+
- name: Create a virtual environment
185+
run: |
186+
echo "Install Python modules..."
187+
sudo apt-get clean
188+
sudo apt-get update
189+
sudo apt-get install -y python3.12-venv
190+
191+
echo "Create and activate Python virtual environment..."
192+
python3 -m venv venv
193+
source venv/bin/activate
194+
echo PATH=$PATH >> $GITHUB_ENV
195+
196+
- name: Install all dependencies
197+
working-directory: sink-connector-lightweight/tests/integration
198+
run: pip3 install -r requirements.txt
199+
200+
- name: Get current date
201+
id: date
202+
run: echo "date=$(date +'%Y-%m-%d_%H%M%S')" >> $GITHUB_OUTPUT
203+
204+
- name: Add ~./local/bin to the PATH
205+
if: always()
206+
working-directory: sink-connector-lightweight/tests/integration
207+
run: echo ~/.local/bin >> $GITHUB_PATH
208+
209+
- name: Run testflows tests
210+
working-directory: sink-connector-lightweight/tests/integration
211+
run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.suite != '' && inputs.suite || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end ${{ inputs.extra_args != '' && inputs.extra_args || '' }} --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
212+
213+
- name: Create tfs results report
214+
if: always()
215+
working-directory: sink-connector-lightweight/tests/integration/logs
216+
run: cat raw.log | tfs report results | tfs document convert > report.html
217+
218+
- name: Create tfs coverage report
219+
if: always()
220+
working-directory: sink-connector-lightweight/tests/integration/logs
221+
run: cat raw.log | tfs report coverage ../requirements/requirements.py | tfs document convert > coverage.html
222+
223+
- name: Upload artifacts to Altinity Test Reports S3 bucket
224+
if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }}
225+
working-directory: sink-connector-lightweight/tests/integration/logs
226+
env:
227+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
228+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
229+
AWS_DEFAULT_REGION: 'eu-west-2'
230+
run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink_lw/ --recursive --exclude "*" --include "*.log" --include "*.html"
231+
232+
- uses: actions/upload-artifact@v4
233+
if: always()
234+
with:
235+
name: testflows-sink-connector-lightweight-replicated-artefacts-arm${{ inputs.artifact_suffix != '' && '-' || '' }}${{ inputs.artifact_suffix }}
236+
path: |
237+
sink-connector-lightweight/tests/integration/logs/*.log
238+
sink-connector-lightweight/tests/integration/env/auto_replicated/configs/*.yml
239+
if-no-files-found: error
240+
retention-days: 60

‎.github/workflows/testflows-sink-connector-lightweight.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Lightweight - TestFlows Tests
1+
name: Lightweight - TestFlows Tests | x86
22
run-name: ${{ inputs.custom_run_name || 'Lightweight - TestFlows Tests' }}
33

44
on:

‎doc/getting_started_jar_kafka.md

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
- The following are the steps to install the Sink Connector Kafka JAR
2+
3+
4+
**Download or Build the Sink Connector Kafka JAR** \
5+
The Sink Connector Kafka JAR is available
6+
in the releases artifacts or can be built from the source code.
7+
```cd sink-connector
8+
mvn clean package
9+
```
10+
The JAR file is available in the target directory.
11+
12+
**Identify or Create a Plugins Directory** \
13+
Kafka Connect loads connectors from the plugin.path defined in its configuration.
14+
If you haven't configured a plugins directory, create one. For example:
15+
```
16+
/usr/local/kafka/connectors/
17+
```
18+
**Place the JAR File in the Plugins Directory** \
19+
Copy the connector JAR file and its dependencies into a subdirectory inside the plugin path:
20+
```
21+
/usr/local/kafka/connectors/my-sink-connector/
22+
```
23+
**Configure Kafka Connect to Use the Plugins Directory** \\
24+
Edit your Kafka Connect worker properties file (connect-distributed.properties or connect-standalone.properties):
25+
properties
26+
```
27+
plugin.path=/usr/local/kafka/connectors/
28+
```
29+
**Restart Kafka Connect**
30+
Restart Kafka Connect to load the new connector:\
31+
32+
*For distributed mode:*
33+
34+
```
35+
bin/connect-distributed.sh config/connect-distributed.properties
36+
```
37+
38+
*For standalone mode:*
39+
```
40+
bin/connect-standalone.sh config/connect-standalone.properties <connector-config-file>
41+
```
42+
**Verify Connector Installation**
43+
44+
Use the Kafka Connect REST API to verify that the connector is installed:
45+
```
46+
curl -sS localhost:8083/connector-plugins | jq .
47+
```
48+
This command should list your newly installed connector.
49+
50+
**Configure and Start the Sink Connector**
51+
52+
Follow the steps below to configure and start the Sink Connector:
53+
54+
[Kafka Quick Start Guide](quickstart_kafka.md)
55+
56+
**Monitor the Connector**
57+
Use the REST API to monitor the status of your connector:
58+
```
59+
curl -sS localhost:8083/connectors/my-sink-connector/status | jq .
60+
```

0 commit comments

Comments
 (0)