Skip to content

Commit fd6d61f

Browse files
dervoetirazvan
dervoeti
and
razvan
authored
feat: custom version for Kafka (#1171)
* feat: support custom product versions * chore: changelog * feat: move version patched to separate patch files * fix: Hive 4.0.0 build (#1182) --------- Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
1 parent df80332 commit fd6d61f

File tree

6 files changed

+534
-10
lines changed

6 files changed

+534
-10
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ All notable changes to this project will be documented in this file.
128128
- nifi: Remove `2.2.0` ([#1114]).
129129
- kafka: Remove `3.7.1` and `3.8.0` ([#1117]).
130130
- spark-connect-client: Remove `3.5.5` ([#1142]).
131+
- kafka: Enable custom versions ([#1171]).
131132
- omid: Enable custom versions ([#1174]).
132133

133134
[nifi-iceberg-bundle]: https://github.com/stackabletech/nifi-iceberg-bundle
@@ -194,6 +195,7 @@ All notable changes to this project will be documented in this file.
194195
[#1165]: https://github.com/stackabletech/docker-images/pull/1165
195196
[#1168]: https://github.com/stackabletech/docker-images/pull/1168
196197
[#1170]: https://github.com/stackabletech/docker-images/pull/1170
198+
[#1171]: https://github.com/stackabletech/docker-images/pull/1171
197199
[#1173]: https://github.com/stackabletech/docker-images/pull/1173
198200
[#1174]: https://github.com/stackabletech/docker-images/pull/1174
199201
[#1179]: https://github.com/stackabletech/docker-images/pull/1179

‎kafka/Dockerfile

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ FROM stackable/image/kafka/kcat AS kcat
66
FROM stackable/image/java-devel AS kafka-builder
77

88
ARG PRODUCT
9+
ARG RELEASE
910
ARG SCALA
1011
ARG OPA_AUTHORIZER
1112
ARG JMX_EXPORTER
@@ -21,22 +22,27 @@ COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackab
2122
RUN <<EOF
2223
cd "$(/stackable/patchable --images-repo-root=src checkout kafka ${PRODUCT})"
2324

25+
NEW_VERSION="${PRODUCT}-stackable${RELEASE}"
26+
27+
# The comment in gradle.properties mentions to update the version in all of these files
28+
find . -type f -print0 | xargs -0 sed -i "s/\-stackable0\.0\.0\-dev/-stackable${RELEASE}/g"
29+
2430
# Create snapshot of the source code including custom patches
25-
tar -czf /stackable/kafka-${PRODUCT}-src.tar.gz .
31+
tar -czf /stackable/kafka-${NEW_VERSION}-src.tar.gz .
2632

2733
# TODO: Try to install gradle via package manager (if possible) instead of fetching it from the internet
2834
# We don't specify "-x test" to skip the tests, as we might bump some Kafka internal dependencies in the future and
2935
# it's a good idea to run the tests in this case.
3036
./gradlew clean releaseTarGz
3137
./gradlew cyclonedxBom
32-
tar -xf core/build/distributions/kafka_${SCALA}-${PRODUCT}.tgz -C /stackable
33-
cp build/reports/bom.json /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json
34-
rm -rf /stackable/kafka_${SCALA}-${PRODUCT}/site-docs/
38+
tar -xf core/build/distributions/kafka_${SCALA}-${NEW_VERSION}.tgz -C /stackable
39+
cp build/reports/bom.json /stackable/kafka_${SCALA}-${NEW_VERSION}.cdx.json
40+
rm -rf /stackable/kafka_${SCALA}-${NEW_VERSION}/site-docs/
3541
(cd .. && rm -rf ${PRODUCT})
3642

3743
# TODO (@NickLarsenNZ): Compile from source: https://github.com/StyraInc/opa-kafka-plugin
3844
curl https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
39-
-o /stackable/kafka_${SCALA}-${PRODUCT}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
45+
-o /stackable/kafka_${SCALA}-${NEW_VERSION}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
4046

4147
# JMX exporter
4248
curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
@@ -65,9 +71,9 @@ LABEL \
6571
summary="The Stackable image for Apache Kafka." \
6672
description="This image is deployed by the Stackable Operator for Apache Kafka."
6773

68-
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
69-
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}/kafka_${SCALA}-${PRODUCT}.cdx.json
70-
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka-${PRODUCT}-src.tar.gz /stackable
74+
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE} /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}
75+
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}.cdx.json
76+
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka-${PRODUCT}-stackable${RELEASE}-src.tar.gz /stackable
7177
COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/
7278
COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/bin/kcat-${KAFKA_KCAT}
7379
COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}-src.tar.gz /stackable
@@ -94,13 +100,13 @@ chown -h ${STACKABLE_USER_UID}:0 /stackable/bin/kcat
94100
# kcat was located in /stackable/kcat - legacy
95101
ln -s /stackable/bin/kcat /stackable/kcat
96102
chown -h ${STACKABLE_USER_UID}:0 /stackable/kcat
97-
ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
103+
ln -s /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE} /stackable/kafka
98104
chown -h ${STACKABLE_USER_UID}:0 /stackable/kafka
99105

100106
# fix missing permissions
101107
chmod g=u /stackable/bin
102108
chmod g=u /stackable/jmx
103-
chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}
109+
chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}
104110
chmod g=u /stackable/*-src.tar.gz
105111
EOF
106112

‎kafka/stackable/patches/3.7.2/0003-Build-custom-Stackable-version.patch

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
From 508c8053273b091b599c5ab9804cee877483703f Mon Sep 17 00:00:00 2001
2+
From: dervoeti <lukas.krug@stackable.tech>
3+
Date: Wed, 18 Jun 2025 12:15:17 +0200
4+
Subject: Build custom Stackable version
5+
6+
---
7+
docs/js/templateData.js | 2 +-
8+
gradle.properties | 2 +-
9+
kafka-merge-pr.py | 2 +-
10+
streams/quickstart/java/pom.xml | 2 +-
11+
.../java/src/main/resources/archetype-resources/pom.xml | 2 +-
12+
streams/quickstart/pom.xml | 2 +-
13+
tests/kafkatest/__init__.py | 2 +-
14+
tests/kafkatest/version.py | 2 +-
15+
8 files changed, 8 insertions(+), 8 deletions(-)
16+
17+
diff --git a/docs/js/templateData.js b/docs/js/templateData.js
18+
index 7700e24003..8affa51e55 100644
19+
--- a/docs/js/templateData.js
20+
+++ b/docs/js/templateData.js
21+
@@ -19,6 +19,6 @@ limitations under the License.
22+
var context={
23+
"version": "37",
24+
"dotVersion": "3.7",
25+
- "fullDotVersion": "3.7.2",
26+
+ "fullDotVersion": "3.7.2-stackable0.0.0-dev",
27+
"scalaVersion": "2.13"
28+
};
29+
diff --git a/gradle.properties b/gradle.properties
30+
index 08ae0ded1e..779dcfb829 100644
31+
--- a/gradle.properties
32+
+++ b/gradle.properties
33+
@@ -23,7 +23,7 @@ group=org.apache.kafka
34+
# - streams/quickstart/pom.xml
35+
# - streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
36+
# - streams/quickstart/java/pom.xml
37+
-version=3.7.2
38+
+version=3.7.2-stackable0.0.0-dev
39+
scalaVersion=2.13.12
40+
# Adding swaggerVersion in gradle.properties to have a single version in place for swagger
41+
# New version of Swagger 2.2.14 requires minimum JDK 11.
42+
diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py
43+
index c16c811178..41b6794b8c 100755
44+
--- a/kafka-merge-pr.py
45+
+++ b/kafka-merge-pr.py
46+
@@ -70,7 +70,7 @@ TEMP_BRANCH_PREFIX = "PR_TOOL"
47+
48+
DEV_BRANCH_NAME = "trunk"
49+
50+
-DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.7.2-SNAPSHOT")
51+
+DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.7.2-stackable0.0.0-dev-SNAPSHOT")
52+
53+
ORIGINAL_HEAD = ""
54+
55+
diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
56+
index 1335f0d58b..e9e8cfe1ec 100644
57+
--- a/streams/quickstart/java/pom.xml
58+
+++ b/streams/quickstart/java/pom.xml
59+
@@ -26,7 +26,7 @@
60+
<parent>
61+
<groupId>org.apache.kafka</groupId>
62+
<artifactId>streams-quickstart</artifactId>
63+
- <version>3.7.2</version>
64+
+ <version>3.7.2-stackable0.0.0-dev</version>
65+
<relativePath>..</relativePath>
66+
</parent>
67+
68+
diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
69+
index 445fab4b83..21cb230fc4 100644
70+
--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
71+
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
72+
@@ -29,7 +29,7 @@
73+
74+
<properties>
75+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
76+
- <kafka.version>3.7.2</kafka.version>
77+
+ <kafka.version>3.7.2-stackable0.0.0-dev</kafka.version>
78+
<slf4j.version>1.7.36</slf4j.version>
79+
</properties>
80+
81+
diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
82+
index 94ac32fcac..2cc1fcd600 100644
83+
--- a/streams/quickstart/pom.xml
84+
+++ b/streams/quickstart/pom.xml
85+
@@ -22,7 +22,7 @@
86+
<groupId>org.apache.kafka</groupId>
87+
<artifactId>streams-quickstart</artifactId>
88+
<packaging>pom</packaging>
89+
- <version>3.7.2</version>
90+
+ <version>3.7.2-stackable0.0.0-dev</version>
91+
92+
<name>Kafka Streams :: Quickstart</name>
93+
94+
diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
95+
index 1c66c2bcfd..9274ce1fd0 100644
96+
--- a/tests/kafkatest/__init__.py
97+
+++ b/tests/kafkatest/__init__.py
98+
@@ -22,4 +22,4 @@
99+
# Instead, in development branches, the version should have a suffix of the form ".devN"
100+
#
101+
# For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0"
102+
-__version__ = '3.7.2'
103+
+__version__ = '3.7.2-stackable0.0.0-dev'
104+
diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
105+
index bed7cffac2..55dc3cbc7d 100644
106+
--- a/tests/kafkatest/version.py
107+
+++ b/tests/kafkatest/version.py
108+
@@ -122,7 +122,7 @@ def get_version(node=None):
109+
return DEV_BRANCH
110+
111+
DEV_BRANCH = KafkaVersion("dev")
112+
-DEV_VERSION = KafkaVersion("3.7.2-SNAPSHOT")
113+
+DEV_VERSION = KafkaVersion("3.7.2-stackable0.0.0-dev")
114+
115+
LATEST_METADATA_VERSION = "3.7"
116+

‎kafka/stackable/patches/3.9.0/0003-Build-custom-Stackable-version.patch

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
From de4d98f9cd65d93bb4bb061c233f6647efc951f4 Mon Sep 17 00:00:00 2001
2+
From: dervoeti <lukas.krug@stackable.tech>
3+
Date: Wed, 18 Jun 2025 12:16:46 +0200
4+
Subject: Build custom Stackable version
5+
6+
---
7+
docs/js/templateData.js | 2 +-
8+
docs/streams/upgrade-guide.html | 2 +-
9+
gradle.properties | 2 +-
10+
kafka-merge-pr.py | 2 +-
11+
streams/quickstart/java/pom.xml | 2 +-
12+
.../java/src/main/resources/archetype-resources/pom.xml | 2 +-
13+
streams/quickstart/pom.xml | 2 +-
14+
tests/kafkatest/__init__.py | 2 +-
15+
tests/kafkatest/version.py | 4 ++--
16+
9 files changed, 10 insertions(+), 10 deletions(-)
17+
18+
diff --git a/docs/js/templateData.js b/docs/js/templateData.js
19+
index 13d352fc70..42d3549382 100644
20+
--- a/docs/js/templateData.js
21+
+++ b/docs/js/templateData.js
22+
@@ -19,6 +19,6 @@ limitations under the License.
23+
var context={
24+
"version": "39",
25+
"dotVersion": "3.9",
26+
- "fullDotVersion": "3.9.0",
27+
+ "fullDotVersion": "3.9.0-stackable0.0.0-dev",
28+
"scalaVersion": "2.13"
29+
};
30+
diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html
31+
index 6a30e6671b..6d2ecb81aa 100644
32+
--- a/docs/streams/upgrade-guide.html
33+
+++ b/docs/streams/upgrade-guide.html
34+
@@ -133,7 +133,7 @@
35+
More details about the new config <code>StreamsConfig#TOPOLOGY_OPTIMIZATION</code> can be found in <a href="https://cwiki.apache.org/confluence/display/KAFKA/KIP-295%3A+Add+Streams+Configuration+Allowing+for+Optional+Topology+Optimization">KIP-295</a>.
36+
</p>
37+
38+
- <h3><a id="streams_api_changes_390" href="#streams_api_changes_390">Streams API changes in 3.9.0</a></h3>
39+
+ <h3><a id="streams_api_changes_390" href="#streams_api_changes_390">Streams API changes in 3.9.0-stackable0.0.0-dev</a></h3>
40+
41+
<p>
42+
The introduction of <a href="https://cwiki.apache.org/confluence/display/KAFKA/KIP-1033%3A+Add+Kafka+Streams+exception+handler+for+exceptions+occurring+during+processing">KIP-1033</a>
43+
diff --git a/gradle.properties b/gradle.properties
44+
index 0c14658608..b79ee9f0cf 100644
45+
--- a/gradle.properties
46+
+++ b/gradle.properties
47+
@@ -23,7 +23,7 @@ group=org.apache.kafka
48+
# - streams/quickstart/pom.xml
49+
# - streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
50+
# - streams/quickstart/java/pom.xml
51+
-version=3.9.0
52+
+version=3.9.0-stackable0.0.0-dev
53+
scalaVersion=2.13.14
54+
# Adding swaggerVersion in gradle.properties to have a single version in place for swagger
55+
# New version of Swagger 2.2.14 requires minimum JDK 11.
56+
diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py
57+
index a775a38435..4a7fbac34e 100755
58+
--- a/kafka-merge-pr.py
59+
+++ b/kafka-merge-pr.py
60+
@@ -70,7 +70,7 @@ TEMP_BRANCH_PREFIX = "PR_TOOL"
61+
62+
DEV_BRANCH_NAME = "trunk"
63+
64+
-DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.9.0")
65+
+DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.9.0-stackable0.0.0-dev")
66+
67+
ORIGINAL_HEAD = ""
68+
69+
diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
70+
index 78e1a70fa7..8698b4eb35 100644
71+
--- a/streams/quickstart/java/pom.xml
72+
+++ b/streams/quickstart/java/pom.xml
73+
@@ -26,7 +26,7 @@
74+
<parent>
75+
<groupId>org.apache.kafka</groupId>
76+
<artifactId>streams-quickstart</artifactId>
77+
- <version>3.9.0</version>
78+
+ <version>3.9.0-stackable0.0.0-dev</version>
79+
<relativePath>..</relativePath>
80+
</parent>
81+
82+
diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
83+
index 2bc235eb16..90d566d1a0 100644
84+
--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
85+
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
86+
@@ -29,7 +29,7 @@
87+
88+
<properties>
89+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
90+
- <kafka.version>3.9.0</kafka.version>
91+
+ <kafka.version>3.9.0-stackable0.0.0-dev</kafka.version>
92+
<slf4j.version>1.7.36</slf4j.version>
93+
</properties>
94+
95+
diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
96+
index b5101583ee..3c08985e2a 100644
97+
--- a/streams/quickstart/pom.xml
98+
+++ b/streams/quickstart/pom.xml
99+
@@ -22,7 +22,7 @@
100+
<groupId>org.apache.kafka</groupId>
101+
<artifactId>streams-quickstart</artifactId>
102+
<packaging>pom</packaging>
103+
- <version>3.9.0</version>
104+
+ <version>3.9.0-stackable0.0.0-dev</version>
105+
106+
<name>Kafka Streams :: Quickstart</name>
107+
108+
diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
109+
index a190e05067..0aa8fa7095 100644
110+
--- a/tests/kafkatest/__init__.py
111+
+++ b/tests/kafkatest/__init__.py
112+
@@ -22,4 +22,4 @@
113+
# Instead, in development branches, the version should have a suffix of the form ".devN"
114+
#
115+
# For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0"
116+
-__version__ = '3.9.0'
117+
+__version__ = '3.9.0-stackable0.0.0-dev'
118+
diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
119+
index 4825a46039..06fd3029f9 100644
120+
--- a/tests/kafkatest/version.py
121+
+++ b/tests/kafkatest/version.py
122+
@@ -122,7 +122,7 @@ def get_version(node=None):
123+
return DEV_BRANCH
124+
125+
DEV_BRANCH = KafkaVersion("dev")
126+
-DEV_VERSION = KafkaVersion("3.9.0-SNAPSHOT")
127+
+DEV_VERSION = KafkaVersion("3.9.0-stackable0.0.0-dev")
128+
129+
# This should match the LATEST_PRODUCTION version defined in MetadataVersion.java
130+
LATEST_STABLE_METADATA_VERSION = "3.9-IV0"
131+
@@ -276,5 +276,5 @@ V_3_8_0 = KafkaVersion("3.8.0")
132+
LATEST_3_8 = V_3_8_0
133+
134+
# 3.9.x version
135+
-V_3_9_0 = KafkaVersion("3.9.0")
136+
+V_3_9_0 = KafkaVersion("3.9.0-stackable0.0.0-dev")
137+
LATEST_3_9 = V_3_9_0

0 commit comments

Comments
 (0)