Add Kafka 3.4.0 to 0.33 x branch and prepare for the 0.33.2 release (#8109)

Signed-off-by: Jakub Scholz <www@scholzj.com>

Author: scholzj

.checksums

@@ -6,20 +6,20 @@
 # if this checksum has changed as part of any non-release specific changes, please apply your changes to the
 #   development version of the helm charts in ./packaging/helm-charts
 ### IMPORTANT ###
-HELM_CHART_CHECKSUM="81a667ae7c51c9bac9bff1a8390bcd2a26ef42ed  -"
+HELM_CHART_CHECKSUM="9c360dbc87edc9f202f4a474ea054a8d381800e7  -"
 
 ### IMPORTANT ###
 # if the below line has changed, this means the ./install directory has changed
 #   the checksum and ./install directory should only be modified on official releases as part of a release
 # if this checksum has changed as part of any non-release specific changes, please apply your changes to the
 #   development version of the helm charts in ./packaging/install
 ### IMPORTANT ###
-INSTALL_CHECKSUM="907385a566e83a4c57e5ab6bdeac5773e77845bd  -"
+INSTALL_CHECKSUM="a69e0450f7de8e5af0710e3d2ac9ad78d97cf0e3  -"
 
 ### IMPORTANT ###
 # if the below line has changed, this means the ./examples directory has changed
 #   the checksum and ./examples directory should only be modified on official releases as part of a release
 # if this checksum has changed as part of any non-release specific changes, please apply your changes to the
 #   development version of the helm charts in ./packaging/examples
 ### IMPORTANT ###
-EXAMPLES_CHECKSUM="a2dc914a1941c7f6c9bab04d7ca01a8c1520e757  -"
+EXAMPLES_CHECKSUM="65ddcbf2f091cc451f1e11473a42f69fcd40d8fd  -"

api/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.strimzi</groupId>
         <artifactId>strimzi</artifactId>
-        <version>0.33.1</version>
+        <version>0.33.2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>api</artifactId>

certificate-manager/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>strimzi</artifactId>
     <groupId>io.strimzi</groupId>
-    <version>0.33.1</version>
+    <version>0.33.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>certificate-manager</artifactId>

cluster-operator/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.strimzi</groupId>
         <artifactId>strimzi</artifactId>
-        <version>0.33.1</version>
+        <version>0.33.2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>cluster-operator</artifactId>

cluster-operator/src/test/java/io/strimzi/operator/cluster/KafkaVersionTestUtils.java

@@ -20,23 +20,23 @@ public class KafkaVersionTestUtils {
 
     private static final Set<String> SUPPORTED_VERSIONS = new KafkaVersion.Lookup(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()).supportedVersions();
 
-    public static final String LATEST_KAFKA_VERSION = "3.3.2";
-    public static final String LATEST_FORMAT_VERSION = "3.3";
-    public static final String LATEST_PROTOCOL_VERSION = "3.3";
+    public static final String LATEST_KAFKA_VERSION = "3.4.0";
+    public static final String LATEST_FORMAT_VERSION = "3.4";
+    public static final String LATEST_PROTOCOL_VERSION = "3.4";
     public static final String LATEST_ZOOKEEPER_VERSION = "3.6.3";
     public static final String LATEST_CHECKSUM = "ABCD1234";
-    public static final String LATEST_THIRD_PARTY_VERSION = "3.3.x";
+    public static final String LATEST_THIRD_PARTY_VERSION = "3.4.x";
     public static final String LATEST_KAFKA_IMAGE = KAFKA_IMAGE_STR + LATEST_KAFKA_VERSION;
     public static final String LATEST_KAFKA_CONNECT_IMAGE = KAFKA_CONNECT_IMAGE_STR + LATEST_KAFKA_VERSION;
     public static final String LATEST_KAFKA_MIRROR_MAKER_IMAGE = KAFKA_MIRROR_MAKER_IMAGE_STR + LATEST_KAFKA_VERSION;
     public static final String LATEST_KAFKA_MIRROR_MAKER_2_IMAGE = KAFKA_MIRROR_MAKER_2_IMAGE_STR + LATEST_KAFKA_VERSION;
 
-    public static final String PREVIOUS_KAFKA_VERSION = "3.2.1";
-    public static final String PREVIOUS_FORMAT_VERSION = "3.2";
-    public static final String PREVIOUS_PROTOCOL_VERSION = "3.2";
+    public static final String PREVIOUS_KAFKA_VERSION = "3.3.2";
+    public static final String PREVIOUS_FORMAT_VERSION = "3.3";
+    public static final String PREVIOUS_PROTOCOL_VERSION = "3.3";
     public static final String PREVIOUS_ZOOKEEPER_VERSION = "3.6.3";
     public static final String PREVIOUS_CHECKSUM = "ABCD1234";
-    public static final String PREVIOUS_THIRD_PARTY_VERSION = "3.2.x";
+    public static final String PREVIOUS_THIRD_PARTY_VERSION = "3.3.x";
     public static final String PREVIOUS_KAFKA_IMAGE = KAFKA_IMAGE_STR + PREVIOUS_KAFKA_VERSION;
     public static final String PREVIOUS_KAFKA_CONNECT_IMAGE = KAFKA_CONNECT_IMAGE_STR + PREVIOUS_KAFKA_VERSION;
     public static final String PREVIOUS_KAFKA_MIRROR_MAKER_IMAGE = KAFKA_MIRROR_MAKER_IMAGE_STR + PREVIOUS_KAFKA_VERSION;

cluster-operator/src/test/java/io/strimzi/operator/cluster/model/KafkaConfigurationTests.java

@@ -89,7 +89,7 @@ public void passwordType() {
     @ParallelTest
     public void invalidVersion() {
         assertConfigError("inter.broker.protocol.version", "dclncswn",
-                "inter.broker.protocol.version has value 'dclncswn' which does not match the required pattern: \\Q0.8.0\\E(\\.[0-9]+)*|\\Q0.8.0\\E|\\Q0.8.1\\E(\\.[0-9]+)*|\\Q0.8.1\\E|\\Q0.8.2\\E(\\.[0-9]+)*|\\Q0.8.2\\E|\\Q0.9.0\\E(\\.[0-9]+)*|\\Q0.9.0\\E|\\Q0.10.0\\E(\\.[0-9]+)*|\\Q0.10.0-IV0\\E|\\Q0.10.0-IV1\\E|\\Q0.10.1\\E(\\.[0-9]+)*|\\Q0.10.1-IV0\\E|\\Q0.10.1-IV1\\E|\\Q0.10.1-IV2\\E|\\Q0.10.2\\E(\\.[0-9]+)*|\\Q0.10.2-IV0\\E|\\Q0.11.0\\E(\\.[0-9]+)*|\\Q0.11.0-IV0\\E|\\Q0.11.0-IV1\\E|\\Q0.11.0-IV2\\E|\\Q1.0\\E(\\.[0-9]+)*|\\Q1.0-IV0\\E|\\Q1.1\\E(\\.[0-9]+)*|\\Q1.1-IV0\\E|\\Q2.0\\E(\\.[0-9]+)*|\\Q2.0-IV0\\E|\\Q2.0-IV1\\E|\\Q2.1\\E(\\.[0-9]+)*|\\Q2.1-IV0\\E|\\Q2.1-IV1\\E|\\Q2.1-IV2\\E|\\Q2.2\\E(\\.[0-9]+)*|\\Q2.2-IV0\\E|\\Q2.2-IV1\\E|\\Q2.3\\E(\\.[0-9]+)*|\\Q2.3-IV0\\E|\\Q2.3-IV1\\E|\\Q2.4\\E(\\.[0-9]+)*|\\Q2.4-IV0\\E|\\Q2.4-IV1\\E|\\Q2.5\\E(\\.[0-9]+)*|\\Q2.5-IV0\\E|\\Q2.6\\E(\\.[0-9]+)*|\\Q2.6-IV0\\E|\\Q2.7\\E(\\.[0-9]+)*|\\Q2.7-IV0\\E|\\Q2.7-IV1\\E|\\Q2.7-IV2\\E|\\Q2.8\\E(\\.[0-9]+)*|\\Q2.8-IV0\\E|\\Q2.8-IV1\\E|\\Q3.0\\E(\\.[0-9]+)*|\\Q3.0-IV0\\E|\\Q3.0-IV1\\E|\\Q3.1\\E(\\.[0-9]+)*|\\Q3.1-IV0\\E|\\Q3.2\\E(\\.[0-9]+)*|\\Q3.2-IV0\\E|\\Q3.3\\E(\\.[0-9]+)*|\\Q3.3-IV0\\E|\\Q3.3-IV1\\E|\\Q3.3-IV2\\E|\\Q3.3-IV3\\E");
+                "inter.broker.protocol.version has value 'dclncswn' which does not match the required pattern: \\Q0.8.0\\E(\\.[0-9]+)*|\\Q0.8.0\\E|\\Q0.8.1\\E(\\.[0-9]+)*|\\Q0.8.1\\E|\\Q0.8.2\\E(\\.[0-9]+)*|\\Q0.8.2\\E|\\Q0.9.0\\E(\\.[0-9]+)*|\\Q0.9.0\\E|\\Q0.10.0\\E(\\.[0-9]+)*|\\Q0.10.0-IV0\\E|\\Q0.10.0-IV1\\E|\\Q0.10.1\\E(\\.[0-9]+)*|\\Q0.10.1-IV0\\E|\\Q0.10.1-IV1\\E|\\Q0.10.1-IV2\\E|\\Q0.10.2\\E(\\.[0-9]+)*|\\Q0.10.2-IV0\\E|\\Q0.11.0\\E(\\.[0-9]+)*|\\Q0.11.0-IV0\\E|\\Q0.11.0-IV1\\E|\\Q0.11.0-IV2\\E|\\Q1.0\\E(\\.[0-9]+)*|\\Q1.0-IV0\\E|\\Q1.1\\E(\\.[0-9]+)*|\\Q1.1-IV0\\E|\\Q2.0\\E(\\.[0-9]+)*|\\Q2.0-IV0\\E|\\Q2.0-IV1\\E|\\Q2.1\\E(\\.[0-9]+)*|\\Q2.1-IV0\\E|\\Q2.1-IV1\\E|\\Q2.1-IV2\\E|\\Q2.2\\E(\\.[0-9]+)*|\\Q2.2-IV0\\E|\\Q2.2-IV1\\E|\\Q2.3\\E(\\.[0-9]+)*|\\Q2.3-IV0\\E|\\Q2.3-IV1\\E|\\Q2.4\\E(\\.[0-9]+)*|\\Q2.4-IV0\\E|\\Q2.4-IV1\\E|\\Q2.5\\E(\\.[0-9]+)*|\\Q2.5-IV0\\E|\\Q2.6\\E(\\.[0-9]+)*|\\Q2.6-IV0\\E|\\Q2.7\\E(\\.[0-9]+)*|\\Q2.7-IV0\\E|\\Q2.7-IV1\\E|\\Q2.7-IV2\\E|\\Q2.8\\E(\\.[0-9]+)*|\\Q2.8-IV0\\E|\\Q2.8-IV1\\E|\\Q3.0\\E(\\.[0-9]+)*|\\Q3.0-IV0\\E|\\Q3.0-IV1\\E|\\Q3.1\\E(\\.[0-9]+)*|\\Q3.1-IV0\\E|\\Q3.2\\E(\\.[0-9]+)*|\\Q3.2-IV0\\E|\\Q3.3\\E(\\.[0-9]+)*|\\Q3.3-IV0\\E|\\Q3.3-IV1\\E|\\Q3.3-IV2\\E|\\Q3.3-IV3\\E|\\Q3.4\\E(\\.[0-9]+)*|\\Q3.4-IV0\\E");
     }
 
     @ParallelTest

config-model-generator/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>strimzi</artifactId>
         <groupId>io.strimzi</groupId>
-        <version>0.33.1</version>
+        <version>0.33.2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

config-model/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>strimzi</artifactId>
         <groupId>io.strimzi</groupId>
-        <version>0.33.1</version>
+        <version>0.33.2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

crd-annotations/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>strimzi</artifactId>
         <groupId>io.strimzi</groupId>
-        <version>0.33.1</version>
+        <version>0.33.2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

crd-generator/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>strimzi</artifactId>
     <groupId>io.strimzi</groupId>
-    <version>0.33.1</version>
+    <version>0.33.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>crd-generator</artifactId>

development-docs/KAFKA_VERSIONS.md

@@ -0,0 +1,38 @@
+# Adding / removing Kafka versions
+
+## Adding new Kafka versions
+
+There is no _definitive_ guide as every version might bring surprises.
+But these are some of the tasks you usually have to do.
+
+* If you add a new minor version, add new 3rd party libs directory to `docker-images/artifacts/kafka-thirdparty-libs`
+  * This directory is typically shared by all patch releases which are part of the same minor release. So it is named `<MAJOR>.<MINOR>.x`- e.g. `1.1.x`
+  * Add a `pom.xml` file with dependencies and make sure they are aligned with the dependencies inside Kafka (e.g. use similar version of Jackson libraries)
+  * Typically, you would do this by copying the directory for the previous minor version
+* Add the new version to `kafka-versions.yaml` file
+  * If the version you are adding is the latest, mark it as `default: true` and mark the previous one as `default: false`
+* If the version you are adding is the latest
+  * Update the Kafka versions and protocol versions in the documentation `attributes.adoc`
+  * If needed, update the examples (only in the `packaging/examples` directory):
+    * The `version` and the `inter.broker.protocol.version` fields
+    * The Docker image tag in the Kafka connect Build example
+  * Update the main `pom.xml` to use the latest Kafka version in the operator
+  * Update `systemtest/src/test/resources/upgrade/StrimziUpgradeST.yaml` with the new version
+  * Update protocol versions in `cluster-operator/src/test/java/io/strimzi/operator/cluster/model/KafkaConfigurationTests.java` if needed
+* Make sure the `cluster-operator/src/test/java/io/strimzi/operator/cluster/KafkaVersionTestUtils.java` is up-to-date
+* If you are adding a release candidate which is not yet published to Maven central, make sure to update the `ST_FILE_PLUGIN_URL_DEFAULT` field in `systemtest/src/main/java/io/strimzi/systemtest/Environment.java` to get the system tests to pass.
+* Run `make all` to update all the installation files, generated docs files, Helm chart etc.
+  You will need to commit also all the files it updates before opening a PR.
+* Add a `CHANGELOG.md` record
+* Run unit tests
+* Run system tests (typically as part of the PR on Azure Pipelines)
+
+## Removing Kafka version
+
+* Edit the `kafka-versions.yaml` file and mark the version as unsupported (do not remove it, just set the `supported` flag to `false`)
+* Run `make all` to update all the installation files, generated docs files, Helm chart etc.
+  You will need to commit also all the files it updates before opening a PR.
+* Make sure the version is not referenced anywhere anymore (apart from the obvious places such as CHANGELOG.md)
+* Add a `CHANGELOG.md` record
+* Run unit tests
+* Run system tests