From e4496fc9f70a67e8142aad266515a2c9c3cfbb02 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 11:49:12 +0530 Subject: [PATCH 01/12] Added support for iam enabled msk using custom role attached to k8s pod --- src/main/docker/Dockerfile | 8 ----- src/main/docker/kafdrop.sh | 2 +- .../kafdrop/config/KafkaConfiguration.java | 31 +++++++++++++++---- src/main/resources/application.yml | 11 ++++--- src/main/resources/client.properties | 5 +++ 5 files changed, 38 insertions(+), 19 deletions(-) delete mode 100644 src/main/docker/Dockerfile create mode 100644 src/main/resources/client.properties diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile deleted file mode 100644 index a4e9c475..00000000 --- a/src/main/docker/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM openjdk:12.0.1-jdk-oraclelinux7 - -ADD kafdrop.sh / -ADD kafdrop*tar.gz / - -RUN chmod +x /kafdrop.sh - -ENTRYPOINT ["/kafdrop.sh"] diff --git a/src/main/docker/kafdrop.sh b/src/main/docker/kafdrop.sh index 47536cd5..e4942c37 100644 --- a/src/main/docker/kafdrop.sh +++ b/src/main/docker/kafdrop.sh @@ -64,4 +64,4 @@ ARGS="--add-opens=java.base/sun.nio.ch=ALL-UNNAMED -Xss256K \ $HEAP_ARGS \ $JVM_OPTS" -exec java $ARGS -jar /kafdrop*/kafdrop*jar ${CMD_ARGS} +exec java $ARGS -jar /etc/kafdrop*jar ${CMD_ARGS} diff --git a/src/main/java/kafdrop/config/KafkaConfiguration.java b/src/main/java/kafdrop/config/KafkaConfiguration.java index e866cf81..4d146bb4 100644 --- a/src/main/java/kafdrop/config/KafkaConfiguration.java +++ b/src/main/java/kafdrop/config/KafkaConfiguration.java @@ -11,26 +11,38 @@ @Component -@ConfigurationProperties(prefix = "kafka") +@ConfigurationProperties (prefix = "kafka") @Data public final class KafkaConfiguration { private static final Logger LOG = LoggerFactory.getLogger(KafkaConfiguration.class); private String brokerConnect; + private Boolean isSecured = false; + private String saslMechanism; + private String securityProtocol; + private String truststoreFile; + private String propertiesFile; + private String keystoreFile; + private String jaasConfig; + + private String clientCallback; + + private String iamEnabled; + public void applyCommon(Properties properties) { properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerConnect); - if (isSecured) { - LOG.warn("The 'isSecured' property is deprecated; consult README.md on the preferred way to configure security"); - properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); - properties.put(SaslConfigs.SASL_MECHANISM, saslMechanism); - } + + LOG.info("Setting security protocol to {}", securityProtocol); + LOG.info("Setting sasl mechanism to {}", saslMechanism); + properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); + properties.put(SaslConfigs.SASL_MECHANISM, saslMechanism); LOG.info("Checking truststore file {}", truststoreFile); if (new File(truststoreFile).isFile()) { @@ -38,6 +50,13 @@ public void applyCommon(Properties properties) { properties.put("ssl.truststore.location", truststoreFile); } + LOG.info("Setting iam enabled {}", iamEnabled); + if (Boolean.parseBoolean(iamEnabled)) { + LOG.info("Setting sasl.jaas.config {} and sasl and callback callback properties {}", jaasConfig, clientCallback); + properties.put(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, clientCallback); + properties.put(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig); + } + LOG.info("Checking keystore file {}", keystoreFile); if (new File(keystoreFile).isFile()) { LOG.info("Assigning keystore location to {}", keystoreFile); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 23bdeb25..7d317485 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -29,8 +29,11 @@ kafdrop.monitor: kafka: brokerConnect: localhost:9092 isSecured: false - saslMechanism: "PLAIN" - securityProtocol: "SASL_PLAINTEXT" - truststoreFile: "${KAFKA_TRUSTSTORE_FILE:kafka.truststore.jks}" + saslMechanism: "${KAFKA_SASL_MECHANISM:PLAIN}" + securityProtocol: "${KAFKA_SECURITY_PROTOCOL:SASL_PLAINTEXT}" + truststoreFile: "${KAFKA_TRUSTSTORE_FILE:/tmp/kafka.truststore.jks}" propertiesFile : "${KAFKA_PROPERTIES_FILE:kafka.properties}" - keystoreFile: "${KAFKA_KEYSTORE_FILE:kafka.keystore.jks}" \ No newline at end of file + keystoreFile: "${KAFKA_KEYSTORE_FILE:kafka.keystore.jks}" + iamEnabled: "${KAFKA_IAM_ENABLED:false}" + jaasConfig: "${KAFKA_JAAS_CONFIG}" + clientCallback: "software.amazon.msk.auth.iam.IAMClientCallbackHandler" \ No newline at end of file diff --git a/src/main/resources/client.properties b/src/main/resources/client.properties new file mode 100644 index 00000000..d70b170f --- /dev/null +++ b/src/main/resources/client.properties @@ -0,0 +1,5 @@ +ssl.truststore.location=/tmp/kafka.client.truststore.jks +security.protocol=SASL_SSL +sasl.mechanism=AWS_MSK_IAM +sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required; +sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler \ No newline at end of file From 24d2a53bbde5d1de30bd03c75bb6554a35d26be5 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 11:55:13 +0530 Subject: [PATCH 02/12] Dockerfile added --- src/main/docker/Dockfile | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/main/docker/Dockfile diff --git a/src/main/docker/Dockfile b/src/main/docker/Dockfile new file mode 100644 index 00000000..a4e9c475 --- /dev/null +++ b/src/main/docker/Dockfile @@ -0,0 +1,8 @@ +FROM openjdk:12.0.1-jdk-oraclelinux7 + +ADD kafdrop.sh / +ADD kafdrop*tar.gz / + +RUN chmod +x /kafdrop.sh + +ENTRYPOINT ["/kafdrop.sh"] From ed63e4c5eaeccc6567d99f0541201765b4ecab97 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 11:56:33 +0530 Subject: [PATCH 03/12] Renamed Dockerfile --- src/main/docker/{Dockfile => Dockerfile} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/docker/{Dockfile => Dockerfile} (100%) diff --git a/src/main/docker/Dockfile b/src/main/docker/Dockerfile similarity index 100% rename from src/main/docker/Dockfile rename to src/main/docker/Dockerfile From 2a979d72b077de45ff276ca99914505c23cee010 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 12:15:39 +0530 Subject: [PATCH 04/12] Removed unwanted client.properties. Additional formatting changes --- .../kafdrop/config/KafkaConfiguration.java | 22 ++++++------------- src/main/resources/client.properties | 5 ----- 2 files changed, 7 insertions(+), 20 deletions(-) delete mode 100644 src/main/resources/client.properties diff --git a/src/main/java/kafdrop/config/KafkaConfiguration.java b/src/main/java/kafdrop/config/KafkaConfiguration.java index 4d146bb4..21dab68b 100644 --- a/src/main/java/kafdrop/config/KafkaConfiguration.java +++ b/src/main/java/kafdrop/config/KafkaConfiguration.java @@ -17,40 +17,32 @@ public final class KafkaConfiguration { private static final Logger LOG = LoggerFactory.getLogger(KafkaConfiguration.class); private String brokerConnect; - private Boolean isSecured = false; - private String saslMechanism; - private String securityProtocol; - private String truststoreFile; - private String propertiesFile; - private String keystoreFile; - private String jaasConfig; - private String clientCallback; - private String iamEnabled; public void applyCommon(Properties properties) { properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerConnect); - LOG.info("Setting security protocol to {}", securityProtocol); - LOG.info("Setting sasl mechanism to {}", saslMechanism); - properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); - properties.put(SaslConfigs.SASL_MECHANISM, saslMechanism); + if (isSecured) { + LOG.info("Setting security protocol to {}", securityProtocol); + LOG.info("Setting sasl mechanism to {}", saslMechanism); + properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); + properties.put(SaslConfigs.SASL_MECHANISM, saslMechanism); + } LOG.info("Checking truststore file {}", truststoreFile); if (new File(truststoreFile).isFile()) { LOG.info("Assigning truststore location to {}", truststoreFile); properties.put("ssl.truststore.location", truststoreFile); } - - LOG.info("Setting iam enabled {}", iamEnabled); + LOG.info("Is iam enabled : {}", iamEnabled); if (Boolean.parseBoolean(iamEnabled)) { LOG.info("Setting sasl.jaas.config {} and sasl and callback callback properties {}", jaasConfig, clientCallback); properties.put(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, clientCallback); diff --git a/src/main/resources/client.properties b/src/main/resources/client.properties deleted file mode 100644 index d70b170f..00000000 --- a/src/main/resources/client.properties +++ /dev/null @@ -1,5 +0,0 @@ -ssl.truststore.location=/tmp/kafka.client.truststore.jks -security.protocol=SASL_SSL -sasl.mechanism=AWS_MSK_IAM -sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule required; -sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler \ No newline at end of file From 98f2206dab2e51486925800c2a908a3329cec911 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 12:16:12 +0530 Subject: [PATCH 05/12] Additional formatting changes --- src/main/java/kafdrop/config/KafkaConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/kafdrop/config/KafkaConfiguration.java b/src/main/java/kafdrop/config/KafkaConfiguration.java index 21dab68b..9471058b 100644 --- a/src/main/java/kafdrop/config/KafkaConfiguration.java +++ b/src/main/java/kafdrop/config/KafkaConfiguration.java @@ -11,7 +11,7 @@ @Component -@ConfigurationProperties (prefix = "kafka") +@ConfigurationProperties(prefix = "kafka") @Data public final class KafkaConfiguration { private static final Logger LOG = LoggerFactory.getLogger(KafkaConfiguration.class); From a018aa0dc13f69ec73acf9e6813a5f55b8f439f9 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 12:18:35 +0530 Subject: [PATCH 06/12] Pom changes --- pom.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pom.xml b/pom.xml index fcb5cd8d..437d59d6 100644 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,11 @@ lombok 1.18.8 + + software.amazon.msk + aws-msk-iam-auth + 1.0.0 + io.confluent kafka-avro-serializer @@ -106,6 +111,10 @@ org.slf4j slf4j-log4j12 + + org.apache.kafka + kafka-clients + @@ -154,6 +163,17 @@ spring-kafka 2.6.7 + + + org.apache.kafka + kafka-clients + 2.8.0 + + + com.amazonaws + aws-java-sdk-sts + 1.11.704 + org.springframework.boot spring-boot-starter-undertow From 93bd4a950f6dca2e7cad97ffafb40072c33cba42 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 14:27:36 +0530 Subject: [PATCH 07/12] Updated dockerfile --- src/main/docker/kafdrop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/docker/kafdrop.sh b/src/main/docker/kafdrop.sh index e4942c37..4b4869dd 100644 --- a/src/main/docker/kafdrop.sh +++ b/src/main/docker/kafdrop.sh @@ -64,4 +64,4 @@ ARGS="--add-opens=java.base/sun.nio.ch=ALL-UNNAMED -Xss256K \ $HEAP_ARGS \ $JVM_OPTS" -exec java $ARGS -jar /etc/kafdrop*jar ${CMD_ARGS} +exec java $ARGS -jar /kafkadrop/kafdrop*jar ${CMD_ARGS} From 041a47da726a573f2ab2b3d7f79c47abb8a39507 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 14:30:08 +0530 Subject: [PATCH 08/12] Updated pom properties. --- pom.xml | 15 ++++----------- src/main/docker/kafdrop.sh | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index 437d59d6..c599f9ca 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,8 @@ UTF-8 3.11.1 1.14.3 + 1.0.0 + 1.11.704 @@ -89,7 +91,7 @@ software.amazon.msk aws-msk-iam-auth - 1.0.0 + ${msk.auth.version} io.confluent @@ -111,10 +113,6 @@ org.slf4j slf4j-log4j12 - - org.apache.kafka - kafka-clients - @@ -164,15 +162,10 @@ 2.6.7 - - org.apache.kafka - kafka-clients - 2.8.0 - com.amazonaws aws-java-sdk-sts - 1.11.704 + ${sts.sdk.version} org.springframework.boot diff --git a/src/main/docker/kafdrop.sh b/src/main/docker/kafdrop.sh index 4b4869dd..47536cd5 100644 --- a/src/main/docker/kafdrop.sh +++ b/src/main/docker/kafdrop.sh @@ -64,4 +64,4 @@ ARGS="--add-opens=java.base/sun.nio.ch=ALL-UNNAMED -Xss256K \ $HEAP_ARGS \ $JVM_OPTS" -exec java $ARGS -jar /kafkadrop/kafdrop*jar ${CMD_ARGS} +exec java $ARGS -jar /kafdrop*/kafdrop*jar ${CMD_ARGS} From 1360e6b160662cbff72d080ffde5d8c7675ba67f Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 14:31:29 +0530 Subject: [PATCH 09/12] Updated yml with is secured --- src/main/resources/application.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7d317485..c5362ccb 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -28,7 +28,7 @@ kafdrop.monitor: kafka: brokerConnect: localhost:9092 - isSecured: false + isSecured: "${KAFKA_IS_SECURED:false}" saslMechanism: "${KAFKA_SASL_MECHANISM:PLAIN}" securityProtocol: "${KAFKA_SECURITY_PROTOCOL:SASL_PLAINTEXT}" truststoreFile: "${KAFKA_TRUSTSTORE_FILE:/tmp/kafka.truststore.jks}" @@ -36,4 +36,4 @@ kafka: keystoreFile: "${KAFKA_KEYSTORE_FILE:kafka.keystore.jks}" iamEnabled: "${KAFKA_IAM_ENABLED:false}" jaasConfig: "${KAFKA_JAAS_CONFIG}" - clientCallback: "software.amazon.msk.auth.iam.IAMClientCallbackHandler" \ No newline at end of file + clientCallback: "software.amazon.msk.auth.iam.IAMClientCallbackHandler" From e0788a6faa0e90c775f24db167910fff1ddd72ad Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 14:31:57 +0530 Subject: [PATCH 10/12] Updated yml with is secured --- src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c5362ccb..52df55e5 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -31,7 +31,7 @@ kafka: isSecured: "${KAFKA_IS_SECURED:false}" saslMechanism: "${KAFKA_SASL_MECHANISM:PLAIN}" securityProtocol: "${KAFKA_SECURITY_PROTOCOL:SASL_PLAINTEXT}" - truststoreFile: "${KAFKA_TRUSTSTORE_FILE:/tmp/kafka.truststore.jks}" + truststoreFile: "${KAFKA_TRUSTSTORE_FILE:/kafka.truststore.jks}" propertiesFile : "${KAFKA_PROPERTIES_FILE:kafka.properties}" keystoreFile: "${KAFKA_KEYSTORE_FILE:kafka.keystore.jks}" iamEnabled: "${KAFKA_IAM_ENABLED:false}" From bc78e8253ddb681d713b46015e599843b5844b03 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 14:32:22 +0530 Subject: [PATCH 11/12] Updated yml with updated properties file --- src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 52df55e5..a80190d0 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -31,7 +31,7 @@ kafka: isSecured: "${KAFKA_IS_SECURED:false}" saslMechanism: "${KAFKA_SASL_MECHANISM:PLAIN}" securityProtocol: "${KAFKA_SECURITY_PROTOCOL:SASL_PLAINTEXT}" - truststoreFile: "${KAFKA_TRUSTSTORE_FILE:/kafka.truststore.jks}" + truststoreFile: "${KAFKA_TRUSTSTORE_FILE:kafka.truststore.jks}" propertiesFile : "${KAFKA_PROPERTIES_FILE:kafka.properties}" keystoreFile: "${KAFKA_KEYSTORE_FILE:kafka.keystore.jks}" iamEnabled: "${KAFKA_IAM_ENABLED:false}" From 77e97a253f03b019c1301f98e397b52db17dc3f5 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 15:27:23 +0530 Subject: [PATCH 12/12] Added warn log --- src/main/java/kafdrop/config/KafkaConfiguration.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/kafdrop/config/KafkaConfiguration.java b/src/main/java/kafdrop/config/KafkaConfiguration.java index 9471058b..0df0c745 100644 --- a/src/main/java/kafdrop/config/KafkaConfiguration.java +++ b/src/main/java/kafdrop/config/KafkaConfiguration.java @@ -31,6 +31,7 @@ public void applyCommon(Properties properties) { properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerConnect); if (isSecured) { + LOG.warn("The 'isSecured' property is deprecated; consult README.md on the preferred way to configure security"); LOG.info("Setting security protocol to {}", securityProtocol); LOG.info("Setting sasl mechanism to {}", saslMechanism); properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol);