From 16cf8dce823a3babad898d61fb6afaf1b65c8267 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 15:13:27 +0100 Subject: [PATCH 01/38] Upgrade to Kafka 4.0 --- streams-bootstrap-core/build.gradle.kts | 2 +- .../java/com/bakdata/kafka/ConsumedX.java | 19 +- .../main/java/com/bakdata/kafka/KStreamX.java | 91 -- .../java/com/bakdata/kafka/KStreamXImpl.java | 131 --- .../main/java/com/bakdata/kafka/KTableX.java | 81 +- .../java/com/bakdata/kafka/KTableXImpl.java | 117 ++- .../java/com/bakdata/kafka/ConsumedXTest.java | 99 +- .../java/com/bakdata/kafka/KStreamXTest.java | 906 ------------------ .../java/com/bakdata/kafka/KTableXTest.java | 124 --- .../java/com/bakdata/kafka/ProducedXTest.java | 12 +- .../com/bakdata/kafka/RepartitionedXTest.java | 12 +- .../bakdata/kafka/SimpleLegacyProcessor.java | 48 - .../com/bakdata/kafka/SimpleTransformer.java | 48 - .../bakdata/kafka/SimpleValueTransformer.java | 48 - .../java/com/bakdata/kafka/KafkaTest.java | 2 +- .../bakdata/kafka/ConsumerGroupVerifier.java | 19 +- .../com/bakdata/kafka/KafkaTestClient.java | 6 +- 17 files changed, 294 insertions(+), 1471 deletions(-) delete mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java delete mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java delete mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java diff --git a/streams-bootstrap-core/build.gradle.kts b/streams-bootstrap-core/build.gradle.kts index 258ad826e..14e5b4282 100644 --- a/streams-bootstrap-core/build.gradle.kts +++ b/streams-bootstrap-core/build.gradle.kts @@ -6,7 +6,7 @@ plugins { } dependencies { - api(group = "com.bakdata.kafka", name = "kafka-streams-utils", version = "1.0.1") + api(group = "com.bakdata.kafka", name = "kafka-streams-utils", version = "1.1.1-SNAPSHOT") implementation(group = "org.apache.kafka", name = "kafka-tools") api(group = "org.apache.kafka", name = "kafka-streams") diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java index e530ba3ed..02baaff91 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java @@ -26,7 +26,8 @@ import java.util.function.Function; import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.Topology.AutoOffsetReset; +import org.apache.kafka.streams.AutoOffsetReset; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.processor.TimestampExtractor; @@ -109,6 +110,14 @@ public static ConsumedX with(final TimestampExtractor timestampExtr return new ConsumedX<>(configurator -> Consumed.with(timestampExtractor)); } + /** + * @see Consumed#with(Topology.AutoOffsetReset) + */ + @Deprecated(since = "5.0.0") + public static ConsumedX with(final Topology.AutoOffsetReset resetPolicy) { + return new ConsumedX<>(configurator -> Consumed.with(resetPolicy)); + } + /** * @see Consumed#with(AutoOffsetReset) */ @@ -159,6 +168,14 @@ public ConsumedX withOffsetResetPolicy(final AutoOffsetReset offsetResetPo return this.modify(consumed -> consumed.withOffsetResetPolicy(offsetResetPolicy)); } + /** + * @see Consumed#withOffsetResetPolicy(Topology.AutoOffsetReset) + */ + @Deprecated(since = "5.0.0") + public ConsumedX withOffsetResetPolicy(final Topology.AutoOffsetReset offsetResetPolicy) { + return this.modify(consumed -> consumed.withOffsetResetPolicy(offsetResetPolicy)); + } + /** * @see Consumed#withTimestampExtractor(TimestampExtractor) */ diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java index 86082a0f4..160cfc36e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java @@ -40,13 +40,10 @@ import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.kstream.StreamJoined; -import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueJoinerWithKey; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.kstream.ValueTransformerSupplier; -import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.processor.TopicNameExtractor; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -479,14 +476,6 @@ KErrorStreamX flatMapValuesCapturingErrors( @Override KStreamX peek(ForeachAction action, Named named); - @Deprecated - @Override - KStreamX[] branch(Named named, Predicate... predicates); - - @Deprecated - @Override - KStreamX[] branch(Predicate... predicates); - @Override BranchedKStreamX split(); @@ -499,14 +488,6 @@ KErrorStreamX flatMapValuesCapturingErrors( @Override KStreamX merge(KStream stream, Named named); - @Deprecated - @Override - KStreamX through(String topic); - - @Deprecated - @Override - KStreamX through(String topic, Produced produced); - @Override KStreamX repartition(); @@ -833,78 +814,6 @@ KStreamX leftJoin(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoinerWithKey valueJoiner, Named named); - @Deprecated - @Override - KStreamX transform( - TransformerSupplier> transformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX transform( - TransformerSupplier> transformerSupplier, - Named named, String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransform( - TransformerSupplier>> transformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransform( - TransformerSupplier>> transformerSupplier, Named named, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX transformValues( - ValueTransformerSupplier valueTransformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX transformValues( - ValueTransformerSupplier valueTransformerSupplier, - Named named, String... stateStoreNames); - - @Deprecated - @Override - KStreamX transformValues( - ValueTransformerWithKeySupplier valueTransformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX transformValues( - ValueTransformerWithKeySupplier valueTransformerSupplier, Named named, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransformValues( - ValueTransformerSupplier> valueTransformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransformValues( - ValueTransformerSupplier> valueTransformerSupplier, - Named named, String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransformValues( - ValueTransformerWithKeySupplier> valueTransformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransformValues( - ValueTransformerWithKeySupplier> valueTransformerSupplier, Named named, - String... stateStoreNames); - @Override KStreamX process( ProcessorSupplier processorSupplier, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java index b815ff320..bf307c8a1 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java @@ -24,7 +24,6 @@ package com.bakdata.kafka; -import java.util.Arrays; import lombok.AccessLevel; import lombok.Getter; import lombok.NonNull; @@ -46,13 +45,10 @@ import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.kstream.StreamJoined; -import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueJoinerWithKey; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.kstream.ValueTransformerSupplier; -import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.processor.TopicNameExtractor; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -369,20 +365,6 @@ public KStreamX peek(final ForeachAction action, fin return this.context.wrap(this.wrapped.peek(action, named)); } - @Override - public KStreamX[] branch(final Predicate... predicates) { - return Arrays.stream(this.wrapped.branch(predicates)) - .map(this.context::wrap) - .toArray(KStreamX[]::new); - } - - @Override - public KStreamX[] branch(final Named named, final Predicate... predicates) { - return Arrays.stream(this.wrapped.branch(named, predicates)) - .map(this.context::wrap) - .toArray(KStreamX[]::new); - } - @Override public BranchedKStreamX split() { return this.context.wrap(this.wrapped.split()); @@ -405,16 +387,6 @@ public KStreamX merge(final KStream stream, final Named named) { return this.context.wrap(this.wrapped.merge(other, named)); } - @Override - public KStreamX through(final String topic) { - return this.context.wrap(this.wrapped.through(topic)); - } - - @Override - public KStreamX through(final String topic, final Produced produced) { - return this.context.wrap(this.wrapped.through(topic, produced)); - } - @Override public KStreamX repartition() { return this.context.wrap(this.wrapped.repartition()); @@ -844,109 +816,6 @@ public KStreamX leftJoin(final GlobalKTable globalTa return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); } - @Override - public KStreamX transform( - final TransformerSupplier> transformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transform(transformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX transform( - final TransformerSupplier> transformerSupplier, final Named named, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transform(transformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX flatTransform( - final TransformerSupplier>> transformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.flatTransform(transformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX flatTransform( - final TransformerSupplier>> transformerSupplier, - final Named named, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.flatTransform(transformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX transformValues( - final ValueTransformerSupplier valueTransformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX transformValues( - final ValueTransformerSupplier valueTransformerSupplier, final Named named, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX transformValues( - final ValueTransformerWithKeySupplier valueTransformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX transformValues( - final ValueTransformerWithKeySupplier valueTransformerSupplier, - final Named named, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX flatTransformValues( - final ValueTransformerSupplier> valueTransformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX flatTransformValues( - final ValueTransformerSupplier> valueTransformerSupplier, final Named named, - final String... stateStoreNames) { - return this.context.wrap( - this.wrapped.flatTransformValues(valueTransformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX flatTransformValues( - final ValueTransformerWithKeySupplier> valueTransformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX flatTransformValues( - final ValueTransformerWithKeySupplier> valueTransformerSupplier, - final Named named, - final String... stateStoreNames) { - return this.context.wrap( - this.wrapped.flatTransformValues(valueTransformerSupplier, named, stateStoreNames)); - } - - @Override - public void process( - final org.apache.kafka.streams.processor.ProcessorSupplier processorSupplier, - final String... stateStoreNames) { - this.wrapped.process(processorSupplier, stateStoreNames); - } - - @Override - public void process( - final org.apache.kafka.streams.processor.ProcessorSupplier processorSupplier, - final Named named, final String... stateStoreNames) { - this.wrapped.process(processorSupplier, named, stateStoreNames); - } - @Override public KStreamX process( final ProcessorSupplier processorSupplier, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java index ac6377d60..3cbe9ec06 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java @@ -24,6 +24,7 @@ package com.bakdata.kafka; +import java.util.function.BiFunction; import java.util.function.Function; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.KeyValue; @@ -313,11 +314,6 @@ KTableX outerJoin(KTable other, KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); - @Deprecated - @Override - KTableX join(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named); - @Override KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined); @@ -332,11 +328,6 @@ KTableX join(KTable other, Function foreignKe KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, MaterializedX> materialized); - @Deprecated - @Override - KTableX join(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named, Materialized> materialized); - @Override KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, @@ -353,11 +344,6 @@ KTableX join(KTable other, Function foreignKe KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); - @Deprecated - @Override - KTableX leftJoin(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named); - @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined); @@ -372,11 +358,6 @@ KTableX leftJoin(KTable other, Function forei KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, MaterializedX> materialized); - @Deprecated - @Override - KTableX leftJoin(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named, Materialized> materialized); - @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, @@ -388,4 +369,64 @@ KTableX leftJoin(KTable other, Function forei KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, MaterializedX> materialized); + + @Override + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner); + + @Override + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined); + + @Override + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, Materialized> materialized); + + /** + * @see #join(KTable, BiFunction, ValueJoiner, Materialized) + */ + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, MaterializedX> materialized); + + @Override + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + Materialized> materialized); + + /** + * @see #join(KTable, BiFunction, ValueJoiner, TableJoined, Materialized) + */ + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + MaterializedX> materialized); + + @Override + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner); + + @Override + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined); + + @Override + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, Materialized> materialized); + + /** + * @see #leftJoin(KTable, BiFunction, ValueJoiner, Materialized) + */ + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, MaterializedX> materialized); + + @Override + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + Materialized> materialized); + + /** + * @see #leftJoin(KTable, BiFunction, ValueJoiner, TableJoined, Materialized) + */ + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + MaterializedX> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java index 73ac9744b..05941f574 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java @@ -24,6 +24,7 @@ package com.bakdata.kafka; +import java.util.function.BiFunction; import java.util.function.Function; import lombok.AccessLevel; import lombok.Getter; @@ -422,14 +423,6 @@ public KTableX join(final KTable other, return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner)); } - @Override - public KTableX join(final KTable other, - final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named) { - final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, named)); - } - @Override public KTableX join(final KTable other, final Function foreignKeyExtractor, @@ -454,15 +447,6 @@ public KTableX join(final KTable other, return this.join(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); } - @Override - public KTableX join(final KTable other, - final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named, - final Materialized> materialized) { - final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, named, materialized)); - } - @Override public KTableX join(final KTable other, final Function foreignKeyExtractor, @@ -489,14 +473,6 @@ public KTableX leftJoin(final KTable other, return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner)); } - @Override - public KTableX leftJoin(final KTable other, - final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named) { - final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, named)); - } - @Override public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, @@ -525,29 +501,104 @@ public KTableX leftJoin(final KTable other, @Override public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named, + final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, named, materialized)); + return this.context.wrap( + this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); } @Override public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final MaterializedX> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, + materialized.configure(this.context.getConfigurator())); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner)); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, tableJoined)); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final Materialized> materialized) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, materialized)); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final MaterializedX> materialized) { + return this.join(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap( - this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); } @Override - public KTableX leftJoin(final KTable other, - final Function foreignKeyExtractor, + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final MaterializedX> materialized) { - return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, - materialized.configure(this.context.getConfigurator())); + return this.join(other, foreignKeyExtractor, joiner, tableJoined, materialized.configure(this.context.getConfigurator())); + } + + @Override + public KTableX leftJoin(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner)); + } + + @Override + public KTableX leftJoin(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined)); + } + + @Override + public KTableX leftJoin(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final Materialized> materialized) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, materialized)); + } + + @Override + public KTableX leftJoin(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final MaterializedX> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); + } + + @Override + public KTableX leftJoin(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final Materialized> materialized) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); + } + + @Override + public KTableX leftJoin(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final MaterializedX> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, materialized.configure(this.context.getConfigurator())); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java index 2599b0018..36b236a4e 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java @@ -36,7 +36,8 @@ import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.streams.Topology.AutoOffsetReset; +import org.apache.kafka.streams.AutoOffsetReset; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.TopologyDescription.Node; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; @@ -271,7 +272,53 @@ void shouldUseOffsetResetPolicy() { @Override public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = - builder.stream("input", ConsumedX.with(AutoOffsetReset.LATEST)); + builder.stream("input", ConsumedX.with(AutoOffsetReset.latest())); + input.to("output"); + } + }; + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + TestHelper.run(runner); + KafkaTest.awaitActive(executableApp); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("baz", "qux"))); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.value()).isEqualTo("qux"); + }); + } + } + } + + @Test + void shouldUseLegacyOffsetResetPolicy() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Topology.AutoOffsetReset.LATEST)); input.to("output"); } }; @@ -317,7 +364,53 @@ void shouldUseOffsetResetPolicyModifier() { @Override public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input", - ConsumedX.as("stream").withOffsetResetPolicy(AutoOffsetReset.LATEST)); + ConsumedX.as("stream").withOffsetResetPolicy(AutoOffsetReset.latest())); + input.to("output"); + } + }; + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + TestHelper.run(runner); + KafkaTest.awaitActive(executableApp); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("baz", "qux"))); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.value()).isEqualTo("qux"); + }); + } + } + } + + @Test + void shouldUseLegacyOffsetResetPolicyModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KStreamX input = builder.stream("input", + ConsumedX.as("stream").withOffsetResetPolicy(Topology.AutoOffsetReset.LATEST)); input.to("output"); } }; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 7c1392820..56923f77e 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -24,7 +24,6 @@ package com.bakdata.kafka; -import static java.util.Collections.emptyList; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -49,12 +48,8 @@ import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.Printed; -import org.apache.kafka.streams.kstream.Produced; -import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.kstream.ValueTransformerSupplier; -import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.FixedKeyRecord; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -3103,48 +3098,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldBranch() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX[] branches = input.branch((k, v) -> true); - branches[0].to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - } - } - - @Test - void shouldBranchNamed() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX[] branches = input.branch(Named.as("branch"), (k, v) -> true); - branches[0].to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - } - } - @Test void shouldTableJoin() { final StringApp app = new StringApp() { @@ -3639,863 +3592,4 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldRouteThrough() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX through = input.through("intermediate"); - through.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .add("foo", "bar"); - topology.streamOutput("intermediate") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - } - } - - @Test - void shouldRouteThroughUsingProduced() { - final DoubleApp app = new DoubleApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KStreamX through = - input.through("intermediate", Produced.with(Serdes.String(), Serdes.String())); - through.to("output", ProducedX.with(Serdes.String(), Serdes.String())); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("foo", "bar"); - topology.streamOutput("intermediate") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.streamOutput("output") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransform() { - final TransformerSupplier> transformer = - () -> new SimpleTransformer<>() { - - @Override - public KeyValue transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return KeyValue.pair("baz", "qux"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transform(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformNamed() { - final TransformerSupplier> transformer = - () -> new SimpleTransformer<>() { - - @Override - public KeyValue transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return KeyValue.pair("baz", "qux"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transform(transformer, Named.as("transform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformUsingStore() { - final TransformerSupplier> transformer = - () -> new SimpleTransformer<>() { - - @Override - public KeyValue transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transform(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformNamedUsingStore() { - final TransformerSupplier> transformer = - () -> new SimpleTransformer<>() { - - @Override - public KeyValue transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transform(transformer, Named.as("transform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransform() { - final TransformerSupplier>> transformer = - () -> new SimpleTransformer<>() { - - @Override - public Iterable> transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return List.of(KeyValue.pair("baz", "qux")); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.flatTransform(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformNamed() { - final TransformerSupplier>> transformer = - () -> new SimpleTransformer<>() { - - @Override - public Iterable> transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return List.of(KeyValue.pair("baz", "qux")); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = - input.flatTransform(transformer, Named.as("flatTransform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformUsingStore() { - final TransformerSupplier>> transformer = - () -> new SimpleTransformer<>() { - - @Override - public Iterable> transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransform(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformNamedUsingStore() { - final TransformerSupplier>> transformer = - () -> new SimpleTransformer<>() { - - @Override - public Iterable> transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransform(transformer, Named.as("flatTransform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformValues() { - final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { - - @Override - public String transform(final String value) { - if ("bar".equals(value)) { - return "baz"; - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transformValues(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformValuesNamed() { - final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { - - @Override - public String transform(final String value) { - if ("bar".equals(value)) { - return "baz"; - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transformValues(transformer, Named.as("transform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformValuesUsingStore() { - final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { - - @Override - public String transform(final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(value, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transformValues(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("bar")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformValuesNamedUsingStore() { - final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { - - @Override - public String transform(final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(value, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transformValues(transformer, Named.as("transform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("bar")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformValues() { - final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { - - @Override - public Iterable transform(final String value) { - if ("bar".equals(value)) { - return List.of("baz"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.flatTransformValues(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformValuesNamed() { - final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { - - @Override - public Iterable transform(final String value) { - if ("bar".equals(value)) { - return List.of("baz"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = - input.flatTransformValues(transformer, Named.as("flatTransform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformValuesUsingStore() { - final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { - - @Override - public Iterable transform(final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(value, value); - return emptyList(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransformValues(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("bar")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformValuesNamedUsingStore() { - final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { - - @Override - public Iterable transform(final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(value, value); - return emptyList(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransformValues(transformer, Named.as("flatTransform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("bar")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformValuesWithKey() { - final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public String transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return "baz"; - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transformValues(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformValuesWithKeyNamed() { - final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public String transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return "baz"; - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transformValues(transformer, Named.as("transform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformValuesWithKeyUsingStore() { - final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public String transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transformValues(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformValuesWithKeyNamedUsingStore() { - final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public String transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transformValues(transformer, Named.as("transform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformValuesWithKey() { - final ValueTransformerWithKeySupplier> transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public Iterable transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return List.of("baz"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.flatTransformValues(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformValuesWithKeyNamed() { - final ValueTransformerWithKeySupplier> transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public Iterable transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return List.of("baz"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = - input.flatTransformValues(transformer, Named.as("flatTransform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformValuesWithKeyUsingStore() { - final ValueTransformerWithKeySupplier> transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public Iterable transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return emptyList(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransformValues(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformValuesWithKeyNamedUsingStore() { - final ValueTransformerWithKeySupplier> transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public Iterable transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return emptyList(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransformValues(transformer, Named.as("flatTransform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldLegacyProcess() { - final org.apache.kafka.streams.processor.ProcessorSupplier processor = - () -> new SimpleLegacyProcessor<>() { - - @Override - public void process(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.process(processor, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldLegacyProcessNamed() { - final org.apache.kafka.streams.processor.ProcessorSupplier processor = - () -> new SimpleLegacyProcessor<>() { - - @Override - public void process(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.process(processor, Named.as("process"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index 19145dec4..18037b478 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -1336,31 +1336,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldFKeyJoinNamed() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KTableX input = builder.table("input"); - final KTableX otherInput = builder.table("other_input"); - final KTableX joined = - input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, Named.as("join")); - joined.toStream().to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .add("foo", "bar"); - topology.input("other_input") - .add("bar", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - @Test void shouldFKeyJoinTableJoined() { final StringApp app = new StringApp() { @@ -1419,40 +1394,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldFKeyJoinNamedUsingMaterialized() { - final DoubleApp app = new DoubleApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KTableX otherInput = - builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KTableX joined = - input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, Named.as("join"), - Materialized.with(Serdes.String(), Serdes.String())); - joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("foo", "bar"); - topology.input("other_input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("bar", "baz"); - topology.streamOutput() - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - @Test void shouldFKeyJoinTableJoinedUsingMaterialized() { final DoubleApp app = new DoubleApp() { @@ -1515,34 +1456,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldLeftFKeyJoinNamed() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KTableX input = builder.table("input"); - final KTableX otherInput = builder.table("other_input"); - final KTableX joined = input.leftJoin(otherInput, Function.identity(), - (v1, v2) -> v2 == null ? v1 : v1 + v2, Named.as("join")); - joined.toStream().to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .add("foo", "bar"); - topology.input("other_input") - .add("bar", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - @Test void shouldLeftFKeyJoinTableJoined() { final StringApp app = new StringApp() { @@ -1608,43 +1521,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldLeftFKeyJoinNamedUsingMaterialized() { - final DoubleApp app = new DoubleApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KTableX otherInput = - builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KTableX joined = - input.leftJoin(otherInput, Function.identity(), (v1, v2) -> v2 == null ? v1 : v1 + v2, - Named.as("join"), Materialized.with(Serdes.String(), Serdes.String())); - joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("foo", "bar"); - topology.input("other_input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("bar", "baz"); - topology.streamOutput() - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - @Test void shouldLeftFKeyJoinTableJoinedUsingMaterialized() { final DoubleApp app = new DoubleApp() { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java index 4dab55f16..30f357775 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java @@ -30,6 +30,8 @@ import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; import com.bakdata.kafka.util.TopologyInformation; import java.util.List; +import java.util.Optional; +import java.util.Set; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.Serdes; @@ -219,7 +221,10 @@ void shouldUseStreamPartitioner() { public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input"); input.to("output", - ProducedX.streamPartitioner((topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1)); + ProducedX.streamPartitioner((topic, key, value, numPartitions) -> { + final int partition = "bar".equals(value) ? 0 : 1; + return Optional.of(Set.of(partition)); + })); } }; try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { @@ -269,7 +274,10 @@ void shouldUseStreamPartitionerModifier() { public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input"); input.to("output", ProducedX.as("output") - .withStreamPartitioner((topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1)); + .withStreamPartitioner((topic, key, value, numPartitions) -> { + final int partition = "bar".equals(value) ? 0 : 1; + return Optional.of(Set.of(partition)); + })); } }; try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java index 68494c6b6..fcaf5124a 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java @@ -33,6 +33,8 @@ import com.bakdata.kafka.util.TopicSettings; import com.bakdata.kafka.util.TopologyInformation; import java.util.List; +import java.util.Optional; +import java.util.Set; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.Serdes; @@ -232,7 +234,10 @@ public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input"); final KStreamX repartitioned = input.repartition(RepartitionedX .streamPartitioner( - (topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1) + (topic, key, value, numPartitions) -> { + final int partition = "bar".equals(value) ? 0 : 1; + return Optional.of(Set.of(partition)); + }) .withNumberOfPartitions(2) .withName("repartition")); repartitioned.to("output"); @@ -301,7 +306,10 @@ public void buildTopology(final StreamsBuilderX builder) { final KStreamX repartitioned = input.repartition( RepartitionedX.as("repartition") .withStreamPartitioner( - (topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1) + (topic, key, value, numPartitions) -> { + final int partition = "bar".equals(value) ? 0 : 1; + return Optional.of(Set.of(partition)); + }) .withNumberOfPartitions(2)); repartitioned.to("output"); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java deleted file mode 100644 index 951ad25a8..000000000 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import org.apache.kafka.streams.processor.Processor; -import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStore; - -abstract class SimpleLegacyProcessor implements Processor { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } - - @Override - public void close() { - // do nothing - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - -} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java deleted file mode 100644 index d458c68c9..000000000 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import org.apache.kafka.streams.kstream.Transformer; -import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStore; - -abstract class SimpleTransformer implements Transformer { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } - - @Override - public void close() { - // do nothing - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - -} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java deleted file mode 100644 index 61d7c4ad9..000000000 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import org.apache.kafka.streams.kstream.ValueTransformer; -import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStore; - -abstract class SimpleValueTransformer implements ValueTransformer { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } - - @Override - public void close() { - // do nothing - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - -} diff --git a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java index 500e0171f..5f1cf1948 100644 --- a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java +++ b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java @@ -36,7 +36,7 @@ @Testcontainers public abstract class KafkaTest { protected static final Duration POLL_TIMEOUT = Duration.ofSeconds(10); - public static final String KAFKA_VERSION = "3.8.1"; + public static final String KAFKA_VERSION = "4.0.0"; private final TestTopologyFactory testTopologyFactory = TestTopologyFactory.withSchemaRegistry(); @Container private final KafkaContainer kafkaCluster = newCluster(); diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/ConsumerGroupVerifier.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/ConsumerGroupVerifier.java index b7260919a..d9637a66a 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/ConsumerGroupVerifier.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/ConsumerGroupVerifier.java @@ -36,7 +36,7 @@ import org.apache.kafka.clients.admin.ConsumerGroupDescription; import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.ConsumerGroupState; +import org.apache.kafka.common.GroupState; import org.apache.kafka.common.TopicPartition; /** @@ -61,30 +61,31 @@ public static ConsumerGroupVerifier verify(final ExecutableStreamsApp app) { } /** - * Check whether consumer group has state {@link ConsumerGroupState#STABLE} - * @return true if consumer group has state {@link ConsumerGroupState#STABLE} + * Check whether consumer group has state {@link GroupState#STABLE} + * @return true if consumer group has state {@link GroupState#STABLE} */ public boolean isActive() { - return this.getState() == ConsumerGroupState.STABLE; + return this.getState() == GroupState.STABLE; } /** - * Check whether consumer group has state {@link ConsumerGroupState#EMPTY} - * @return true if consumer group has state {@link ConsumerGroupState#EMPTY} + * Check whether consumer group has state {@link GroupState#EMPTY} + * @return true if consumer group has state {@link GroupState#EMPTY} */ public boolean isClosed() { - return this.getState() == ConsumerGroupState.EMPTY; + return this.getState() == GroupState.EMPTY; } /** * Get current state of consumer group + * * @return current state of consumer group */ - public ConsumerGroupState getState() { + public GroupState getState() { try (final ImprovedAdminClient admin = this.adminClientSupplier.get(); final ConsumerGroupClient consumerGroupClient = admin.getConsumerGroupClient()) { final ConsumerGroupDescription description = consumerGroupClient.describe(this.group); - final ConsumerGroupState state = description.state(); + final GroupState state = description.groupState(); log.debug("Consumer group '{}' has state {}", this.group, state); return state; } diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java index 1d1a68c16..7abdf54d9 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java @@ -34,7 +34,7 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.AutoOffsetResetStrategy; /** * Client that supports communication with Kafka clusters in test setups, including topic management, reading from @@ -65,12 +65,12 @@ public SenderBuilder send() { /** * Prepare reading data from the cluster. {@link ConsumerConfig#AUTO_OFFSET_RESET_CONFIG} is configured to - * {@link OffsetResetStrategy#EARLIEST} + * {@link AutoOffsetResetStrategy#EARLIEST} * @return configured {@code ReaderBuilder} */ public ReaderBuilder read() { return new ReaderBuilder(this.endpointConfig.createKafkaProperties()) - .with(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, OffsetResetStrategy.EARLIEST.toString()); + .with(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AutoOffsetResetStrategy.EARLIEST.toString()); } /** From 4e5f6a7ddc763620dc4db4343365714a1f1b3ce9 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 15:21:08 +0100 Subject: [PATCH 02/38] Upgrade to Java 17 --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 0d38f7f4f..72fa80069 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -22,7 +22,7 @@ subprojects { configure { toolchain { - languageVersion = JavaLanguageVersion.of(11) + languageVersion = JavaLanguageVersion.of(17) } } } From 4db620cfbc9c55cae680ebf09f1e298113ad677c Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 15:28:15 +0100 Subject: [PATCH 03/38] Upgrade to Java 17 --- streams-bootstrap-cli/build.gradle.kts | 4 +++- streams-bootstrap-core/build.gradle.kts | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/streams-bootstrap-cli/build.gradle.kts b/streams-bootstrap-cli/build.gradle.kts index 0de98123c..588655ea1 100644 --- a/streams-bootstrap-cli/build.gradle.kts +++ b/streams-bootstrap-cli/build.gradle.kts @@ -20,7 +20,9 @@ dependencies { testImplementation(group = "org.mockito", name = "mockito-junit-jupiter", version = mockitoVersion) testImplementation(testFixtures(project(":streams-bootstrap-core"))) testImplementation(group = "com.ginsberg", name = "junit5-system-exit", version = "1.1.2") - testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") + testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") { + exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients + } val log4jVersion: String by project testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) } diff --git a/streams-bootstrap-core/build.gradle.kts b/streams-bootstrap-core/build.gradle.kts index 14e5b4282..fa202b48f 100644 --- a/streams-bootstrap-core/build.gradle.kts +++ b/streams-bootstrap-core/build.gradle.kts @@ -11,8 +11,12 @@ dependencies { api(group = "org.apache.kafka", name = "kafka-streams") api(group = "org.apache.kafka", name = "kafka-clients") - implementation(group = "io.confluent", name = "kafka-schema-serializer") - api(group = "io.confluent", name = "kafka-schema-registry-client") + implementation(group = "io.confluent", name = "kafka-schema-serializer") { + exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients + } + api(group = "io.confluent", name = "kafka-schema-registry-client") { + exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients + } implementation( group = "org.slf4j", name = "slf4j-api", @@ -38,7 +42,9 @@ dependencies { val testContainersVersion: String by project testFixturesApi(group = "org.testcontainers", name = "junit-jupiter", version = testContainersVersion) testFixturesApi(group = "org.testcontainers", name = "kafka", version = testContainersVersion) - testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") + testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") { + exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients + } val log4jVersion: String by project testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) val awaitilityVersion: String by project From a671f06f34e4390ae77bde4333cba462acfc8aba Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 16:03:31 +0100 Subject: [PATCH 04/38] Update --- .../main/java/com/bakdata/kafka/ConfiguredStreamsApp.java | 5 ++++- .../src/main/java/com/bakdata/kafka/TestTopologyFactory.java | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java index 503b14101..9e2894a24 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -56,6 +56,9 @@ private static Map createBaseConfig() { // compression kafkaConfig.put(StreamsConfig.producerPrefix(ProducerConfig.COMPRESSION_TYPE_CONFIG), "gzip"); + //TODO enable +// kafkaConfig.put(StreamsConfig.consumerPrefix(ConsumerConfig.GROUP_PROTOCOL_CONFIG), GroupProtocol.CONSUMER); + return kafkaConfig; } diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java index 9a15cd04b..e75078f7b 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java @@ -52,6 +52,7 @@ public final class TestTopologyFactory { private static final Map STREAMS_TEST_CONFIG = Map.of( // Disable caching to allow immediate aggregations StreamsConfig.STATESTORE_CACHE_MAX_BYTES_CONFIG, Long.toString(0L), + //TODO remove with new protocol ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, Integer.toString(10_000) ); private final String schemaRegistryUrl; From 205134dcf909c15bcb741b9b0ce8c98f0d4cbc42 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 16:23:29 +0100 Subject: [PATCH 05/38] Fix AutoOffsetReset --- .../src/main/java/com/bakdata/kafka/KafkaTestClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java index 7abdf54d9..96028c9ef 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java @@ -70,7 +70,7 @@ public SenderBuilder send() { */ public ReaderBuilder read() { return new ReaderBuilder(this.endpointConfig.createKafkaProperties()) - .with(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AutoOffsetResetStrategy.EARLIEST.toString()); + .with(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AutoOffsetResetStrategy.EARLIEST.type().toString()); } /** From 9702bff7bff11aba2dfd2a847027a644fb87a17c Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 17:58:31 +0100 Subject: [PATCH 06/38] Fix GlobalTable test --- .../test/java/com/bakdata/kafka/StreamsBuilderXTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsBuilderXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsBuilderXTest.java index c3248cc36..310e61a2d 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsBuilderXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsBuilderXTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.regex.Pattern; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -523,7 +524,10 @@ public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); final GlobalKTable otherInput = - builder.globalTable("table_input", MaterializedX.with(Serdes.String(), Serdes.String())); + builder.globalTable("table_input", + MaterializedX.>as("store") + .withKeySerde(Serdes.String()) + .withValueSerde( Serdes.String())); final KStreamX joined = input.join(otherInput, (k, v) -> k, (v1, v2) -> v1 + v2); joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } From 44d12bdfc0d056f0090448420763379bbf3e0df0 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 18:02:28 +0100 Subject: [PATCH 07/38] Fix GlobalTable test --- streams-bootstrap-core/src/test/resources/log4j2.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/streams-bootstrap-core/src/test/resources/log4j2.xml b/streams-bootstrap-core/src/test/resources/log4j2.xml index 0d4071ce2..238c14e5a 100644 --- a/streams-bootstrap-core/src/test/resources/log4j2.xml +++ b/streams-bootstrap-core/src/test/resources/log4j2.xml @@ -30,5 +30,8 @@ + + + From fbf4a1e9add9b8cb21fb64320c55e5c6ff8f6af7 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 18:04:02 +0100 Subject: [PATCH 08/38] Update dependencies --- streams-bootstrap-cli/build.gradle.kts | 2 +- streams-bootstrap-core/build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/streams-bootstrap-cli/build.gradle.kts b/streams-bootstrap-cli/build.gradle.kts index 0de98123c..92b9028af 100644 --- a/streams-bootstrap-cli/build.gradle.kts +++ b/streams-bootstrap-cli/build.gradle.kts @@ -19,7 +19,7 @@ dependencies { testImplementation(group = "org.mockito", name = "mockito-core", version = mockitoVersion) testImplementation(group = "org.mockito", name = "mockito-junit-jupiter", version = mockitoVersion) testImplementation(testFixtures(project(":streams-bootstrap-core"))) - testImplementation(group = "com.ginsberg", name = "junit5-system-exit", version = "1.1.2") + testImplementation(group = "com.ginsberg", name = "junit5-system-exit", version = "2.0.2") testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") val log4jVersion: String by project testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) diff --git a/streams-bootstrap-core/build.gradle.kts b/streams-bootstrap-core/build.gradle.kts index 258ad826e..b06a4ca5d 100644 --- a/streams-bootstrap-core/build.gradle.kts +++ b/streams-bootstrap-core/build.gradle.kts @@ -19,7 +19,7 @@ dependencies { version = "2.0.16" ) implementation(group = "org.jooq", name = "jool", version = "0.9.15") - implementation(group = "io.github.resilience4j", name = "resilience4j-retry", version = "1.7.1") + implementation(group = "io.github.resilience4j", name = "resilience4j-retry", version = "2.3.0") api(platform("com.bakdata.kafka:error-handling-bom:1.7.0")) api(group = "com.bakdata.kafka", name = "error-handling-core") From a71794a23a76b184dc5ed386b93390eb05671775 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 18:09:00 +0100 Subject: [PATCH 09/38] Update dependencies --- streams-bootstrap-cli/build.gradle.kts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/streams-bootstrap-cli/build.gradle.kts b/streams-bootstrap-cli/build.gradle.kts index 92b9028af..c187755f0 100644 --- a/streams-bootstrap-cli/build.gradle.kts +++ b/streams-bootstrap-cli/build.gradle.kts @@ -24,3 +24,15 @@ dependencies { val log4jVersion: String by project testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) } + +tasks.withType { + jvmArgumentProviders.add(CommandLineArgumentProvider { + listOf( + "-javaagent:${ + configurations.testRuntimeClasspath.get().files.find { + it.name.contains("junit5-system-exit") + } + }" + ) + }) +} From 6eb422abe93844c8a05227165bf1bdb0acee9225 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 18:43:13 +0100 Subject: [PATCH 10/38] Update dependencies --- streams-bootstrap-cli/src/test/resources/log4j2.xml | 6 ++++++ streams-bootstrap-core/src/test/resources/log4j2.xml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/streams-bootstrap-cli/src/test/resources/log4j2.xml b/streams-bootstrap-cli/src/test/resources/log4j2.xml index 0d4071ce2..adb7f38e3 100644 --- a/streams-bootstrap-cli/src/test/resources/log4j2.xml +++ b/streams-bootstrap-cli/src/test/resources/log4j2.xml @@ -30,5 +30,11 @@ + + + + + + diff --git a/streams-bootstrap-core/src/test/resources/log4j2.xml b/streams-bootstrap-core/src/test/resources/log4j2.xml index 238c14e5a..adb7f38e3 100644 --- a/streams-bootstrap-core/src/test/resources/log4j2.xml +++ b/streams-bootstrap-core/src/test/resources/log4j2.xml @@ -33,5 +33,8 @@ + + + From efd7139b25efbdd8583bacd00f31b87143a195e8 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 19:11:33 +0100 Subject: [PATCH 11/38] Update dependencies --- streams-bootstrap-cli/src/test/resources/log4j2.xml | 3 +++ streams-bootstrap-core/src/test/resources/log4j2.xml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/streams-bootstrap-cli/src/test/resources/log4j2.xml b/streams-bootstrap-cli/src/test/resources/log4j2.xml index adb7f38e3..dfbe33f8d 100644 --- a/streams-bootstrap-cli/src/test/resources/log4j2.xml +++ b/streams-bootstrap-cli/src/test/resources/log4j2.xml @@ -36,5 +36,8 @@ + + + diff --git a/streams-bootstrap-core/src/test/resources/log4j2.xml b/streams-bootstrap-core/src/test/resources/log4j2.xml index adb7f38e3..dfbe33f8d 100644 --- a/streams-bootstrap-core/src/test/resources/log4j2.xml +++ b/streams-bootstrap-core/src/test/resources/log4j2.xml @@ -36,5 +36,8 @@ + + + From f9b22eb1f94a22059fea10133273067fc4786c5e Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 21:03:52 +0100 Subject: [PATCH 12/38] Update dependencies --- streams-bootstrap-cli/build.gradle.kts | 4 +- .../src/test/resources/log4j2.xml | 9 + streams-bootstrap-core/build.gradle.kts | 14 +- .../bakdata/kafka/ConfiguredStreamsApp.java | 5 +- .../java/com/bakdata/kafka/ConsumedX.java | 19 +- .../main/java/com/bakdata/kafka/KStreamX.java | 91 -- .../java/com/bakdata/kafka/KStreamXImpl.java | 131 --- .../main/java/com/bakdata/kafka/KTableX.java | 81 +- .../java/com/bakdata/kafka/KTableXImpl.java | 119 ++- .../java/com/bakdata/kafka/ConsumedXTest.java | 99 +- .../java/com/bakdata/kafka/KStreamXTest.java | 906 ------------------ .../java/com/bakdata/kafka/KTableXTest.java | 124 --- .../java/com/bakdata/kafka/ProducedXTest.java | 12 +- .../com/bakdata/kafka/RepartitionedXTest.java | 12 +- .../bakdata/kafka/SimpleLegacyProcessor.java | 48 - .../com/bakdata/kafka/SimpleTransformer.java | 48 - .../bakdata/kafka/SimpleValueTransformer.java | 48 - .../bakdata/kafka/StreamsBuilderXTest.java | 7 +- .../src/test/resources/log4j2.xml | 9 + .../java/com/bakdata/kafka/KafkaTest.java | 2 +- .../bakdata/kafka/ConsumerGroupVerifier.java | 19 +- .../com/bakdata/kafka/KafkaTestClient.java | 6 +- .../bakdata/kafka/TestTopologyFactory.java | 1 + 23 files changed, 342 insertions(+), 1472 deletions(-) delete mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java delete mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java delete mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java diff --git a/streams-bootstrap-cli/build.gradle.kts b/streams-bootstrap-cli/build.gradle.kts index c187755f0..506f83997 100644 --- a/streams-bootstrap-cli/build.gradle.kts +++ b/streams-bootstrap-cli/build.gradle.kts @@ -20,7 +20,9 @@ dependencies { testImplementation(group = "org.mockito", name = "mockito-junit-jupiter", version = mockitoVersion) testImplementation(testFixtures(project(":streams-bootstrap-core"))) testImplementation(group = "com.ginsberg", name = "junit5-system-exit", version = "2.0.2") - testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") + testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") { + exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients + } val log4jVersion: String by project testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) } diff --git a/streams-bootstrap-cli/src/test/resources/log4j2.xml b/streams-bootstrap-cli/src/test/resources/log4j2.xml index 0d4071ce2..dfbe33f8d 100644 --- a/streams-bootstrap-cli/src/test/resources/log4j2.xml +++ b/streams-bootstrap-cli/src/test/resources/log4j2.xml @@ -30,5 +30,14 @@ + + + + + + + + + diff --git a/streams-bootstrap-core/build.gradle.kts b/streams-bootstrap-core/build.gradle.kts index b06a4ca5d..b225fad0b 100644 --- a/streams-bootstrap-core/build.gradle.kts +++ b/streams-bootstrap-core/build.gradle.kts @@ -6,13 +6,17 @@ plugins { } dependencies { - api(group = "com.bakdata.kafka", name = "kafka-streams-utils", version = "1.0.1") + api(group = "com.bakdata.kafka", name = "kafka-streams-utils", version = "1.1.1-SNAPSHOT") implementation(group = "org.apache.kafka", name = "kafka-tools") api(group = "org.apache.kafka", name = "kafka-streams") api(group = "org.apache.kafka", name = "kafka-clients") - implementation(group = "io.confluent", name = "kafka-schema-serializer") - api(group = "io.confluent", name = "kafka-schema-registry-client") + implementation(group = "io.confluent", name = "kafka-schema-serializer") { + exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients + } + api(group = "io.confluent", name = "kafka-schema-registry-client") { + exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients + } implementation( group = "org.slf4j", name = "slf4j-api", @@ -38,7 +42,9 @@ dependencies { val testContainersVersion: String by project testFixturesApi(group = "org.testcontainers", name = "junit-jupiter", version = testContainersVersion) testFixturesApi(group = "org.testcontainers", name = "kafka", version = testContainersVersion) - testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") + testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") { + exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients + } val log4jVersion: String by project testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) val awaitilityVersion: String by project diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java index 503b14101..9e2894a24 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -56,6 +56,9 @@ private static Map createBaseConfig() { // compression kafkaConfig.put(StreamsConfig.producerPrefix(ProducerConfig.COMPRESSION_TYPE_CONFIG), "gzip"); + //TODO enable +// kafkaConfig.put(StreamsConfig.consumerPrefix(ConsumerConfig.GROUP_PROTOCOL_CONFIG), GroupProtocol.CONSUMER); + return kafkaConfig; } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java index e530ba3ed..02baaff91 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java @@ -26,7 +26,8 @@ import java.util.function.Function; import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.Topology.AutoOffsetReset; +import org.apache.kafka.streams.AutoOffsetReset; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.processor.TimestampExtractor; @@ -109,6 +110,14 @@ public static ConsumedX with(final TimestampExtractor timestampExtr return new ConsumedX<>(configurator -> Consumed.with(timestampExtractor)); } + /** + * @see Consumed#with(Topology.AutoOffsetReset) + */ + @Deprecated(since = "5.0.0") + public static ConsumedX with(final Topology.AutoOffsetReset resetPolicy) { + return new ConsumedX<>(configurator -> Consumed.with(resetPolicy)); + } + /** * @see Consumed#with(AutoOffsetReset) */ @@ -159,6 +168,14 @@ public ConsumedX withOffsetResetPolicy(final AutoOffsetReset offsetResetPo return this.modify(consumed -> consumed.withOffsetResetPolicy(offsetResetPolicy)); } + /** + * @see Consumed#withOffsetResetPolicy(Topology.AutoOffsetReset) + */ + @Deprecated(since = "5.0.0") + public ConsumedX withOffsetResetPolicy(final Topology.AutoOffsetReset offsetResetPolicy) { + return this.modify(consumed -> consumed.withOffsetResetPolicy(offsetResetPolicy)); + } + /** * @see Consumed#withTimestampExtractor(TimestampExtractor) */ diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java index 86082a0f4..160cfc36e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java @@ -40,13 +40,10 @@ import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.kstream.StreamJoined; -import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueJoinerWithKey; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.kstream.ValueTransformerSupplier; -import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.processor.TopicNameExtractor; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -479,14 +476,6 @@ KErrorStreamX flatMapValuesCapturingErrors( @Override KStreamX peek(ForeachAction action, Named named); - @Deprecated - @Override - KStreamX[] branch(Named named, Predicate... predicates); - - @Deprecated - @Override - KStreamX[] branch(Predicate... predicates); - @Override BranchedKStreamX split(); @@ -499,14 +488,6 @@ KErrorStreamX flatMapValuesCapturingErrors( @Override KStreamX merge(KStream stream, Named named); - @Deprecated - @Override - KStreamX through(String topic); - - @Deprecated - @Override - KStreamX through(String topic, Produced produced); - @Override KStreamX repartition(); @@ -833,78 +814,6 @@ KStreamX leftJoin(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoinerWithKey valueJoiner, Named named); - @Deprecated - @Override - KStreamX transform( - TransformerSupplier> transformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX transform( - TransformerSupplier> transformerSupplier, - Named named, String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransform( - TransformerSupplier>> transformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransform( - TransformerSupplier>> transformerSupplier, Named named, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX transformValues( - ValueTransformerSupplier valueTransformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX transformValues( - ValueTransformerSupplier valueTransformerSupplier, - Named named, String... stateStoreNames); - - @Deprecated - @Override - KStreamX transformValues( - ValueTransformerWithKeySupplier valueTransformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX transformValues( - ValueTransformerWithKeySupplier valueTransformerSupplier, Named named, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransformValues( - ValueTransformerSupplier> valueTransformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransformValues( - ValueTransformerSupplier> valueTransformerSupplier, - Named named, String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransformValues( - ValueTransformerWithKeySupplier> valueTransformerSupplier, - String... stateStoreNames); - - @Deprecated - @Override - KStreamX flatTransformValues( - ValueTransformerWithKeySupplier> valueTransformerSupplier, Named named, - String... stateStoreNames); - @Override KStreamX process( ProcessorSupplier processorSupplier, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java index b815ff320..bf307c8a1 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java @@ -24,7 +24,6 @@ package com.bakdata.kafka; -import java.util.Arrays; import lombok.AccessLevel; import lombok.Getter; import lombok.NonNull; @@ -46,13 +45,10 @@ import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.kstream.StreamJoined; -import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.kstream.ValueJoiner; import org.apache.kafka.streams.kstream.ValueJoinerWithKey; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.kstream.ValueTransformerSupplier; -import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.processor.TopicNameExtractor; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -369,20 +365,6 @@ public KStreamX peek(final ForeachAction action, fin return this.context.wrap(this.wrapped.peek(action, named)); } - @Override - public KStreamX[] branch(final Predicate... predicates) { - return Arrays.stream(this.wrapped.branch(predicates)) - .map(this.context::wrap) - .toArray(KStreamX[]::new); - } - - @Override - public KStreamX[] branch(final Named named, final Predicate... predicates) { - return Arrays.stream(this.wrapped.branch(named, predicates)) - .map(this.context::wrap) - .toArray(KStreamX[]::new); - } - @Override public BranchedKStreamX split() { return this.context.wrap(this.wrapped.split()); @@ -405,16 +387,6 @@ public KStreamX merge(final KStream stream, final Named named) { return this.context.wrap(this.wrapped.merge(other, named)); } - @Override - public KStreamX through(final String topic) { - return this.context.wrap(this.wrapped.through(topic)); - } - - @Override - public KStreamX through(final String topic, final Produced produced) { - return this.context.wrap(this.wrapped.through(topic, produced)); - } - @Override public KStreamX repartition() { return this.context.wrap(this.wrapped.repartition()); @@ -844,109 +816,6 @@ public KStreamX leftJoin(final GlobalKTable globalTa return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); } - @Override - public KStreamX transform( - final TransformerSupplier> transformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transform(transformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX transform( - final TransformerSupplier> transformerSupplier, final Named named, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transform(transformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX flatTransform( - final TransformerSupplier>> transformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.flatTransform(transformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX flatTransform( - final TransformerSupplier>> transformerSupplier, - final Named named, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.flatTransform(transformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX transformValues( - final ValueTransformerSupplier valueTransformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX transformValues( - final ValueTransformerSupplier valueTransformerSupplier, final Named named, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX transformValues( - final ValueTransformerWithKeySupplier valueTransformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX transformValues( - final ValueTransformerWithKeySupplier valueTransformerSupplier, - final Named named, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX flatTransformValues( - final ValueTransformerSupplier> valueTransformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX flatTransformValues( - final ValueTransformerSupplier> valueTransformerSupplier, final Named named, - final String... stateStoreNames) { - return this.context.wrap( - this.wrapped.flatTransformValues(valueTransformerSupplier, named, stateStoreNames)); - } - - @Override - public KStreamX flatTransformValues( - final ValueTransformerWithKeySupplier> valueTransformerSupplier, - final String... stateStoreNames) { - return this.context.wrap(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); - } - - @Override - public KStreamX flatTransformValues( - final ValueTransformerWithKeySupplier> valueTransformerSupplier, - final Named named, - final String... stateStoreNames) { - return this.context.wrap( - this.wrapped.flatTransformValues(valueTransformerSupplier, named, stateStoreNames)); - } - - @Override - public void process( - final org.apache.kafka.streams.processor.ProcessorSupplier processorSupplier, - final String... stateStoreNames) { - this.wrapped.process(processorSupplier, stateStoreNames); - } - - @Override - public void process( - final org.apache.kafka.streams.processor.ProcessorSupplier processorSupplier, - final Named named, final String... stateStoreNames) { - this.wrapped.process(processorSupplier, named, stateStoreNames); - } - @Override public KStreamX process( final ProcessorSupplier processorSupplier, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java index ac6377d60..3cbe9ec06 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java @@ -24,6 +24,7 @@ package com.bakdata.kafka; +import java.util.function.BiFunction; import java.util.function.Function; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.KeyValue; @@ -313,11 +314,6 @@ KTableX outerJoin(KTable other, KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); - @Deprecated - @Override - KTableX join(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named); - @Override KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined); @@ -332,11 +328,6 @@ KTableX join(KTable other, Function foreignKe KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, MaterializedX> materialized); - @Deprecated - @Override - KTableX join(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named, Materialized> materialized); - @Override KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, @@ -353,11 +344,6 @@ KTableX join(KTable other, Function foreignKe KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); - @Deprecated - @Override - KTableX leftJoin(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named); - @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined); @@ -372,11 +358,6 @@ KTableX leftJoin(KTable other, Function forei KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, MaterializedX> materialized); - @Deprecated - @Override - KTableX leftJoin(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named, Materialized> materialized); - @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, @@ -388,4 +369,64 @@ KTableX leftJoin(KTable other, Function forei KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, MaterializedX> materialized); + + @Override + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner); + + @Override + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined); + + @Override + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, Materialized> materialized); + + /** + * @see #join(KTable, BiFunction, ValueJoiner, Materialized) + */ + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, MaterializedX> materialized); + + @Override + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + Materialized> materialized); + + /** + * @see #join(KTable, BiFunction, ValueJoiner, TableJoined, Materialized) + */ + KTableX join(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + MaterializedX> materialized); + + @Override + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner); + + @Override + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined); + + @Override + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, Materialized> materialized); + + /** + * @see #leftJoin(KTable, BiFunction, ValueJoiner, Materialized) + */ + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, MaterializedX> materialized); + + @Override + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + Materialized> materialized); + + /** + * @see #leftJoin(KTable, BiFunction, ValueJoiner, TableJoined, Materialized) + */ + KTableX leftJoin(KTable other, BiFunction foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + MaterializedX> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java index 73ac9744b..e7ca61e4c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java @@ -24,6 +24,7 @@ package com.bakdata.kafka; +import java.util.function.BiFunction; import java.util.function.Function; import lombok.AccessLevel; import lombok.Getter; @@ -422,14 +423,6 @@ public KTableX join(final KTable other, return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner)); } - @Override - public KTableX join(final KTable other, - final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named) { - final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, named)); - } - @Override public KTableX join(final KTable other, final Function foreignKeyExtractor, @@ -454,15 +447,6 @@ public KTableX join(final KTable other, return this.join(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); } - @Override - public KTableX join(final KTable other, - final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named, - final Materialized> materialized) { - final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, named, materialized)); - } - @Override public KTableX join(final KTable other, final Function foreignKeyExtractor, @@ -489,14 +473,6 @@ public KTableX leftJoin(final KTable other, return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner)); } - @Override - public KTableX leftJoin(final KTable other, - final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named) { - final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, named)); - } - @Override public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, @@ -525,16 +501,103 @@ public KTableX leftJoin(final KTable other, @Override public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named, + final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); - return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, named, materialized)); + return this.context.wrap( + this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); } @Override public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, + final MaterializedX> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, + materialized.configure(this.context.getConfigurator())); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner)); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, tableJoined)); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final Materialized> materialized) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, materialized)); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, + final MaterializedX> materialized) { + return this.join(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final Materialized> materialized) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); + } + + @Override + public KTableX join(final KTable other, final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final MaterializedX> materialized) { + return this.join(other, foreignKeyExtractor, joiner, tableJoined, + materialized.configure(this.context.getConfigurator())); + } + + @Override + public KTableX leftJoin(final KTable other, + final BiFunction foreignKeyExtractor, + final ValueJoiner joiner) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner)); + } + + @Override + public KTableX leftJoin(final KTable other, + final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined)); + } + + @Override + public KTableX leftJoin(final KTable other, + final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final Materialized> materialized) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, materialized)); + } + + @Override + public KTableX leftJoin(final KTable other, + final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, + final MaterializedX> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, + materialized.configure(this.context.getConfigurator())); + } + + @Override + public KTableX leftJoin(final KTable other, + final BiFunction foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); return this.context.wrap( @@ -543,7 +606,7 @@ public KTableX leftJoin(final KTable other, @Override public KTableX leftJoin(final KTable other, - final Function foreignKeyExtractor, + final BiFunction foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final MaterializedX> materialized) { return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java index 2599b0018..36b236a4e 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java @@ -36,7 +36,8 @@ import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.streams.Topology.AutoOffsetReset; +import org.apache.kafka.streams.AutoOffsetReset; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.TopologyDescription.Node; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; @@ -271,7 +272,53 @@ void shouldUseOffsetResetPolicy() { @Override public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = - builder.stream("input", ConsumedX.with(AutoOffsetReset.LATEST)); + builder.stream("input", ConsumedX.with(AutoOffsetReset.latest())); + input.to("output"); + } + }; + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + TestHelper.run(runner); + KafkaTest.awaitActive(executableApp); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("baz", "qux"))); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.value()).isEqualTo("qux"); + }); + } + } + } + + @Test + void shouldUseLegacyOffsetResetPolicy() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Topology.AutoOffsetReset.LATEST)); input.to("output"); } }; @@ -317,7 +364,53 @@ void shouldUseOffsetResetPolicyModifier() { @Override public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input", - ConsumedX.as("stream").withOffsetResetPolicy(AutoOffsetReset.LATEST)); + ConsumedX.as("stream").withOffsetResetPolicy(AutoOffsetReset.latest())); + input.to("output"); + } + }; + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + TestHelper.run(runner); + KafkaTest.awaitActive(executableApp); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("baz", "qux"))); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.value()).isEqualTo("qux"); + }); + } + } + } + + @Test + void shouldUseLegacyOffsetResetPolicyModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KStreamX input = builder.stream("input", + ConsumedX.as("stream").withOffsetResetPolicy(Topology.AutoOffsetReset.LATEST)); input.to("output"); } }; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 7c1392820..56923f77e 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -24,7 +24,6 @@ package com.bakdata.kafka; -import static java.util.Collections.emptyList; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -49,12 +48,8 @@ import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.Printed; -import org.apache.kafka.streams.kstream.Produced; -import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.kstream.ValueTransformerSupplier; -import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.FixedKeyRecord; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -3103,48 +3098,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldBranch() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX[] branches = input.branch((k, v) -> true); - branches[0].to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - } - } - - @Test - void shouldBranchNamed() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX[] branches = input.branch(Named.as("branch"), (k, v) -> true); - branches[0].to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - } - } - @Test void shouldTableJoin() { final StringApp app = new StringApp() { @@ -3639,863 +3592,4 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldRouteThrough() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX through = input.through("intermediate"); - through.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .add("foo", "bar"); - topology.streamOutput("intermediate") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - } - } - - @Test - void shouldRouteThroughUsingProduced() { - final DoubleApp app = new DoubleApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KStreamX through = - input.through("intermediate", Produced.with(Serdes.String(), Serdes.String())); - through.to("output", ProducedX.with(Serdes.String(), Serdes.String())); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("foo", "bar"); - topology.streamOutput("intermediate") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.streamOutput("output") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransform() { - final TransformerSupplier> transformer = - () -> new SimpleTransformer<>() { - - @Override - public KeyValue transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return KeyValue.pair("baz", "qux"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transform(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformNamed() { - final TransformerSupplier> transformer = - () -> new SimpleTransformer<>() { - - @Override - public KeyValue transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return KeyValue.pair("baz", "qux"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transform(transformer, Named.as("transform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformUsingStore() { - final TransformerSupplier> transformer = - () -> new SimpleTransformer<>() { - - @Override - public KeyValue transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transform(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformNamedUsingStore() { - final TransformerSupplier> transformer = - () -> new SimpleTransformer<>() { - - @Override - public KeyValue transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transform(transformer, Named.as("transform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransform() { - final TransformerSupplier>> transformer = - () -> new SimpleTransformer<>() { - - @Override - public Iterable> transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return List.of(KeyValue.pair("baz", "qux")); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.flatTransform(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformNamed() { - final TransformerSupplier>> transformer = - () -> new SimpleTransformer<>() { - - @Override - public Iterable> transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return List.of(KeyValue.pair("baz", "qux")); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = - input.flatTransform(transformer, Named.as("flatTransform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformUsingStore() { - final TransformerSupplier>> transformer = - () -> new SimpleTransformer<>() { - - @Override - public Iterable> transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransform(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformNamedUsingStore() { - final TransformerSupplier>> transformer = - () -> new SimpleTransformer<>() { - - @Override - public Iterable> transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransform(transformer, Named.as("flatTransform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformValues() { - final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { - - @Override - public String transform(final String value) { - if ("bar".equals(value)) { - return "baz"; - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transformValues(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformValuesNamed() { - final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { - - @Override - public String transform(final String value) { - if ("bar".equals(value)) { - return "baz"; - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transformValues(transformer, Named.as("transform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformValuesUsingStore() { - final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { - - @Override - public String transform(final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(value, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transformValues(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("bar")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformValuesNamedUsingStore() { - final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { - - @Override - public String transform(final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(value, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transformValues(transformer, Named.as("transform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("bar")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformValues() { - final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { - - @Override - public Iterable transform(final String value) { - if ("bar".equals(value)) { - return List.of("baz"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.flatTransformValues(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformValuesNamed() { - final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { - - @Override - public Iterable transform(final String value) { - if ("bar".equals(value)) { - return List.of("baz"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = - input.flatTransformValues(transformer, Named.as("flatTransform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformValuesUsingStore() { - final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { - - @Override - public Iterable transform(final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(value, value); - return emptyList(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransformValues(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("bar")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformValuesNamedUsingStore() { - final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { - - @Override - public Iterable transform(final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(value, value); - return emptyList(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransformValues(transformer, Named.as("flatTransform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("bar")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformValuesWithKey() { - final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public String transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return "baz"; - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transformValues(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformValuesWithKeyNamed() { - final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public String transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return "baz"; - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.transformValues(transformer, Named.as("transform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldTransformValuesWithKeyUsingStore() { - final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public String transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transformValues(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldTransformValuesWithKeyNamedUsingStore() { - final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public String transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return null; - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.transformValues(transformer, Named.as("transform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformValuesWithKey() { - final ValueTransformerWithKeySupplier> transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public Iterable transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return List.of("baz"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = input.flatTransformValues(transformer); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformValuesWithKeyNamed() { - final ValueTransformerWithKeySupplier> transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public Iterable transform(final String key, final String value) { - if ("foo".equals(key) && "bar".equals(value)) { - return List.of("baz"); - } - throw new UnsupportedOperationException(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KStreamX input = builder.stream("input"); - final KStreamX transformed = - input.flatTransformValues(transformer, Named.as("flatTransform")); - transformed.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldFlatTransformValuesWithKeyUsingStore() { - final ValueTransformerWithKeySupplier> transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public Iterable transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return emptyList(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransformValues(transformer, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldFlatTransformValuesWithKeyNamedUsingStore() { - final ValueTransformerWithKeySupplier> transformer = - () -> new SimpleValueTransformerWithKey<>() { - - @Override - public Iterable transform(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - return emptyList(); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.flatTransformValues(transformer, Named.as("flatTransform"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldLegacyProcess() { - final org.apache.kafka.streams.processor.ProcessorSupplier processor = - () -> new SimpleLegacyProcessor<>() { - - @Override - public void process(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.process(processor, "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - - @Test - void shouldLegacyProcessNamed() { - final org.apache.kafka.streams.processor.ProcessorSupplier processor = - () -> new SimpleLegacyProcessor<>() { - - @Override - public void process(final String key, final String value) { - final KeyValueStore store = this.getStateStore("my-store"); - store.put(key, value); - } - }; - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final StoreBuilder> store = builder.stores() - .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde()); - builder.addStateStore(store); - final KStreamX input = builder.stream("input"); - input.process(processor, Named.as("process"), "my-store"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - } - } - } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index 19145dec4..18037b478 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -1336,31 +1336,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldFKeyJoinNamed() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KTableX input = builder.table("input"); - final KTableX otherInput = builder.table("other_input"); - final KTableX joined = - input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, Named.as("join")); - joined.toStream().to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .add("foo", "bar"); - topology.input("other_input") - .add("bar", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - @Test void shouldFKeyJoinTableJoined() { final StringApp app = new StringApp() { @@ -1419,40 +1394,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldFKeyJoinNamedUsingMaterialized() { - final DoubleApp app = new DoubleApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KTableX otherInput = - builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KTableX joined = - input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, Named.as("join"), - Materialized.with(Serdes.String(), Serdes.String())); - joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("foo", "bar"); - topology.input("other_input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("bar", "baz"); - topology.streamOutput() - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - @Test void shouldFKeyJoinTableJoinedUsingMaterialized() { final DoubleApp app = new DoubleApp() { @@ -1515,34 +1456,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldLeftFKeyJoinNamed() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KTableX input = builder.table("input"); - final KTableX otherInput = builder.table("other_input"); - final KTableX joined = input.leftJoin(otherInput, Function.identity(), - (v1, v2) -> v2 == null ? v1 : v1 + v2, Named.as("join")); - joined.toStream().to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .add("foo", "bar"); - topology.input("other_input") - .add("bar", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - @Test void shouldLeftFKeyJoinTableJoined() { final StringApp app = new StringApp() { @@ -1608,43 +1521,6 @@ public void buildTopology(final StreamsBuilderX builder) { } } - @Test - void shouldLeftFKeyJoinNamedUsingMaterialized() { - final DoubleApp app = new DoubleApp() { - @Override - public void buildTopology(final StreamsBuilderX builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KTableX otherInput = - builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KTableX joined = - input.leftJoin(otherInput, Function.identity(), (v1, v2) -> v2 == null ? v1 : v1 + v2, - Named.as("join"), Materialized.with(Serdes.String(), Serdes.String())); - joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); - } - }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("foo", "bar"); - topology.input("other_input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("bar", "baz"); - topology.streamOutput() - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - @Test void shouldLeftFKeyJoinTableJoinedUsingMaterialized() { final DoubleApp app = new DoubleApp() { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java index 4dab55f16..30f357775 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java @@ -30,6 +30,8 @@ import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; import com.bakdata.kafka.util.TopologyInformation; import java.util.List; +import java.util.Optional; +import java.util.Set; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.Serdes; @@ -219,7 +221,10 @@ void shouldUseStreamPartitioner() { public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input"); input.to("output", - ProducedX.streamPartitioner((topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1)); + ProducedX.streamPartitioner((topic, key, value, numPartitions) -> { + final int partition = "bar".equals(value) ? 0 : 1; + return Optional.of(Set.of(partition)); + })); } }; try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { @@ -269,7 +274,10 @@ void shouldUseStreamPartitionerModifier() { public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input"); input.to("output", ProducedX.as("output") - .withStreamPartitioner((topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1)); + .withStreamPartitioner((topic, key, value, numPartitions) -> { + final int partition = "bar".equals(value) ? 0 : 1; + return Optional.of(Set.of(partition)); + })); } }; try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java index 68494c6b6..fcaf5124a 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java @@ -33,6 +33,8 @@ import com.bakdata.kafka.util.TopicSettings; import com.bakdata.kafka.util.TopologyInformation; import java.util.List; +import java.util.Optional; +import java.util.Set; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.Serdes; @@ -232,7 +234,10 @@ public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input"); final KStreamX repartitioned = input.repartition(RepartitionedX .streamPartitioner( - (topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1) + (topic, key, value, numPartitions) -> { + final int partition = "bar".equals(value) ? 0 : 1; + return Optional.of(Set.of(partition)); + }) .withNumberOfPartitions(2) .withName("repartition")); repartitioned.to("output"); @@ -301,7 +306,10 @@ public void buildTopology(final StreamsBuilderX builder) { final KStreamX repartitioned = input.repartition( RepartitionedX.as("repartition") .withStreamPartitioner( - (topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1) + (topic, key, value, numPartitions) -> { + final int partition = "bar".equals(value) ? 0 : 1; + return Optional.of(Set.of(partition)); + }) .withNumberOfPartitions(2)); repartitioned.to("output"); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java deleted file mode 100644 index 951ad25a8..000000000 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import org.apache.kafka.streams.processor.Processor; -import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStore; - -abstract class SimpleLegacyProcessor implements Processor { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } - - @Override - public void close() { - // do nothing - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - -} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java deleted file mode 100644 index d458c68c9..000000000 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import org.apache.kafka.streams.kstream.Transformer; -import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStore; - -abstract class SimpleTransformer implements Transformer { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } - - @Override - public void close() { - // do nothing - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - -} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java deleted file mode 100644 index 61d7c4ad9..000000000 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import org.apache.kafka.streams.kstream.ValueTransformer; -import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStore; - -abstract class SimpleValueTransformer implements ValueTransformer { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } - - @Override - public void close() { - // do nothing - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - -} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsBuilderXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsBuilderXTest.java index c3248cc36..dae88eaa6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsBuilderXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsBuilderXTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.regex.Pattern; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -523,7 +524,11 @@ public void buildTopology(final StreamsBuilderX builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); final GlobalKTable otherInput = - builder.globalTable("table_input", MaterializedX.with(Serdes.String(), Serdes.String())); + builder.globalTable("table_input", + // Kafka 4.0 likely contains a bug if Materialized does not have a name + MaterializedX.>as("store") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String())); final KStreamX joined = input.join(otherInput, (k, v) -> k, (v1, v2) -> v1 + v2); joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } diff --git a/streams-bootstrap-core/src/test/resources/log4j2.xml b/streams-bootstrap-core/src/test/resources/log4j2.xml index 0d4071ce2..dfbe33f8d 100644 --- a/streams-bootstrap-core/src/test/resources/log4j2.xml +++ b/streams-bootstrap-core/src/test/resources/log4j2.xml @@ -30,5 +30,14 @@ + + + + + + + + + diff --git a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java index 500e0171f..5f1cf1948 100644 --- a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java +++ b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java @@ -36,7 +36,7 @@ @Testcontainers public abstract class KafkaTest { protected static final Duration POLL_TIMEOUT = Duration.ofSeconds(10); - public static final String KAFKA_VERSION = "3.8.1"; + public static final String KAFKA_VERSION = "4.0.0"; private final TestTopologyFactory testTopologyFactory = TestTopologyFactory.withSchemaRegistry(); @Container private final KafkaContainer kafkaCluster = newCluster(); diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/ConsumerGroupVerifier.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/ConsumerGroupVerifier.java index b7260919a..d9637a66a 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/ConsumerGroupVerifier.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/ConsumerGroupVerifier.java @@ -36,7 +36,7 @@ import org.apache.kafka.clients.admin.ConsumerGroupDescription; import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.ConsumerGroupState; +import org.apache.kafka.common.GroupState; import org.apache.kafka.common.TopicPartition; /** @@ -61,30 +61,31 @@ public static ConsumerGroupVerifier verify(final ExecutableStreamsApp app) { } /** - * Check whether consumer group has state {@link ConsumerGroupState#STABLE} - * @return true if consumer group has state {@link ConsumerGroupState#STABLE} + * Check whether consumer group has state {@link GroupState#STABLE} + * @return true if consumer group has state {@link GroupState#STABLE} */ public boolean isActive() { - return this.getState() == ConsumerGroupState.STABLE; + return this.getState() == GroupState.STABLE; } /** - * Check whether consumer group has state {@link ConsumerGroupState#EMPTY} - * @return true if consumer group has state {@link ConsumerGroupState#EMPTY} + * Check whether consumer group has state {@link GroupState#EMPTY} + * @return true if consumer group has state {@link GroupState#EMPTY} */ public boolean isClosed() { - return this.getState() == ConsumerGroupState.EMPTY; + return this.getState() == GroupState.EMPTY; } /** * Get current state of consumer group + * * @return current state of consumer group */ - public ConsumerGroupState getState() { + public GroupState getState() { try (final ImprovedAdminClient admin = this.adminClientSupplier.get(); final ConsumerGroupClient consumerGroupClient = admin.getConsumerGroupClient()) { final ConsumerGroupDescription description = consumerGroupClient.describe(this.group); - final ConsumerGroupState state = description.state(); + final GroupState state = description.groupState(); log.debug("Consumer group '{}' has state {}", this.group, state); return state; } diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java index 1d1a68c16..96028c9ef 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/KafkaTestClient.java @@ -34,7 +34,7 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.AutoOffsetResetStrategy; /** * Client that supports communication with Kafka clusters in test setups, including topic management, reading from @@ -65,12 +65,12 @@ public SenderBuilder send() { /** * Prepare reading data from the cluster. {@link ConsumerConfig#AUTO_OFFSET_RESET_CONFIG} is configured to - * {@link OffsetResetStrategy#EARLIEST} + * {@link AutoOffsetResetStrategy#EARLIEST} * @return configured {@code ReaderBuilder} */ public ReaderBuilder read() { return new ReaderBuilder(this.endpointConfig.createKafkaProperties()) - .with(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, OffsetResetStrategy.EARLIEST.toString()); + .with(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, AutoOffsetResetStrategy.EARLIEST.type().toString()); } /** diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java index 9a15cd04b..e75078f7b 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java @@ -52,6 +52,7 @@ public final class TestTopologyFactory { private static final Map STREAMS_TEST_CONFIG = Map.of( // Disable caching to allow immediate aggregations StreamsConfig.STATESTORE_CACHE_MAX_BYTES_CONFIG, Long.toString(0L), + //TODO remove with new protocol ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, Integer.toString(10_000) ); private final String schemaRegistryUrl; From f8e8fc1c6992aeca4c5dd4b9ad36b7179869ce0a Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Mar 2025 21:06:06 +0100 Subject: [PATCH 13/38] Disable parallelism --- build.gradle.kts | 4 ++++ gradle.properties | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 72fa80069..e20d64b86 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,6 +8,10 @@ plugins { allprojects { group = "com.bakdata.kafka" + tasks.withType { + maxParallelForks = 1 + } + repositories { mavenCentral() maven(url = "https://packages.confluent.io/maven/") diff --git a/gradle.properties b/gradle.properties index 920e907a4..f8c0d9c3c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ version=4.0.2-SNAPSHOT org.gradle.caching=true -org.gradle.parallel=true +org.gradle.parallel=false testContainersVersion=1.20.5 fluentKafkaVersion=3.2.1 junitVersion=5.11.4 From ef1b24f10b111cce17357bcd854cbe2e8ece79a5 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 20 Mar 2025 09:28:00 +0100 Subject: [PATCH 14/38] Do not use native --- .../src/testFixtures/java/com/bakdata/kafka/KafkaTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java index 5f1cf1948..2fddcbb08 100644 --- a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java +++ b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java @@ -42,7 +42,7 @@ public abstract class KafkaTest { private final KafkaContainer kafkaCluster = newCluster(); public static KafkaContainer newCluster() { - return new KafkaContainer(DockerImageName.parse("apache/kafka-native") + return new KafkaContainer(DockerImageName.parse("apache/kafka") .withTag(KAFKA_VERSION)); } From 512029edacf55466922a5ce62b5fc8392b0686b1 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 20 Mar 2025 09:50:20 +0100 Subject: [PATCH 15/38] Revert parallelism --- build.gradle.kts | 4 ---- gradle.properties | 2 +- .../src/testFixtures/java/com/bakdata/kafka/KafkaTest.java | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index e20d64b86..72fa80069 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,10 +8,6 @@ plugins { allprojects { group = "com.bakdata.kafka" - tasks.withType { - maxParallelForks = 1 - } - repositories { mavenCentral() maven(url = "https://packages.confluent.io/maven/") diff --git a/gradle.properties b/gradle.properties index f8c0d9c3c..920e907a4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ version=4.0.2-SNAPSHOT org.gradle.caching=true -org.gradle.parallel=false +org.gradle.parallel=true testContainersVersion=1.20.5 fluentKafkaVersion=3.2.1 junitVersion=5.11.4 diff --git a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java index 2fddcbb08..83d4dab0c 100644 --- a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java +++ b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java @@ -42,7 +42,7 @@ public abstract class KafkaTest { private final KafkaContainer kafkaCluster = newCluster(); public static KafkaContainer newCluster() { - return new KafkaContainer(DockerImageName.parse("apache/kafka") + return new KafkaContainer(DockerImageName.parse("apache/kafka") //FIXME native image is flaky .withTag(KAFKA_VERSION)); } From 75bf60f8ce0ea3533c37cac8d34fcc607a19a1df Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 20 Mar 2025 10:17:23 +0100 Subject: [PATCH 16/38] Add ConsumerGroupVerifierTest --- streams-bootstrap-test/build.gradle.kts | 14 +++ .../kafka/ConsumerGroupVerifierTest.java | 108 ++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java diff --git a/streams-bootstrap-test/build.gradle.kts b/streams-bootstrap-test/build.gradle.kts index bb58067f3..991b1979f 100644 --- a/streams-bootstrap-test/build.gradle.kts +++ b/streams-bootstrap-test/build.gradle.kts @@ -12,4 +12,18 @@ dependencies { name = "fluent-kafka-streams-tests-junit5", version = fluentKafkaVersion ) + + val junitVersion: String by project + testRuntimeOnly(group = "org.junit.jupiter", name = "junit-jupiter-engine", version = junitVersion) + testImplementation(group = "org.junit.jupiter", name = "junit-jupiter-api", version = junitVersion) + testImplementation(group = "org.junit.jupiter", name = "junit-jupiter-params", version = junitVersion) + val assertJVersion: String by project + testImplementation(group = "org.assertj", name = "assertj-core", version = assertJVersion) + val testContainersVersion: String by project + testImplementation(group = "org.testcontainers", name = "junit-jupiter", version = testContainersVersion) + testImplementation(group = "org.testcontainers", name = "kafka", version = testContainersVersion) + val log4jVersion: String by project + testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) + val awaitilityVersion: String by project + testImplementation(group = "org.awaitility", name = "awaitility", version = awaitilityVersion) } diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java new file mode 100644 index 000000000..3d0ad3663 --- /dev/null +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java @@ -0,0 +1,108 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; +import java.time.Duration; +import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.Serdes.StringSerde; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionFactory; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.kafka.KafkaContainer; +import org.testcontainers.utility.DockerImageName; + +@Testcontainers +class ConsumerGroupVerifierTest { + + @Container + private final KafkaContainer kafkaCluster = + new KafkaContainer(DockerImageName.parse("apache/kafka") //FIXME native image is flaky + .withTag("4.0.0")); // TODO use single point of truth + + private static ConditionFactory await() { + return Awaitility.await() + .pollInterval(Duration.ofSeconds(2L)) + .atMost(Duration.ofSeconds(20L)); + } + + @Test + void shouldVerify() { + final StreamsApp app = new StreamsApp() { + + @Override + public void buildTopology(final StreamsBuilderX builder) { + builder.streamInput().toOutputTopic(); + } + + @Override + public String getUniqueAppId(final StreamsTopicConfig topics) { + return "group"; + } + + @Override + public SerdeConfig defaultSerializationConfig() { + return new SerdeConfig(StringSerde.class, StringSerde.class); + } + }; + final ConfiguredStreamsApp configuredApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .outputTopic("output") + .build(), TestTopologyFactory.createStreamsTestConfig())); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(this.kafkaCluster.getBootstrapServers()) + .build(); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(executableApp); + try (final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of( + new SimpleProducerRecord<>("foo", "bar") + )); + new Thread(runner).start(); + await().untilAsserted(() -> assertThat(verifier.isActive()).isTrue()); + await().untilAsserted(() -> assertThat(verifier.hasFinishedProcessing()).isTrue()); + testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("input", Duration.ofSeconds(10L)); + } + await().untilAsserted(() -> assertThat(verifier.isClosed()).isTrue()); + } + +} From 8bedc779cb607589d0ba23bf5bfb9e4164647eef Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 10:59:09 +0100 Subject: [PATCH 17/38] Update --- .../java/com/bakdata/kafka/ApacheKafkaContainerCluster.java | 2 -- .../test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java index d8d0c8c2d..641615e11 100644 --- a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java +++ b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java @@ -84,8 +84,6 @@ public ApacheKafkaContainerCluster(final String version, final int brokersNum, f .withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", internalTopicsRf + "") .withEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", internalTopicsRf + "") .withEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", internalTopicsRf + "") - .withEnv("KAFKA_LISTENERS", - "PLAINTEXT://:9092,BROKER://:9093,CONTROLLER://:9094") //TODO remove with 3.9.1 https://issues.apache.org/jira/browse/KAFKA-18281 .withStartupTimeout(Duration.ofMinutes(1))) .collect(Collectors.toList()); } diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java index 3d0ad3663..166c437e4 100644 --- a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java @@ -34,6 +34,7 @@ import org.apache.kafka.common.serialization.Serdes.StringSerde; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.utils.AppInfoParser; import org.awaitility.Awaitility; import org.awaitility.core.ConditionFactory; import org.junit.jupiter.api.Test; @@ -48,7 +49,7 @@ class ConsumerGroupVerifierTest { @Container private final KafkaContainer kafkaCluster = new KafkaContainer(DockerImageName.parse("apache/kafka") //FIXME native image is flaky - .withTag("4.0.0")); // TODO use single point of truth + .withTag(AppInfoParser.getVersion())); private static ConditionFactory await() { return Awaitility.await() From a1fdd506841b1c1c6079d00db8515e32b92fa740 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 11:24:56 +0100 Subject: [PATCH 18/38] Add tests --- .../java/com/bakdata/kafka/KTableXTest.java | 246 ++++++++++++++++++ 1 file changed, 246 insertions(+) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index 18037b478..2f99c04f6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -1558,6 +1558,252 @@ public void buildTopology(final StreamsBuilderX builder) { } } + @Test + void shouldFKeyJoinWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.join(otherInput, (k, v) -> v, (v1, v2) -> v1 + v2); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFKeyJoinTableJoinedWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = + input.join(otherInput, (k, v) -> v, (v1, v2) -> v1 + v2, TableJoined.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFKeyJoinWithKeyUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = input.join(otherInput, (k, v) -> v, (v1, v2) -> v1 + v2, + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFKeyJoinTableJoinedWithKeyUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.join(otherInput, (k, v) -> v, (v1, v2) -> v1 + v2, TableJoined.as("join"), + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.leftJoin(otherInput, (k, v) -> v, + (v1, v2) -> v2 == null ? v1 : v1 + v2); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinTableJoinedWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.leftJoin(otherInput, (k, v) -> v, + (v1, v2) -> v2 == null ? v1 : v1 + v2, TableJoined.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinWithKeyUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.leftJoin(otherInput, (k, v) -> v, (v1, v2) -> v2 == null ? v1 : v1 + v2, + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinTableJoinedWithKeyUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final StreamsBuilderX builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.leftJoin(otherInput, (k, v) -> v, (v1, v2) -> v2 == null ? v1 : v1 + v2, + TableJoined.as("join"), MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + @Test void shouldSuppress() { final StringApp app = new StringApp() { From 7e3642b20ea78ab4d3a986a8cdbae0346448c209 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 12:30:35 +0100 Subject: [PATCH 19/38] Add tests for test module --- streams-bootstrap-test/build.gradle.kts | 14 +++ .../kafka/ConsumerGroupVerifierTest.java | 109 ++++++++++++++++++ .../src/test/resources/log4j2.xml | 34 ++++++ 3 files changed, 157 insertions(+) create mode 100644 streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java create mode 100644 streams-bootstrap-test/src/test/resources/log4j2.xml diff --git a/streams-bootstrap-test/build.gradle.kts b/streams-bootstrap-test/build.gradle.kts index bb58067f3..991b1979f 100644 --- a/streams-bootstrap-test/build.gradle.kts +++ b/streams-bootstrap-test/build.gradle.kts @@ -12,4 +12,18 @@ dependencies { name = "fluent-kafka-streams-tests-junit5", version = fluentKafkaVersion ) + + val junitVersion: String by project + testRuntimeOnly(group = "org.junit.jupiter", name = "junit-jupiter-engine", version = junitVersion) + testImplementation(group = "org.junit.jupiter", name = "junit-jupiter-api", version = junitVersion) + testImplementation(group = "org.junit.jupiter", name = "junit-jupiter-params", version = junitVersion) + val assertJVersion: String by project + testImplementation(group = "org.assertj", name = "assertj-core", version = assertJVersion) + val testContainersVersion: String by project + testImplementation(group = "org.testcontainers", name = "junit-jupiter", version = testContainersVersion) + testImplementation(group = "org.testcontainers", name = "kafka", version = testContainersVersion) + val log4jVersion: String by project + testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) + val awaitilityVersion: String by project + testImplementation(group = "org.awaitility", name = "awaitility", version = awaitilityVersion) } diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java new file mode 100644 index 000000000..5cb0c0ef7 --- /dev/null +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java @@ -0,0 +1,109 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; +import java.time.Duration; +import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.Serdes.StringSerde; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.utils.AppInfoParser; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionFactory; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.kafka.KafkaContainer; +import org.testcontainers.utility.DockerImageName; + +@Testcontainers +class ConsumerGroupVerifierTest { + + @Container + private final KafkaContainer kafkaCluster = + new KafkaContainer(DockerImageName.parse("apache/kafka-native") + .withTag(AppInfoParser.getVersion())); + + private static ConditionFactory await() { + return Awaitility.await() + .pollInterval(Duration.ofSeconds(2L)) + .atMost(Duration.ofSeconds(20L)); + } + + @Test + void shouldVerify() { + final StreamsApp app = new StreamsApp() { + + @Override + public void buildTopology(final StreamsBuilderX builder) { + builder.streamInput().toOutputTopic(); + } + + @Override + public String getUniqueAppId(final StreamsTopicConfig topics) { + return "group"; + } + + @Override + public SerdeConfig defaultSerializationConfig() { + return new SerdeConfig(StringSerde.class, StringSerde.class); + } + }; + final ConfiguredStreamsApp configuredApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .outputTopic("output") + .build(), TestTopologyFactory.createStreamsTestConfig())); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(this.kafkaCluster.getBootstrapServers()) + .build(); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(executableApp); + try (final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of( + new SimpleProducerRecord<>("foo", "bar") + )); + new Thread(runner).start(); + await().untilAsserted(() -> assertThat(verifier.isActive()).isTrue()); + await().untilAsserted(() -> assertThat(verifier.hasFinishedProcessing()).isTrue()); + testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("input", Duration.ofSeconds(10L)); + } + await().untilAsserted(() -> assertThat(verifier.isClosed()).isTrue()); + } + +} diff --git a/streams-bootstrap-test/src/test/resources/log4j2.xml b/streams-bootstrap-test/src/test/resources/log4j2.xml new file mode 100644 index 000000000..0d4071ce2 --- /dev/null +++ b/streams-bootstrap-test/src/test/resources/log4j2.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 564e2eaf930747c4501e9b2b97bdb42c8031a625 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 12:50:21 +0100 Subject: [PATCH 20/38] Add tests for test module --- .../java/com/bakdata/kafka/ConsumerGroupVerifierTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java index 5cb0c0ef7..952f940b2 100644 --- a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java @@ -49,7 +49,10 @@ class ConsumerGroupVerifierTest { @Container private final KafkaContainer kafkaCluster = new KafkaContainer(DockerImageName.parse("apache/kafka-native") - .withTag(AppInfoParser.getVersion())); + .withTag(AppInfoParser.getVersion())) + .withEnv("KAFKA_LISTENERS", + "PLAINTEXT://:9092,BROKER://:9093,CONTROLLER://:9094"); + //TODO remove with 3.9.1 https://issues.apache.org/jira/browse/KAFKA-18281 private static ConditionFactory await() { return Awaitility.await() From e44619d52f41de9bf7e9c75577d7a8495d457fd5 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 13:03:00 +0100 Subject: [PATCH 21/38] Add tests for test module --- .../test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java index 952f940b2..bfd09ea5a 100644 --- a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java @@ -107,6 +107,7 @@ public SerdeConfig defaultSerializationConfig() { .from("input", Duration.ofSeconds(10L)); } await().untilAsserted(() -> assertThat(verifier.isClosed()).isTrue()); + assertThat(verifier.isActive()).isFalse(); } } From 5724eeaf395ee8a33be8c1e4b9ed0198c7246441 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 13:03:50 +0100 Subject: [PATCH 22/38] Add tests for test module --- .../test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java index 166c437e4..8d6649ef8 100644 --- a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java @@ -104,6 +104,7 @@ public SerdeConfig defaultSerializationConfig() { .from("input", Duration.ofSeconds(10L)); } await().untilAsserted(() -> assertThat(verifier.isClosed()).isTrue()); + assertThat(verifier.isActive()).isFalse(); } } From cf5ea371e96f7328b7533d8e97ef93fe43b6abbf Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 13:15:16 +0100 Subject: [PATCH 23/38] Add tests for test module --- streams-bootstrap-cli/build.gradle.kts | 2 +- streams-bootstrap-core/build.gradle.kts | 7 +--- streams-bootstrap-test/build.gradle.kts | 6 ++-- .../kafka/ConsumerGroupVerifierTest.java | 34 ++++--------------- .../kafka/ApacheKafkaContainerCluster.java | 0 .../java/com/bakdata/kafka/KafkaTest.java | 0 6 files changed, 11 insertions(+), 38 deletions(-) rename {streams-bootstrap-core => streams-bootstrap-test}/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java (100%) rename {streams-bootstrap-core => streams-bootstrap-test}/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java (100%) diff --git a/streams-bootstrap-cli/build.gradle.kts b/streams-bootstrap-cli/build.gradle.kts index 588655ea1..d238659c8 100644 --- a/streams-bootstrap-cli/build.gradle.kts +++ b/streams-bootstrap-cli/build.gradle.kts @@ -18,7 +18,7 @@ dependencies { val mockitoVersion: String by project testImplementation(group = "org.mockito", name = "mockito-core", version = mockitoVersion) testImplementation(group = "org.mockito", name = "mockito-junit-jupiter", version = mockitoVersion) - testImplementation(testFixtures(project(":streams-bootstrap-core"))) + testImplementation(testFixtures(project(":streams-bootstrap-test"))) testImplementation(group = "com.ginsberg", name = "junit5-system-exit", version = "1.1.2") testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") { exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients diff --git a/streams-bootstrap-core/build.gradle.kts b/streams-bootstrap-core/build.gradle.kts index c3c2233db..1cd370057 100644 --- a/streams-bootstrap-core/build.gradle.kts +++ b/streams-bootstrap-core/build.gradle.kts @@ -38,17 +38,12 @@ dependencies { testImplementation(group = "org.mockito", name = "mockito-core", version = mockitoVersion) testImplementation(group = "org.mockito", name = "mockito-junit-jupiter", version = mockitoVersion) - testFixturesApi(project(":streams-bootstrap-test")) - val testContainersVersion: String by project - testFixturesApi(group = "org.testcontainers", name = "junit-jupiter", version = testContainersVersion) - testFixturesApi(group = "org.testcontainers", name = "kafka", version = testContainersVersion) + testImplementation(testFixtures(project(":streams-bootstrap-test"))) testImplementation(group = "io.confluent", name = "kafka-streams-avro-serde") { exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients } val log4jVersion: String by project testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) - val awaitilityVersion: String by project - testFixturesApi(group = "org.awaitility", name = "awaitility", version = awaitilityVersion) } tasks.withType { diff --git a/streams-bootstrap-test/build.gradle.kts b/streams-bootstrap-test/build.gradle.kts index 991b1979f..a5ebddae1 100644 --- a/streams-bootstrap-test/build.gradle.kts +++ b/streams-bootstrap-test/build.gradle.kts @@ -20,10 +20,10 @@ dependencies { val assertJVersion: String by project testImplementation(group = "org.assertj", name = "assertj-core", version = assertJVersion) val testContainersVersion: String by project - testImplementation(group = "org.testcontainers", name = "junit-jupiter", version = testContainersVersion) - testImplementation(group = "org.testcontainers", name = "kafka", version = testContainersVersion) + testFixturesApi(group = "org.testcontainers", name = "junit-jupiter", version = testContainersVersion) + testFixturesApi(group = "org.testcontainers", name = "kafka", version = testContainersVersion) val log4jVersion: String by project testImplementation(group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = log4jVersion) val awaitilityVersion: String by project - testImplementation(group = "org.awaitility", name = "awaitility", version = awaitilityVersion) + testFixturesApi(group = "org.awaitility", name = "awaitility", version = awaitilityVersion) } diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java index bfd09ea5a..607ea7494 100644 --- a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java @@ -34,31 +34,9 @@ import org.apache.kafka.common.serialization.Serdes.StringSerde; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.common.utils.AppInfoParser; -import org.awaitility.Awaitility; -import org.awaitility.core.ConditionFactory; import org.junit.jupiter.api.Test; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.kafka.KafkaContainer; -import org.testcontainers.utility.DockerImageName; -@Testcontainers -class ConsumerGroupVerifierTest { - - @Container - private final KafkaContainer kafkaCluster = - new KafkaContainer(DockerImageName.parse("apache/kafka-native") - .withTag(AppInfoParser.getVersion())) - .withEnv("KAFKA_LISTENERS", - "PLAINTEXT://:9092,BROKER://:9093,CONTROLLER://:9094"); - //TODO remove with 3.9.1 https://issues.apache.org/jira/browse/KAFKA-18281 - - private static ConditionFactory await() { - return Awaitility.await() - .pollInterval(Duration.ofSeconds(2L)) - .atMost(Duration.ofSeconds(20L)); - } +class ConsumerGroupVerifierTest extends KafkaTest { @Test void shouldVerify() { @@ -85,12 +63,11 @@ public SerdeConfig defaultSerializationConfig() { .outputTopic("output") .build(), TestTopologyFactory.createStreamsTestConfig())); final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() - .bootstrapServers(this.kafkaCluster.getBootstrapServers()) + .bootstrapServers(this.getBootstrapServers()) .build(); final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); testClient.createTopic("input"); - final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(executableApp); try (final StreamsRunner runner = executableApp.createRunner()) { testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) @@ -99,14 +76,15 @@ public SerdeConfig defaultSerializationConfig() { new SimpleProducerRecord<>("foo", "bar") )); new Thread(runner).start(); - await().untilAsserted(() -> assertThat(verifier.isActive()).isTrue()); - await().untilAsserted(() -> assertThat(verifier.hasFinishedProcessing()).isTrue()); + awaitActive(executableApp); + awaitProcessing(executableApp); testClient.read() .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) .from("input", Duration.ofSeconds(10L)); } - await().untilAsserted(() -> assertThat(verifier.isClosed()).isTrue()); + awaitClosed(executableApp); + final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(executableApp); assertThat(verifier.isActive()).isFalse(); } diff --git a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java similarity index 100% rename from streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java rename to streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/ApacheKafkaContainerCluster.java diff --git a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java similarity index 100% rename from streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java rename to streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java From adfe7a11e558f4e22bb11f3f7dbe25cdb30f05d2 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 13:24:59 +0100 Subject: [PATCH 24/38] Update --- streams-bootstrap-test/src/test/resources/log4j2.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/streams-bootstrap-test/src/test/resources/log4j2.xml b/streams-bootstrap-test/src/test/resources/log4j2.xml index 0d4071ce2..dfbe33f8d 100644 --- a/streams-bootstrap-test/src/test/resources/log4j2.xml +++ b/streams-bootstrap-test/src/test/resources/log4j2.xml @@ -30,5 +30,14 @@ + + + + + + + + + From 0335cb97d713de9d6da19f7495664bb6272c2975 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 13:39:52 +0100 Subject: [PATCH 25/38] Update --- .../kafka/ConsumerGroupVerifierTest.java | 28 +++------ .../com/bakdata/kafka/SimpleStreamsApp.java | 45 +++++++++++++++ .../kafka/TestTopologyFactoryTest.java | 57 +++++++++++++++++++ 3 files changed, 111 insertions(+), 19 deletions(-) create mode 100644 streams-bootstrap-test/src/test/java/com/bakdata/kafka/SimpleStreamsApp.java create mode 100644 streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java index 607ea7494..a4a288dc2 100644 --- a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/ConsumerGroupVerifierTest.java @@ -30,8 +30,8 @@ import java.time.Duration; import java.util.List; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.Serdes.StringSerde; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; @@ -40,23 +40,7 @@ class ConsumerGroupVerifierTest extends KafkaTest { @Test void shouldVerify() { - final StreamsApp app = new StreamsApp() { - - @Override - public void buildTopology(final StreamsBuilderX builder) { - builder.streamInput().toOutputTopic(); - } - - @Override - public String getUniqueAppId(final StreamsTopicConfig topics) { - return "group"; - } - - @Override - public SerdeConfig defaultSerializationConfig() { - return new SerdeConfig(StringSerde.class, StringSerde.class); - } - }; + final StreamsApp app = new SimpleStreamsApp(); final ConfiguredStreamsApp configuredApp = new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() .inputTopics(List.of("input")) @@ -78,10 +62,16 @@ public SerdeConfig defaultSerializationConfig() { new Thread(runner).start(); awaitActive(executableApp); awaitProcessing(executableApp); - testClient.read() + final List> records = testClient.read() .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) .from("input", Duration.ofSeconds(10L)); + assertThat(records) + .hasSize(1) + .anySatisfy(rekord -> { + assertThat(rekord.key()).isEqualTo("foo"); + assertThat(rekord.value()).isEqualTo("bar"); + }); } awaitClosed(executableApp); final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(executableApp); diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/SimpleStreamsApp.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/SimpleStreamsApp.java new file mode 100644 index 000000000..081103f20 --- /dev/null +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/SimpleStreamsApp.java @@ -0,0 +1,45 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.serialization.Serdes.StringSerde; + +class SimpleStreamsApp implements StreamsApp { + + @Override + public void buildTopology(final StreamsBuilderX builder) { + builder.streamInput().toOutputTopic(); + } + + @Override + public String getUniqueAppId(final StreamsTopicConfig topics) { + return "group"; + } + + @Override + public SerdeConfig defaultSerializationConfig() { + return new SerdeConfig(StringSerde.class, StringSerde.class); + } +} diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java new file mode 100644 index 000000000..309c74f8a --- /dev/null +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java @@ -0,0 +1,57 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.junit5.TestTopologyExtension; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +class TestTopologyFactoryTest extends KafkaTest { + + @RegisterExtension + private final TestTopologyExtension testTopology = TestTopologyFactory.withoutSchemaRegistry() + .createTopologyExtension(createApp()); + + private static ConfiguredStreamsApp createApp() { + final StreamsApp app = new SimpleStreamsApp(); + return new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .outputTopic("output") + .build(), TestTopologyFactory.createStreamsTestConfig())); + } + + @Test + void shouldVerify() { + this.testTopology.input() + .add("foo", "bar"); + this.testTopology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + +} From 522f307381d8ef2ede3f238b4385312b5f068e84 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 13:52:02 +0100 Subject: [PATCH 26/38] Test new protocol --- .../main/java/com/bakdata/kafka/ConfiguredStreamsApp.java | 6 ++++-- .../main/java/com/bakdata/kafka/TestTopologyFactory.java | 4 +--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java index 9e2894a24..1cd265aed 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java @@ -30,6 +30,8 @@ import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.GroupProtocol; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; @@ -56,8 +58,8 @@ private static Map createBaseConfig() { // compression kafkaConfig.put(StreamsConfig.producerPrefix(ProducerConfig.COMPRESSION_TYPE_CONFIG), "gzip"); - //TODO enable -// kafkaConfig.put(StreamsConfig.consumerPrefix(ConsumerConfig.GROUP_PROTOCOL_CONFIG), GroupProtocol.CONSUMER); + kafkaConfig.put(StreamsConfig.consumerPrefix(ConsumerConfig.GROUP_PROTOCOL_CONFIG), + GroupProtocol.CONSUMER.toString()); return kafkaConfig; } diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java index e75078f7b..b7de27587 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java @@ -51,9 +51,7 @@ public final class TestTopologyFactory { private static final String MOCK_URL_PREFIX = "mock://"; private static final Map STREAMS_TEST_CONFIG = Map.of( // Disable caching to allow immediate aggregations - StreamsConfig.STATESTORE_CACHE_MAX_BYTES_CONFIG, Long.toString(0L), - //TODO remove with new protocol - ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, Integer.toString(10_000) + StreamsConfig.STATESTORE_CACHE_MAX_BYTES_CONFIG, Long.toString(0L) ); private final String schemaRegistryUrl; From 5ca71644460f4d717e7465c7d17795ff510b828e Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 14:16:03 +0100 Subject: [PATCH 27/38] Test new protocol --- .../src/testFixtures/java/com/bakdata/kafka/KafkaTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java index f1d244586..76fbeac57 100644 --- a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java +++ b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java @@ -27,7 +27,6 @@ import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import java.time.Duration; import org.apache.kafka.common.utils.AppInfoParser; -import org.apache.kafka.common.utils.AppInfoParser; import org.awaitility.Awaitility; import org.awaitility.core.ConditionFactory; import org.testcontainers.junit.jupiter.Container; @@ -44,7 +43,8 @@ public abstract class KafkaTest { public static KafkaContainer newCluster() { return new KafkaContainer(DockerImageName.parse("apache/kafka") //FIXME native image is flaky - .withTag(AppInfoParser.getVersion())); + .withTag(AppInfoParser.getVersion())) + .withEnv("KAFKA_GROUP_CONSUMER_SESSION_TIMEOUT_MS", "10000"); } protected static void awaitProcessing(final ExecutableStreamsApp app) { From b916965ee0fe2d38559b9b816b28884d90001647 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 21 Mar 2025 14:39:26 +0100 Subject: [PATCH 28/38] Test new protocol --- .../src/testFixtures/java/com/bakdata/kafka/KafkaTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java index 76fbeac57..9428b4bbc 100644 --- a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java +++ b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java @@ -44,7 +44,8 @@ public abstract class KafkaTest { public static KafkaContainer newCluster() { return new KafkaContainer(DockerImageName.parse("apache/kafka") //FIXME native image is flaky .withTag(AppInfoParser.getVersion())) - .withEnv("KAFKA_GROUP_CONSUMER_SESSION_TIMEOUT_MS", "10000"); + .withEnv("KAFKA_GROUP_CONSUMER_SESSION_TIMEOUT_MS", "10000") + .withEnv("KAFKA_GROUP_CONSUMER_MIN_SESSION_TIMEOUT_MS", "10000"); } protected static void awaitProcessing(final ExecutableStreamsApp app) { From 50ae132f72cd49a2a75c2297428985347b653c93 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 24 Mar 2025 07:09:38 +0100 Subject: [PATCH 29/38] Revert new protocol --- .../java/com/bakdata/kafka/ConfiguredProducerApp.java | 5 +++-- .../java/com/bakdata/kafka/ConfiguredStreamsApp.java | 9 +++------ .../main/java/com/bakdata/kafka/TestTopologyFactory.java | 3 ++- .../testFixtures/java/com/bakdata/kafka/KafkaTest.java | 4 +--- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProducerApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProducerApp.java index bf806ecbe..b6437b1bd 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProducerApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProducerApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,7 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.record.CompressionType; /** * A {@link ProducerApp} with a corresponding {@link AppConfiguration} @@ -50,7 +51,7 @@ private static Map createBaseConfig() { kafkaConfig.put(ProducerConfig.ACKS_CONFIG, "all"); // compression - kafkaConfig.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); + kafkaConfig.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, CompressionType.GZIP.toString()); return kafkaConfig; } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java index 1cd265aed..8b6f0ee77 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamsApp.java @@ -30,9 +30,8 @@ import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.GroupProtocol; import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; @@ -56,10 +55,8 @@ private static Map createBaseConfig() { kafkaConfig.put(StreamsConfig.producerPrefix(ProducerConfig.ACKS_CONFIG), "all"); // compression - kafkaConfig.put(StreamsConfig.producerPrefix(ProducerConfig.COMPRESSION_TYPE_CONFIG), "gzip"); - - kafkaConfig.put(StreamsConfig.consumerPrefix(ConsumerConfig.GROUP_PROTOCOL_CONFIG), - GroupProtocol.CONSUMER.toString()); + kafkaConfig.put(StreamsConfig.producerPrefix(ProducerConfig.COMPRESSION_TYPE_CONFIG), + CompressionType.GZIP.toString()); return kafkaConfig; } diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java index b7de27587..9a15cd04b 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java @@ -51,7 +51,8 @@ public final class TestTopologyFactory { private static final String MOCK_URL_PREFIX = "mock://"; private static final Map STREAMS_TEST_CONFIG = Map.of( // Disable caching to allow immediate aggregations - StreamsConfig.STATESTORE_CACHE_MAX_BYTES_CONFIG, Long.toString(0L) + StreamsConfig.STATESTORE_CACHE_MAX_BYTES_CONFIG, Long.toString(0L), + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, Integer.toString(10_000) ); private final String schemaRegistryUrl; diff --git a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java index 9428b4bbc..667d11a54 100644 --- a/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java +++ b/streams-bootstrap-test/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java @@ -43,9 +43,7 @@ public abstract class KafkaTest { public static KafkaContainer newCluster() { return new KafkaContainer(DockerImageName.parse("apache/kafka") //FIXME native image is flaky - .withTag(AppInfoParser.getVersion())) - .withEnv("KAFKA_GROUP_CONSUMER_SESSION_TIMEOUT_MS", "10000") - .withEnv("KAFKA_GROUP_CONSUMER_MIN_SESSION_TIMEOUT_MS", "10000"); + .withTag(AppInfoParser.getVersion())); } protected static void awaitProcessing(final ExecutableStreamsApp app) { From 3d931fbbf9299c6ed9e7bdd4d7d3cbcbc6a85fe4 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 26 Mar 2025 14:56:03 +0100 Subject: [PATCH 30/38] Add comment on intermediate topics --- .../src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java index 97116e66c..9858fdb65 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java @@ -111,6 +111,8 @@ public static void runResetter(final Collection inputTopics, final Colle } final Collection existingIntermediateTopics = filterExistingTopics(intermediateTopics, allTopics); if (!existingIntermediateTopics.isEmpty()) { + //TODO intermediate topics are deprecated. Consider resetting offsets to latest manually or delete them + // on reset argList.addAll(List.of("--intermediate-topics", String.join(",", existingIntermediateTopics))); } final String[] args = argList.toArray(String[]::new); From 6cfda7a294fecdbf61a990af0b929b26eec3c245 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 26 Mar 2025 15:44:48 +0100 Subject: [PATCH 31/38] Handle intermediate topics --- .github/workflows/build-and-publish.yaml | 1 + .../bakdata/kafka/StreamsCleanUpRunner.java | 42 +++++++++---------- .../integration/StreamsCleanUpRunnerTest.java | 6 +-- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml index 6c45be9f5..b3e5bcb46 100644 --- a/.github/workflows/build-and-publish.yaml +++ b/.github/workflows/build-and-publish.yaml @@ -13,6 +13,7 @@ jobs: uses: bakdata/ci-templates/.github/workflows/java-gradle-library.yaml@1.64.0 with: java-version: 17 + gradle-refresh-dependencies: true secrets: sonar-token: ${{ secrets.SONARCLOUD_TOKEN }} sonar-organization: ${{ secrets.SONARCLOUD_ORGANIZATION }} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java index 9858fdb65..43fd380c6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpRunner.java @@ -91,11 +91,10 @@ public static StreamsCleanUpRunner create(final @NonNull Topology topology, * Streams Reset Tool * * @param inputTopics list of input topics of the streams app - * @param intermediateTopics list of intermediate topics of the streams app * @param allTopics list of all topics that exists in the Kafka cluster * @param streamsAppConfig configuration properties of the streams app */ - public static void runResetter(final Collection inputTopics, final Collection intermediateTopics, + public static void runResetter(final Collection inputTopics, final Collection allTopics, final ImprovedStreamsConfig streamsAppConfig) { // StreamsResetter's internal AdminClient can only be configured with a properties file final String appId = streamsAppConfig.getAppId(); @@ -109,12 +108,6 @@ public static void runResetter(final Collection inputTopics, final Colle if (!existingInputTopics.isEmpty()) { argList.addAll(List.of("--input-topics", String.join(",", existingInputTopics))); } - final Collection existingIntermediateTopics = filterExistingTopics(intermediateTopics, allTopics); - if (!existingIntermediateTopics.isEmpty()) { - //TODO intermediate topics are deprecated. Consider resetting offsets to latest manually or delete them - // on reset - argList.addAll(List.of("--intermediate-topics", String.join(",", existingIntermediateTopics))); - } final String[] args = argList.toArray(String[]::new); final StreamsResetter resetter = new StreamsResetter(); final int returnCode = resetter.execute(args); @@ -205,14 +198,17 @@ private class Task { private void reset() { final Collection allTopics = this.adminClient.getTopicClient().listTopics(); + this.reset(allTopics); + } + + private void reset(final Collection allTopics) { final List inputTopics = StreamsCleanUpRunner.this.topologyInformation.getInputTopics(allTopics); - final List intermediateTopics = - StreamsCleanUpRunner.this.topologyInformation.getIntermediateTopics(allTopics); - runResetter(inputTopics, intermediateTopics, allTopics, StreamsCleanUpRunner.this.config); + runResetter(inputTopics, allTopics, StreamsCleanUpRunner.this.config); // the StreamsResetter is responsible for deleting internal topics StreamsCleanUpRunner.this.topologyInformation.getInternalTopics() .forEach(this::resetInternalTopic); + this.deleteIntermediateTopics(allTopics); try (final KafkaStreams kafkaStreams = this.createStreams()) { kafkaStreams.cleanUp(); } @@ -225,22 +221,26 @@ private KafkaStreams createStreams() { } private void cleanAndReset() { - this.reset(); - this.clean(); + final Collection allTopics = this.adminClient.getTopicClient().listTopics(); + this.reset(allTopics); + this.clean(allTopics); } - private void clean() { - this.deleteTopics(); + private void clean(final Collection allTopics) { + this.deleteOutputTopics(allTopics); this.deleteConsumerGroup(); StreamsCleanUpRunner.this.cleanHooks.runCleanHooks(); } - /** - * Delete output topics - */ - private void deleteTopics() { - final List externalTopics = StreamsCleanUpRunner.this.topologyInformation.getExternalSinkTopics(); - externalTopics.forEach(this::deleteTopic); + private void deleteIntermediateTopics(final Collection allTopics) { + final List intermediateTopics = + StreamsCleanUpRunner.this.topologyInformation.getIntermediateTopics(allTopics); + intermediateTopics.forEach(this::deleteTopic); + } + + private void deleteOutputTopics(final Collection allTopics) { + final List outputTopics = StreamsCleanUpRunner.this.topologyInformation.getOutputTopics(allTopics); + outputTopics.forEach(this::deleteTopic); } private void resetInternalTopic(final String topic) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java index 08348646b..bbd4d330f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java @@ -91,7 +91,7 @@ class StreamsCleanUpRunnerTest extends KafkaTest { @TempDir private Path stateDir; - static KeyValue toKeyValue(final ConsumerRecord consumerRecord) { + static KeyValue toKeyValue(final ConsumerRecord consumerRecord) { return new KeyValue<>(consumerRecord.key(), consumerRecord.value()); } @@ -327,7 +327,7 @@ void shouldDeleteIntermediateTopics() { } awaitClosed(executableApp); - clean(executableApp); + reset(executableApp); try (final ImprovedAdminClient admin = testClient.admin(); final TopicClient topicClient = admin.getTopicClient()) { @@ -533,7 +533,7 @@ void shouldDeleteSchemaOfIntermediateTopics() final String manualSubject = ComplexTopologyApplication.THROUGH_TOPIC + "-value"; this.softly.assertThat(client.getAllSubjects()) .contains(inputSubject, manualSubject); - clean(executableApp); + reset(executableApp); this.softly.assertThat(client.getAllSubjects()) .doesNotContain(manualSubject) From 41e65a80b53a15fd57abb79f2d519075a1f9afad Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 26 Mar 2025 15:55:12 +0100 Subject: [PATCH 32/38] Handle intermediate topics --- .../kafka/integration/StreamsCleanUpRunnerTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java index bbd4d330f..0b71e62ba 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java @@ -271,7 +271,6 @@ void shouldDeleteInternalTopics() { uniqueAppId + "-KSTREAM-AGGREGATE-STATE-STORE-0000000008-repartition"; final String backingTopic = uniqueAppId + "-KSTREAM-REDUCE-STATE-STORE-0000000003-changelog"; - final String manualTopic = ComplexTopologyApplication.THROUGH_TOPIC; try (final ImprovedAdminClient admin = testClient.admin(); final TopicClient topicClient = admin.getTopicClient()) { @@ -280,7 +279,6 @@ void shouldDeleteInternalTopics() { } this.softly.assertThat(topicClient.exists(internalTopic)).isTrue(); this.softly.assertThat(topicClient.exists(backingTopic)).isTrue(); - this.softly.assertThat(topicClient.exists(manualTopic)).isTrue(); } awaitClosed(executableApp); @@ -293,7 +291,6 @@ void shouldDeleteInternalTopics() { } this.softly.assertThat(topicClient.exists(internalTopic)).isFalse(); this.softly.assertThat(topicClient.exists(backingTopic)).isFalse(); - this.softly.assertThat(topicClient.exists(manualTopic)).isTrue(); } } } @@ -497,14 +494,13 @@ void shouldDeleteSchemaOfInternalTopics() uniqueAppId + "-KSTREAM-AGGREGATE-STATE-STORE-0000000008-repartition" + "-value"; final String backingSubject = uniqueAppId + "-KSTREAM-REDUCE-STATE-STORE-0000000003-changelog" + "-value"; - final String manualSubject = ComplexTopologyApplication.THROUGH_TOPIC + "-value"; this.softly.assertThat(client.getAllSubjects()) - .contains(inputSubject, internalSubject, backingSubject, manualSubject); + .contains(inputSubject, internalSubject, backingSubject); reset(executableApp); this.softly.assertThat(client.getAllSubjects()) .doesNotContain(internalSubject, backingSubject) - .contains(inputSubject, manualSubject); + .contains(inputSubject); } } From 8b7bcc717d24c8c13a5acec09948e28e8c83cd5c Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 26 Mar 2025 15:57:27 +0100 Subject: [PATCH 33/38] Handle intermediate topics --- .../bakdata/kafka/integration/StreamsCleanUpRunnerTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java index 0b71e62ba..bb2a6259b 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java @@ -538,7 +538,7 @@ void shouldDeleteSchemaOfIntermediateTopics() } @Test - void shouldCallCleanupHookForInternalTopics() { + void shouldCallCleanupHookForInternalAndIntermediateTopics() { try (final ConfiguredStreamsApp app = this.createComplexCleanUpHookApplication(); final ExecutableStreamsApp executableApp = app.withEndpoint(this.createEndpoint())) { reset(executableApp); @@ -546,6 +546,7 @@ void shouldCallCleanupHookForInternalTopics() { verify(this.topicHook).deleted(uniqueAppId + "-KSTREAM-AGGREGATE-STATE-STORE-0000000008-repartition"); verify(this.topicHook).deleted(uniqueAppId + "-KSTREAM-AGGREGATE-STATE-STORE-0000000008-changelog"); verify(this.topicHook).deleted(uniqueAppId + "-KSTREAM-REDUCE-STATE-STORE-0000000003-changelog"); + verify(this.topicHook).deleted(ComplexTopologyApplication.THROUGH_TOPIC); verify(this.topicHook).close(); verifyNoMoreInteractions(this.topicHook); } From f89e6a768f359137f524af927944af2301588625 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 6 May 2025 17:00:07 +0200 Subject: [PATCH 34/38] Update --- .../test/java/com/bakdata/kafka/TestTopologyFactoryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java index 309c74f8a..5f05fde1d 100644 --- a/streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java +++ b/streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java @@ -40,7 +40,7 @@ private static ConfiguredStreamsApp createApp() { return new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() .inputTopics(List.of("input")) .outputTopic("output") - .build(), TestTopologyFactory.createStreamsTestConfig())); + .build())); } @Test From aadaed69e48640ed9361c272324adabd33af9d81 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 19 Jun 2025 14:53:56 +0200 Subject: [PATCH 35/38] Fix --- .../java/com/bakdata/kafka/ConsumedXTest.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java index aded9e2c0..a0a5eec15 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java @@ -279,19 +279,19 @@ public void buildTopology(final StreamsBuilderX builder) { }; try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { kafkaCluster.start(); - final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() - .bootstrapServers(kafkaCluster.getBootstrapServers()) - .build(); - final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + final RuntimeConfiguration configuration = RuntimeConfiguration.create(kafkaCluster.getBootstrapServers()) + .withNoStateStoreCaching() + .withSessionTimeout(SESSION_TIMEOUT); + final KafkaTestClient testClient = new KafkaTestClient(configuration); testClient.createTopic("input"); testClient.createTopic("output"); testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); - try (final ConfiguredStreamsApp configuredApp = app.configureApp( - TestTopologyFactory.createStreamsTestConfig()); - final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + try (final ConfiguredStreamsApp configuredApp = app.configureApp(); + final ExecutableStreamsApp executableApp = configuredApp + .withRuntimeConfiguration(configuration); final StreamsRunner runner = executableApp.createRunner()) { TestHelper.run(runner); KafkaTest.awaitActive(executableApp); @@ -371,19 +371,19 @@ public void buildTopology(final StreamsBuilderX builder) { }; try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { kafkaCluster.start(); - final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() - .bootstrapServers(kafkaCluster.getBootstrapServers()) - .build(); - final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + final RuntimeConfiguration configuration = RuntimeConfiguration.create(kafkaCluster.getBootstrapServers()) + .withNoStateStoreCaching() + .withSessionTimeout(SESSION_TIMEOUT); + final KafkaTestClient testClient = new KafkaTestClient(configuration); testClient.createTopic("input"); testClient.createTopic("output"); testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); - try (final ConfiguredStreamsApp configuredApp = app.configureApp( - TestTopologyFactory.createStreamsTestConfig()); - final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + try (final ConfiguredStreamsApp configuredApp = app.configureApp(); + final ExecutableStreamsApp executableApp = configuredApp + .withRuntimeConfiguration(configuration); final StreamsRunner runner = executableApp.createRunner()) { TestHelper.run(runner); KafkaTest.awaitActive(executableApp); From e016dc625a8b1a36c59bce3e84fb2a388b953c3e Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 23 Jun 2025 09:34:41 +0200 Subject: [PATCH 36/38] Update snapshot repo --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index c6f7a6d36..74268e8f3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,7 +11,7 @@ allprojects { repositories { mavenCentral() maven(url = "https://packages.confluent.io/maven/") - maven(url = "https://s01.oss.sonatype.org/content/repositories/snapshots") + maven(url = "https://central.sonatype.com/repository/maven-snapshots") } } From 822e3096069e326ba1b157e2862acb8e1c3e7dc0 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 23 Jun 2025 13:17:34 +0200 Subject: [PATCH 37/38] Use stable version --- .github/workflows/build-and-publish.yaml | 1 - gradle/libs.versions.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml index 660a10c51..318c93514 100644 --- a/.github/workflows/build-and-publish.yaml +++ b/.github/workflows/build-and-publish.yaml @@ -13,7 +13,6 @@ jobs: uses: bakdata/ci-templates/.github/workflows/java-gradle-library.yaml@1.66.1 with: java-version: 17 - gradle-refresh-dependencies: true secrets: sonar-token: ${{ secrets.SONARCLOUD_TOKEN }} sonar-organization: ${{ secrets.SONARCLOUD_ORGANIZATION }} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index cf36d0c69..4fbbec855 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,7 +4,7 @@ mockito = "5.17.0" testcontainers = "1.21.0" [libraries] -kafka-streams-utils = { group = "com.bakdata.kafka", name = "kafka-streams-utils", version = "1.1.2-SNAPSHOT" } +kafka-streams-utils = { group = "com.bakdata.kafka", name = "kafka-streams-utils", version = "1.2.0" } kafka-tools = { group = "org.apache.kafka", name = "kafka-tools" } kafka-streams = { group = "org.apache.kafka", name = "kafka-streams" } kafka-clients = { group = "org.apache.kafka", name = "kafka-clients" } From a5961990c2a23f4ea08028efdd9dc3f9e2a54308 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 23 Jun 2025 13:38:40 +0200 Subject: [PATCH 38/38] fIX --- streams-bootstrap-cli-test/src/test/resources/log4j2.xml | 9 +++++++++ .../src/test/java/com/bakdata/kafka/ConsumedXTest.java | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/streams-bootstrap-cli-test/src/test/resources/log4j2.xml b/streams-bootstrap-cli-test/src/test/resources/log4j2.xml index a3089a43a..aed046ea1 100644 --- a/streams-bootstrap-cli-test/src/test/resources/log4j2.xml +++ b/streams-bootstrap-cli-test/src/test/resources/log4j2.xml @@ -27,6 +27,15 @@ + + + + + + + + + diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java index fa1ad38f8..b0e713ce8 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java @@ -294,7 +294,7 @@ public void buildTopology(final StreamsBuilderX builder) { final ExecutableStreamsApp executableApp = configuredApp .withRuntimeConfiguration(configuration); final StreamsRunner runner = executableApp.createRunner()) { - TestHelper.run(runner); + runAsync(runner); KafkaTest.awaitActive(executableApp); testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) @@ -386,7 +386,7 @@ public void buildTopology(final StreamsBuilderX builder) { final ExecutableStreamsApp executableApp = configuredApp .withRuntimeConfiguration(configuration); final StreamsRunner runner = executableApp.createRunner()) { - TestHelper.run(runner); + runAsync(runner); KafkaTest.awaitActive(executableApp); testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class)