diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000000..de42c718dc --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,41 @@ +github: + prebuilds: + master: true + branches: true + +tasks: + - name: Open tutorial in browser + command: gp preview https://developer.confluent.io/tutorials/creating-first-apache-kafka-producer-application/kafka.html + - name: Docker + init: gp sync-await code-copy + command: docker compose -f tutorial-workspace/docker-compose.yml up -d ; exit ; clear + - name: Terminal + before: gp preview https://developer.confluent.io/tutorials/creating-first-apache-kafka-producer-application/kafka.html + init: | + curl -L --http1.1 https://cnfl.io/cli | sudo sh -s -- -b /usr/local/bin + mkdir tutorial-workspace + mv _includes/tutorials/kafka-producer-application/kafka/code/* tutorial-workspace + rm -rf -- !(tutorial-workspace) + rm -rf tutorial-workspace/tutorial-steps .git* .semaphore/ + sdk default java 17.0.7.fx-zulu + command: gp sync-done code-copy ; cd tutorial-workspace ; clear + +vscode: + extensions: + - github.github-vscode-theme + - vscjava.vscode-java-pack + - vscjava.vscode-java-debug + +ports: + # zookeeper + - port: 2181 + onOpen: ignore + visibility: private + # broker + - port: 29092 + onOpen: ignore + visibility: private + # SR + - port: 8081 + onOpen: ignore + visibility: private \ No newline at end of file diff --git a/_data/harnesses/kafka-producer-application/gitpod.yml b/_data/harnesses/kafka-producer-application/gitpod.yml new file mode 100644 index 0000000000..b69d2ef894 --- /dev/null +++ b/_data/harnesses/kafka-producer-application/gitpod.yml @@ -0,0 +1,48 @@ +dev: + steps: + - title: Background + content: + - action: skip + render: + file: tutorials/kafka-producer-application/kafka/markup/gitpod/background.adoc + + - title: Application tour + content: + - action: skip + render: + file: tutorials/kafka-producer-application/kafka/markup/gitpod/app-tour.adoc + + - title: Compile the KafkaProducer application + content: + - action: execute + file: tutorial-steps/gitpod/build-uberjar.sh + render: + file: tutorials/kafka-producer-application/kafka/markup/gitpod/build-uberjar.adoc + + - title: Create a topic + content: + - action: execute + file: tutorial-steps/gitpod/harness-create-topic.sh + render: + file: tutorials/kafka-producer-application/kafka/markup/gitpod/create-topic.adoc + + - title: Run the KafkaProducer application + content: + - action: execute + file: tutorial-steps/gitpod/run-dev-app.sh + render: + file: tutorials/kafka-producer-application/kafka/markup/gitpod/run-dev-app.adoc + + - title: Confirm records sent by consuming from topic + content: + - action: execute_async + file: tutorial-steps/gitpod/harness-console-consumer.sh + stdout: tutorial-steps/gitpod/outputs/actual-output.txt + render: + file: tutorials/kafka-producer-application/kafka/markup/gitpod/run-consumer.adoc + + - title: Next steps + content: + - action: skip + render: + file: tutorials/kafka-producer-application/kafka/markup/gitpod/next-steps.adoc diff --git a/_data/tutorials.yaml b/_data/tutorials.yaml index f89f04c097..5deef41513 100644 --- a/_data/tutorials.yaml +++ b/_data/tutorials.yaml @@ -604,12 +604,12 @@ streams-to-table: kafka: disabled confluent: enabled kafka-producer-application: - title: How to build your first Apache KafkaProducer application + title: How to build your first Apache Kafka® producer application meta-description: build your first Kafka producer application canonical: confluent slug: /creating-first-apache-kafka-producer-application question: How do you get started building your first Kafka producer application? - introduction: You'd like to integrate a KafkaProducer into your event-driven application, + introduction: You'd like to integrate a Kafka producer into your event-driven application, but you're not sure where to start. In this tutorial, you'll build a small application that uses a KafkaProducer to write records to Kafka. status: diff --git a/_includes/gitpod-content.html b/_includes/gitpod-content.html new file mode 100644 index 0000000000..7541747a78 --- /dev/null +++ b/_includes/gitpod-content.html @@ -0,0 +1,48 @@ +
+
+
+
+
+ {% for step in site.data.harnesses[page.static_data][page.stack].dev.steps %} +
+

+ {{ step.title }} +

+
+
+ + {% for section in step.content %} + + {% if section.action == "docker_ksql_cli_session" or section.action == "docker_flinksql_cli_session" %} + + {% if section.render.skip != true %} + {% capture prose %}{% include {{ section.render.file }} %}{% endcapture %} + {{ prose | asciidocify }} + {% endif %} + + {% for subsection in section.stdin %} + {% capture prose %}{% include {{ subsection.render.file }} %}{% endcapture %} + {{ prose | asciidocify }} + {% endfor %} + + {% elsif section.render.skip != true %} + + {% capture prose %}{% include {{ section.render.file }} %}{% endcapture %} + {{ prose | asciidocify }} + + {% endif %} + + {% endfor %} + +
+
+
+ {% endfor %} +
+
+
+
+
diff --git a/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/build-uberjar.sh b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/build-uberjar.sh new file mode 100644 index 0000000000..12ffd144a0 --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/build-uberjar.sh @@ -0,0 +1 @@ +./gradlew shadowJar diff --git a/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/console-consumer.sh b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/console-consumer.sh new file mode 100644 index 0000000000..db8d3f6d59 --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/console-consumer.sh @@ -0,0 +1,5 @@ +kafka-console-consumer --topic output-topic \ + --bootstrap-server broker:9092 \ + --from-beginning \ + --property print.key=true \ + --property key.separator=" : " \ No newline at end of file diff --git a/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/create-topic.sh b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/create-topic.sh new file mode 100644 index 0000000000..1c02750d3e --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/create-topic.sh @@ -0,0 +1 @@ +kafka-topics --create --topic output-topic --bootstrap-server broker:9092 --replication-factor 1 --partitions 1 \ No newline at end of file diff --git a/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/expected-output.txt b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/expected-output.txt new file mode 100644 index 0000000000..ad5592f80f --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/expected-output.txt @@ -0,0 +1,11 @@ +1 : value +2 : words +3 : All Streams +4 : Lead to +5 : Kafka +6 : Go to +7 : Kafka Summit +8 : How can +9 : a 10 ounce +10 : bird carry a +11 : 5lb coconut diff --git a/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/list-topics.sh b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/list-topics.sh new file mode 100644 index 0000000000..cae1e66ef3 --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/list-topics.sh @@ -0,0 +1 @@ +docker exec broker kafka-topics --list --bootstrap-server broker:29092 \ No newline at end of file diff --git a/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/open-docker-shell.sh b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/open-docker-shell.sh new file mode 100644 index 0000000000..4b1bccf79a --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/open-docker-shell.sh @@ -0,0 +1 @@ +docker exec -it broker bash \ No newline at end of file diff --git a/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/run-dev-app.sh b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/run-dev-app.sh new file mode 100644 index 0000000000..fdd2ce3a6c --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/run-dev-app.sh @@ -0,0 +1 @@ +java -jar build/libs/kafka-producer-application-standalone-0.0.1.jar configuration/dev.properties input.txt diff --git a/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/app-tour.adoc b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/app-tour.adoc new file mode 100644 index 0000000000..46e5bd1dab --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/app-tour.adoc @@ -0,0 +1,59 @@ +In the file explorer on the right, navigate to `tutorial-workspace/src/main/java/io/confluent/developer/`.Right click +on `KafkaProducerApplication.java`, and then click `Open to the Side`. + +Let's look at some of the key points of this program. About 20 lines down: + +[source, java] +.KafkaProducerApplication constructor +---- + +public class KafkaProducerApplication { + + private final Producer producer; + final String outTopic; + + public KafkaProducerApplication(final Producer producer, <1> + final String topic) { <2> + this.producer = producer; + outTopic = topic; + } + +---- + +<1> Passing in the `Producer` instance as a constructor parameter. +<2> The topic to write records to + + +In this tutorial you'll inject the dependencies in the `KafkaProducerApplication.main()` method. +Having this thin wrapper class around a `Producer` is not required, but it does help with making our code easier to test. + +(In practice you may want to use a dependency injection framework library, such as the https://spring.io/projects/spring-framework[Spring Framework]). + + +Next let's take a look at the `KafkaProducerApplication.produce` method right below the constructor: +[source, java] +.KafkaProducerApplication.produce +---- +public Future produce(final String message) { + final String[] parts = message.split("-"); <1> + final String key, value; + if (parts.length > 1) { + key = parts[0]; + value = parts[1]; + } else { + key = null; + value = parts[0]; + } + final ProducerRecord producerRecord = new ProducerRecord<>(outTopic, key, value); <2> + return producer.send(producerRecord); <3> +} +---- + +<1> Process the String for sending message +<2> Create the `ProducerRecord` +<3> Send the record to the broker + +The `KafkaProducerApplication.produce` method does some processing on a `String`, and then sends the https://kafka.apache.org/25/javadoc/org/apache/kafka/clients/producer/ProducerRecord.html[`ProducerRecord`]. While this code is a trivial example, it's enough to show the example of using a `KafkaProducer`. +Notice that https://kafka.apache.org/34/javadoc/org/apache/kafka/clients/producer/KafkaProducer.html#send-org.apache.kafka.clients.producer.ProducerRecord-[`KafkaProducer.send`] returns a https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/concurrent/Future.html[Future] with a type of https://kafka.apache.org/34/javadoc/org/apache/kafka/clients/producer/RecordMetadata.html[RecordMetadata]. + +The `KafkaProducer.send` method is asynchronous and returns as soon as the provided record is placed in the buffer of records to be sent to the broker. Once the broker acknowledges that the record has been appended to its log, the broker completes the produce request, which the application receives as `RecordMetadata`—information about the committed message. This tutorial prints the `timestamp` and `offset` for each record sent using the `RecordMetadata` object. Note that calling `Future.get()` for any record will block until the produce request completes. \ No newline at end of file diff --git a/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/background.adoc b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/background.adoc new file mode 100644 index 0000000000..87699c88de --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/background.adoc @@ -0,0 +1,4 @@ +This tutorial walks you through how to write a basic Kafka producer application in Java. + +The complete source code for this tutorial application resides in the `tutorial-workspace` directory of the Gitpod +workspace. Also, Kafka starts up automatically in Docker. \ No newline at end of file diff --git a/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/build-uberjar.adoc b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/build-uberjar.adoc new file mode 100644 index 0000000000..5e7cd455ca --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/build-uberjar.adoc @@ -0,0 +1,5 @@ +In your terminal window on the bottom of the Gitpod workspace, run the following command to build the application: + ++++++ +
{% include_raw tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/build-uberjar.sh %}
++++++ diff --git a/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/create-topic.adoc b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/create-topic.adoc new file mode 100644 index 0000000000..a95f7d521e --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/create-topic.adoc @@ -0,0 +1,20 @@ + +In this step we're going to create a topic for use during this tutorial. Before proceeding, ensure that Kafka is up and running by listing topics: ++++++ +
{% include_raw tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/list-topics.sh %}
++++++ + +You should see the `__consumer_offsets` and `_schemas` topics listed. If you see an error like `No such container: broker`, the images may still be downloading in the terminal named `Docker`. + +When Kafka is up and running, run this command to open a shell on the broker Docker container ++++++ +
{% include_raw tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/open-docker-shell.sh %}
++++++ + +Next, create the topic that the producer can write to + ++++++ +
{% include_raw tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/create-topic.sh %}
++++++ + +Enter `CTRL+D` to exit the broker shell. diff --git a/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/next-steps.adoc b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/next-steps.adoc new file mode 100644 index 0000000000..d0dda32693 --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/next-steps.adoc @@ -0,0 +1,5 @@ +Now that you have a working producer application and a way to test with `kafka-console-consumer`, you may want to modify the application to do something more interesting. For example, the program ends after printing each line from the file. How might you modify it to print a record every second? How would you change it to emit results of an API call every second and never end? Can you figure out how to emit weather data from https://open-meteo.com/[Open-Meteo] every second for your current location and store the results in a `temperature` topic? + +When you are done tinkering, ensure that you shut down your Gitpod workspace so that you don't waste credits or incur cost unecessarily. You can do this by navigating to https://gitpod.io/workspaces[https://gitpod.io/workspaces], clicking the three dots on the right of the running workspace, and clicking `Delete` followed by `Delete Workspace`. + +Once everything is cleaned up, head on back to Confluent Developer to try link:{{ site.url }}[another tutorial]! diff --git a/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/run-consumer.adoc b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/run-consumer.adoc new file mode 100644 index 0000000000..7fd138eb7e --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/run-consumer.adoc @@ -0,0 +1,21 @@ +Now we'll run a console consumer that will read topics from the output topic to confirm your application published the expected records. + +First, open a shell on the broker Docker container: + ++++++ +
{% include_raw tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/open-docker-shell.sh %}
++++++ + +Now run the console consumer: + ++++++ +
{% include_raw tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/console-consumer.sh %}
++++++ + +The output from the consumer can vary if you added any of your own records, but it should look something like this: + +++++ +
{% include_raw tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/expected-output.txt %}
+++++ + +Now close the consumer with a `CTRL+C` then the broker shell with a `CTRL+D` diff --git a/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/run-dev-app.adoc b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/run-dev-app.adoc new file mode 100644 index 0000000000..893e8fed2d --- /dev/null +++ b/_includes/tutorials/kafka-producer-application/kafka/markup/gitpod/run-dev-app.adoc @@ -0,0 +1,27 @@ +Now that you have an uberjar for the KafkaProducerApplication, you can launch it in the workspace. ++++++ +
{% include_raw tutorials/kafka-producer-application/kafka/code/tutorial-steps/gitpod/run-dev-app.sh %}
++++++ + +After you run the previous command, the application will process the file and you should something like this on the console: + +[source, text] +---- +Offsets and timestamps committed in batch from input.txt +Record written to offset 0 timestamp 1597352120029 +Record written to offset 1 timestamp 1597352120037 +Record written to offset 2 timestamp 1597352120037 +Record written to offset 3 timestamp 1597352120037 +Record written to offset 4 timestamp 1597352120037 +Record written to offset 5 timestamp 1597352120037 +Record written to offset 6 timestamp 1597352120037 +Record written to offset 7 timestamp 1597352120037 +Record written to offset 8 timestamp 1597352120037 +Record written to offset 9 timestamp 1597352120037 +Record written to offset 10 timestamp 1597352120038 +---- + +Now you can experiment some by creating your own file in base directory and re-run the above command and substitute your file name for `input.txt` + +Remember any data before the `-` is the key and data after is the value. + diff --git a/_layouts/gitpod.html b/_layouts/gitpod.html new file mode 100644 index 0000000000..267479fea3 --- /dev/null +++ b/_layouts/gitpod.html @@ -0,0 +1,73 @@ +--- +--- + + + + + {% include vendors.html %} + + + + + + + + {% if site.data.tutorials[page.static_data].canonical %} + {% unless page.stack == site.data.tutorials[page.static_data].canonical or page.stack == 'ksql' %} + + {% endunless %} + {% endif %} + + {{ site.data.tutorials[page.static_data].title }} using {{ site.data.stacks[page.stack].pretty }} + + + + + + + + + + + + + {% include feedback-button.html %} + +
+
+
+
+

{{ site.data.tutorials[page.static_data].title }}

+
+
+
+
+ +
+ {% if page.help_wanted %} + {% include contribute-tutorial.html %} + {% else %} + {% if page.help_wanted_lang %} + {% include contribute-tutorial-lang.html %} + {% else %} + {% include gitpod-content.html %} + {% endif %} + {% endif %} +
+ + + + + + + + + + + + diff --git a/assets/sass/tutorial.scss b/assets/sass/tutorial.scss index ff6555c1cd..e881b6c819 100644 --- a/assets/sass/tutorial.scss +++ b/assets/sass/tutorial.scss @@ -66,6 +66,10 @@ padding-top: 0; } + .hero-gitpod { + min-height: 150px !important; + } + .hero { min-height: 265px; padding-top: 0; diff --git a/tutorials/kafka-producer-application/gitpod.html b/tutorials/kafka-producer-application/gitpod.html new file mode 100644 index 0000000000..8fac1fb50b --- /dev/null +++ b/tutorials/kafka-producer-application/gitpod.html @@ -0,0 +1,6 @@ +--- +layout: gitpod +permalink: /creating-first-apache-kafka-producer-application/gitpod +stack: gitpod +static_data: kafka-producer-application +---