Skip to content

Commit 457870b

Browse files
davetroianodavetroiano
authored
feat: add Flink SQL cumulating windows tutorial (#1633)
1 parent 0fdb2b2 commit 457870b

File tree

59 files changed

+1161
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+1161
-0
lines changed

‎.semaphore/semaphore.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,9 @@ blocks:
505505
- name: Flink SQL hopping windows tests
506506
commands:
507507
- make -C _includes/tutorials/hopping-windows/flinksql/code tutorial
508+
- name: Flink SQL cumulating windows tests
509+
commands:
510+
- make -C _includes/tutorials/cumulating-windows/flinksql/code tutorial
508511
- name: Flink SQL test for deduplication
509512
commands:
510513
- make -C _includes/tutorials/finding-distinct/flinksql/code tutorial

‎_data/harnesses/cumulating-windows/flinksql.yml

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
dev:
2+
steps:
3+
- title: Prerequisites
4+
content:
5+
- action: skip
6+
render:
7+
file: shared/markup/dev/docker-prerequisite.adoc
8+
9+
- title: Initialize the project
10+
content:
11+
- action: execute
12+
file: tutorial-steps/dev/init.sh
13+
render:
14+
file: tutorials/cumulating-windows/flinksql/markup/dev/init.adoc
15+
16+
- title: Get Confluent Platform and Apache Flink®
17+
content:
18+
- action: make_file
19+
file: docker-compose.yml
20+
render:
21+
file: tutorials/cumulating-windows/flinksql/markup/dev/make-docker-compose.adoc
22+
23+
- action: execute_async
24+
file: tutorial-steps/dev/docker-compose-up.sh
25+
render:
26+
file: tutorials/cumulating-windows/flinksql/markup/dev/start-compose.adoc
27+
28+
- action: execute
29+
file: tutorial-steps/dev/wait-for-containers.sh
30+
render:
31+
skip: true
32+
33+
- title: Write the program interactively using the CLI
34+
content:
35+
- action: docker_flinksql_cli_session
36+
container: flink-sql-client
37+
docker_bootup_file: tutorial-steps/dev/start-cli.sh
38+
column_width: 20
39+
render:
40+
file: tutorials/cumulating-windows/flinksql/markup/dev/start-cli.adoc
41+
stdin:
42+
- file: tutorial-steps/dev/create-orders.sql
43+
render:
44+
file: tutorials/cumulating-windows/flinksql/markup/dev/create-orders.adoc
45+
46+
- file: tutorial-steps/dev/populate-orders.sql
47+
render:
48+
file: tutorials/cumulating-windows/flinksql/markup/dev/populate-orders.adoc
49+
50+
- file: tutorial-steps/dev/transient-query-revenue-per-hour-cumulating.sql
51+
render:
52+
file: tutorials/cumulating-windows/flinksql/markup/dev/transient-query-revenue-per-hour-cumulating.adoc
53+
54+
- file: tutorial-steps/dev/create-revenue-per-hour-cumulating.sql
55+
render:
56+
file: tutorials/cumulating-windows/flinksql/markup/dev/create-revenue-per-hour-cumulating.adoc
57+
58+
stdout:
59+
directory: tutorial-steps/dev/outputs
60+
61+
- name: wait for table execution to complete
62+
action: sleep
63+
ms: 10000
64+
render:
65+
skip: true
66+
67+
- title: Validate output
68+
content:
69+
- action: execute
70+
file: tutorial-steps/dev/validate-revenue-per-hour-cumulating.sh
71+
stdout: tutorial-steps/dev/outputs/validate-revenue-per-hour-cumulating.log
72+
render:
73+
file: tutorials/cumulating-windows/flinksql/markup/dev/validate-revenue-per-hour-cumulating.adoc
74+
75+
test:
76+
steps:
77+
- title: Decide what testing tools to use
78+
content:
79+
- action: skip
80+
render:
81+
file: tutorials/cumulating-windows/flinksql/markup/test/test-architecture.adoc
82+
83+
- title: Create the test skeleton
84+
content:
85+
- action: execute
86+
file: tutorial-steps/test/make-test-dirs.sh
87+
render:
88+
file: tutorials/cumulating-windows/flinksql/markup/test/make-test-dirs.adoc
89+
90+
- action: make_file
91+
file: build.gradle
92+
render:
93+
file: tutorials/cumulating-windows/flinksql/markup/test/make-build-gradle.adoc
94+
95+
- action: execute
96+
file: tutorial-steps/test/gradle-wrapper.sh
97+
render:
98+
file: tutorials/cumulating-windows/flinksql/markup/test/make-gradle-wrapper.adoc
99+
100+
- title: Create SQL resources
101+
content:
102+
- action: make_file
103+
file: src/test/resources/create-orders.sql.template
104+
render:
105+
file: tutorials/cumulating-windows/flinksql/markup/test/create-resource-create-orders.sql.template.adoc
106+
107+
- action: make_file
108+
file: src/test/resources/populate-orders.sql
109+
render:
110+
file: tutorials/cumulating-windows/flinksql/markup/test/create-resource-populate-orders.sql.adoc
111+
112+
- action: make_file
113+
file: src/test/resources/create-revenue-per-hour-cumulating.sql.template
114+
render:
115+
file: tutorials/cumulating-windows/flinksql/markup/test/create-resource-create-revenue-per-hour-cumulating.sql.template.adoc
116+
117+
- action: make_file
118+
file: src/test/resources/query-revenue-per-hour-cumulating.sql
119+
render:
120+
file: tutorials/cumulating-windows/flinksql/markup/test/create-resource-query-revenue-per-hour-cumulating.sql.adoc
121+
122+
- action: make_file
123+
file: src/test/resources/expected-revenue-per-hour-cumulating.txt
124+
render:
125+
file: tutorials/cumulating-windows/flinksql/markup/test/create-resource-expected-revenue-per-hour-cumulating.txt.adoc
126+
127+
- title: Write a test
128+
content:
129+
- action: make_file
130+
file: src/test/java/io/confluent/developer/AbstractFlinkKafkaTest.java
131+
render:
132+
file: tutorials/cumulating-windows/flinksql/markup/test/make-test-base.adoc
133+
134+
- action: make_file
135+
file: src/test/java/io/confluent/developer/FlinkSqlCumulatingWindowTest.java
136+
render:
137+
file: tutorials/cumulating-windows/flinksql/markup/test/make-test.adoc
138+
139+
- title: Invoke the test
140+
content:
141+
- action: execute
142+
file: tutorial-steps/test/invoke-test.sh
143+
render:
144+
file: tutorials/cumulating-windows/flinksql/markup/test/invoke-test.adoc
145+
146+
ccloud:
147+
steps:
148+
- title: Run your app with Confluent Cloud
149+
content:
150+
- action: skip
151+
render:
152+
file: shared/markup/ccloud/try-ccloud.adoc

‎_data/tutorials.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,19 @@ hopping-windows:
180180
status:
181181
ksql: enabled
182182
flinksql: enabled
183+
cumulating-windows:
184+
title: How to create cumulating windows
185+
meta-description: create cumulating windows for time-series events
186+
slug: /create-cumulating-windows
187+
question: If you have time series events in a Kafka topic, how can you simultaneously group them
188+
into fixed-size tumbling windows in addition to splitting each tumbling window into several windows having the same
189+
window start but having window ends that vary by a fixed step size?
190+
introduction: You want to build an hourly revenue dashboard that also outputs revenue at the half hour mark so that
191+
you can simultaneously look at the hourly trend alongside any trends within the hour (e.g., does revenue
192+
peak earlier in the hour or later?). In this tutorial, we'll write a streaming application that aggregates revenue from
193+
a table of orders and outputs hourly revenue in addition to revenue at the end of the first half hour of each hour.
194+
status:
195+
flinksql: enabled
183196
session-windows:
184197
title: Create session windows
185198
meta-description: create session windows for time-series events

‎_includes/tutorials/cumulating-windows/flinksql/code/.gitignore

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
tutorial-steps/dev/outputs/
2+
3+
# Ignore Gradle project-specific cache directory
4+
.gradle
5+
6+
# Ignore Gradle build output directory
7+
build

‎_includes/tutorials/cumulating-windows/flinksql/code/Makefile

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
STEPS_DIR := tutorial-steps
2+
DEV_OUTPUTS_DIR := $(STEPS_DIR)/dev/outputs
3+
TEMP_DIR := $(shell mktemp -d)
4+
SEQUENCE := "dev, test, ccloud"
5+
6+
tutorial:
7+
rm -r $(DEV_OUTPUTS_DIR) || true
8+
mkdir $(DEV_OUTPUTS_DIR)
9+
harness-runner ../../../../../_data/harnesses/cumulating-windows/flinksql.yml $(TEMP_DIR) $(SEQUENCE)
10+
diff --strip-trailing-cr $(STEPS_DIR)/dev/expected-revenue-per-hour-cumulating.log $(DEV_OUTPUTS_DIR)/validate-revenue-per-hour-cumulating.log
11+
reset

‎_includes/tutorials/cumulating-windows/flinksql/code/build.gradle

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
buildscript {
2+
repositories {
3+
mavenCentral()
4+
}
5+
}
6+
7+
plugins {
8+
id "java"
9+
}
10+
11+
sourceCompatibility = JavaVersion.VERSION_11
12+
targetCompatibility = JavaVersion.VERSION_11
13+
version = "0.0.1"
14+
15+
repositories {
16+
mavenCentral()
17+
}
18+
19+
dependencies {
20+
testImplementation "com.google.guava:guava:31.1-jre"
21+
testImplementation "junit:junit:4.13.2"
22+
testImplementation 'org.testcontainers:testcontainers:1.17.6'
23+
testImplementation 'org.testcontainers:kafka:1.17.6'
24+
testImplementation "org.apache.flink:flink-sql-connector-kafka:1.16.0"
25+
testImplementation "org.apache.flink:flink-sql-avro-confluent-registry:1.16.0"
26+
testImplementation "org.apache.flink:flink-test-utils:1.16.0"
27+
testImplementation "org.apache.flink:flink-test-utils-junit:1.16.0"
28+
testImplementation "org.apache.flink:flink-table-api-java-bridge:1.16.0"
29+
testImplementation "org.apache.flink:flink-table-planner_2.12:1.16.0"
30+
testImplementation "org.apache.flink:flink-table-planner_2.12:1.16.0:tests"
31+
testImplementation "org.apache.flink:flink-statebackend-rocksdb:1.16.0"
32+
}

‎_includes/tutorials/cumulating-windows/flinksql/code/docker-compose.yml

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
version: '2'
2+
services:
3+
broker:
4+
image: confluentinc/cp-kafka:7.4.1
5+
hostname: broker
6+
container_name: broker
7+
ports:
8+
- 29092:29092
9+
environment:
10+
KAFKA_BROKER_ID: 1
11+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
12+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092
13+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
14+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
15+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
16+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
17+
KAFKA_PROCESS_ROLES: broker,controller
18+
KAFKA_NODE_ID: 1
19+
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker:29093
20+
KAFKA_LISTENERS: PLAINTEXT://broker:9092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:29092
21+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
22+
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
23+
KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
24+
CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
25+
schema-registry:
26+
image: confluentinc/cp-schema-registry:7.3.0
27+
hostname: schema-registry
28+
container_name: schema-registry
29+
depends_on:
30+
- broker
31+
ports:
32+
- 8081:8081
33+
environment:
34+
SCHEMA_REGISTRY_HOST_NAME: schema-registry
35+
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker:9092
36+
flink-sql-client:
37+
image: cnfldemos/flink-sql-client-kafka:1.16.0-scala_2.12-java11
38+
hostname: flink-sql-client
39+
container_name: flink-sql-client
40+
depends_on:
41+
- flink-jobmanager
42+
environment:
43+
FLINK_JOBMANAGER_HOST: flink-jobmanager
44+
volumes:
45+
- ./settings/:/settings
46+
flink-jobmanager:
47+
image: cnfldemos/flink-kafka:1.16.0-scala_2.12-java11
48+
hostname: flink-jobmanager
49+
container_name: flink-jobmanager
50+
ports:
51+
- 9081:9081
52+
command: jobmanager
53+
environment:
54+
- |
55+
FLINK_PROPERTIES=
56+
jobmanager.rpc.address: flink-jobmanager
57+
rest.bind-port: 9081
58+
flink-taskmanager:
59+
image: cnfldemos/flink-kafka:1.16.0-scala_2.12-java11
60+
hostname: flink-taskmanager
61+
container_name: flink-taskmanager
62+
depends_on:
63+
- flink-jobmanager
64+
command: taskmanager
65+
scale: 1
66+
environment:
67+
- |
68+
FLINK_PROPERTIES=
69+
jobmanager.rpc.address: flink-jobmanager
70+
taskmanager.numberOfTaskSlots: 10

‎_includes/tutorials/cumulating-windows/flinksql/code/gradle/wrapper/gradle-wrapper.jar

59.3 KB
Binary file not shown.

‎_includes/tutorials/cumulating-windows/flinksql/code/gradle/wrapper/gradle-wrapper.properties

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
distributionBase=GRADLE_USER_HOME
2+
distributionPath=wrapper/dists
3+
distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
4+
zipStoreBase=GRADLE_USER_HOME
5+
zipStorePath=wrapper/dists

0 commit comments

Comments
 (0)