Skip to content

Commit 5cb377b

Browse files
xeniaperazvan
xeniape
and
razvan
authored
chore: add overrides integrationtests (#490)
* chore: add hint for pod template * recapture pod_template to immutable * add overrides integrationtest * fix overrides integrationtest * bump stackable-operator version * add todo key * remove recapturing for now * remove recapturing for now * test env overrides for celery executors * reorganize steps * test env overrides for k8s executors * fix yaml lint --------- Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
1 parent 9ce3aa7 commit 5cb377b

13 files changed

+435
-154
lines changed

‎Cargo.lock

Lines changed: 76 additions & 69 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.nix

Lines changed: 103 additions & 82 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] }
2222
serde_json = "1.0"
2323
serde_yaml = "0.9"
2424
snafu = "0.8"
25-
stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.73.0" }
25+
stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" }
2626
strum = { version = "0.26", features = ["derive"] }
2727
tokio = { version = "1.39", features = ["full"] }
2828
tracing = "0.1"

‎crate-hashes.json

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/templates/kuttl/overrides/00-assert.yaml.j2

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
apiVersion: apps/v1
2+
kind: StatefulSet
3+
metadata:
4+
name: airflow-postgresql
5+
status:
6+
availableReplicas: 1
7+
---
8+
apiVersion: apps/v1
9+
kind: StatefulSet
10+
metadata:
11+
name: airflow-redis-master
12+
status:
13+
availableReplicas: 1
14+
---
15+
apiVersion: apps/v1
16+
kind: StatefulSet
17+
metadata:
18+
name: airflow-redis-replicas
19+
status:
20+
availableReplicas: 1

‎tests/templates/kuttl/overrides/00-install-dbs.yaml.j2

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestStep
4+
commands:
5+
- script: >-
6+
helm install airflow-postgresql
7+
--namespace $NAMESPACE
8+
--version 12.1.5
9+
--set auth.username=airflow
10+
--set auth.password=airflow
11+
--set auth.database=airflow
12+
--repo https://charts.bitnami.com/bitnami postgresql
13+
timeout: 240
14+
- script: >-
15+
helm install airflow-redis
16+
--namespace $NAMESPACE
17+
--version 17.3.7
18+
--set auth.password=redis
19+
--set replica.replicaCount=1
20+
--repo https://charts.bitnami.com/bitnami redis
21+
timeout: 240

‎tests/templates/kuttl/overrides/10-assert.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
metadata:
5+
name: test-available-condition
6+
timeout: 600
7+
commands:
8+
- script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-celery --timeout 301s

‎tests/templates/kuttl/overrides/10-install-airflow.yaml.j2

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
---
2+
apiVersion: v1
3+
kind: Secret
4+
metadata:
5+
name: airflow-credentials
6+
type: Opaque
7+
stringData:
8+
adminUser.username: airflow
9+
adminUser.firstname: Airflow
10+
adminUser.lastname: Admin
11+
adminUser.email: airflow@airflow.com
12+
adminUser.password: airflow
13+
connections.secretKey: thisISaSECRET_1234
14+
connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow
15+
connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow
16+
connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0
17+
---
18+
apiVersion: v1
19+
kind: Secret
20+
metadata:
21+
name: test-override
22+
type: Opaque
23+
stringData:
24+
adminUser.username: airflow
25+
adminUser.firstname: Airflow
26+
adminUser.lastname: Admin
27+
adminUser.email: airflow@airflow.com
28+
adminUser.password: airflow
29+
connections.secretKey: thisISaSECRET_1234
30+
connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow
31+
connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow
32+
connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0
33+
---
34+
apiVersion: airflow.stackable.tech/v1alpha1
35+
kind: AirflowCluster
36+
metadata:
37+
name: airflow-celery
38+
spec:
39+
image:
40+
productVersion: "{{ test_scenario['values']['airflow-latest'] }}"
41+
clusterConfig:
42+
loadExamples: true
43+
exposeConfig: false
44+
listenerClass: external-unstable
45+
credentialsSecret: airflow-credentials
46+
webservers:
47+
envOverrides:
48+
COMMON_VAR: role-value # overridden by role group below
49+
ROLE_VAR: role-value # only defined here at role level
50+
credentialsSecret: test-override
51+
roleGroups:
52+
default:
53+
replicas: 1
54+
envOverrides:
55+
COMMON_VAR: group-value # overrides role value
56+
GROUP_VAR: group-value # only defined here at group level
57+
celeryExecutors:
58+
envOverrides:
59+
COMMON_VAR: role-value # overridden by role group below
60+
ROLE_VAR: role-value # only defined here at role level
61+
roleGroups:
62+
default:
63+
replicas: 2
64+
config:
65+
resources:
66+
cpu:
67+
min: 410m
68+
podOverrides:
69+
spec:
70+
containers:
71+
- name: airflow
72+
resources:
73+
limits:
74+
cpu: 810m
75+
envOverrides:
76+
COMMON_VAR: group-value # overrides role value
77+
GROUP_VAR: group-value # only defined here at group level
78+
config:
79+
resources:
80+
cpu:
81+
min: 400m
82+
max: 800m
83+
memory:
84+
limit: 2Gi
85+
podOverrides:
86+
spec:
87+
containers:
88+
- name: airflow
89+
resources:
90+
limits:
91+
cpu: 750m
92+
requests:
93+
cpu: 350m
94+
schedulers:
95+
roleGroups:
96+
default:
97+
replicas: 1
98+
envOverrides:
99+
credentialsSecret: test-override

‎tests/templates/kuttl/overrides/11-assert.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 600
5+
commands:
6+
#
7+
# Test envOverrides
8+
#
9+
- script: |
10+
kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
11+
kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
12+
kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
13+
kubectl -n $NAMESPACE get sts airflow-celery-webserver-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "credentialsSecret" and .value == "test-override")'
14+
15+
16+
- script: |
17+
kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
18+
kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
19+
kubectl -n $NAMESPACE get sts airflow-celery-worker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "airflow") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'

‎tests/templates/kuttl/overrides/20-assert.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
metadata:
5+
name: test-available-condition
6+
timeout: 600
7+
commands:
8+
- script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow-kubernetes --timeout 301s

0 commit comments

Comments
 (0)