Skip to content

add readiness and liveliness support for git sync relay sidecars #50218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

add readiness and liveliness support for git sync relay sidecars #50218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
852496b
add liveliness and readiness probes support in git sync template
pgvishnuram May 5, 2025
b491fe9
update templates
pgvishnuram May 5, 2025
94c88c4
add tests
pgvishnuram May 5, 2025
8e09a39
add git sync worker tests
pgvishnuram May 5, 2025
6baf86d
add webserver git sync tests
pgvishnuram May 5, 2025
df9ce07
fix pre-commit
pgvishnuram May 5, 2025
203a1d0
fix webserver tests
pgvishnuram May 5, 2025
a0d8300
update scheduler tests
pgvishnuram May 5, 2025
d047c67
fix ci precommit failures
pgvishnuram May 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions chart/templates/_helpers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,12 @@ If release name contains chart name it will be used as a full name.
{{- toYaml . | nindent 4 }}
{{- end }}
resources: {{ toYaml .Values.dags.gitSync.resources | nindent 4 }}
{{- if and .Values.dags.gitSync.livenessProbe (not .is_init) }}
livenessProbe: {{ tpl (toYaml .Values.dags.gitSync.livenessProbe) . | nindent 4 }}
{{- end }}
{{- if and .Values.dags.gitSync.readinessProbe (not .is_init) }}
readinessProbe: {{ tpl (toYaml .Values.dags.gitSync.readinessProbe) . | nindent 4 }}
{{- end }}
volumeMounts:
- name: dags
mountPath: /git
Expand Down
10 changes: 10 additions & 0 deletions chart/values.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8814,6 +8814,16 @@
}
]
},
"livenessProbe": {
"description": "Liveness probe configuration for git sync container.",
"type": "object",
"$ref": "#/definitions/io.k8s.api.core.v1.Probe"
},
"readinessProbe": {
"description": "Readiness probe configuration for git sync container.",
"type": "object",
"$ref": "#/definitions/io.k8s.api.core.v1.Probe"
},
"emptyDirConfig": {
"description": "Configuration for dags empty dir volume.",
"type": "object",
Expand Down
3 changes: 3 additions & 0 deletions chart/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2899,6 +2899,9 @@ dags:
# container level lifecycle hooks
containerLifecycleHooks: {}

readinessProbe: {}
livenessProbe: {}

# Mount additional volumes into git-sync. It can be templated like in the following example:
# extraVolumeMounts:
# - name: my-templated-extra-volume
Expand Down
43 changes: 43 additions & 0 deletions helm-tests/tests/helm_tests/other/test_git_sync_scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,3 +408,46 @@ def test_resources_are_configurable(self):
jmespath.search("spec.template.spec.containers[1].resources.requests.memory", docs[0]) == "169Mi"
)
assert jmespath.search("spec.template.spec.containers[1].resources.requests.cpu", docs[0]) == "300m"

def test_liveliness_and_readiness_probes_are_configurable(self):
livenessProbe = {
"failureThreshold": 10,
"exec": {"command": ["/bin/true"]},
"initialDelaySeconds": 0,
"periodSeconds": 1,
"successThreshold": 1,
"timeoutSeconds": 5,
}
readinessProbe = {
"failureThreshold": 10,
"exec": {"command": ["/bin/true"]},
"initialDelaySeconds": 0,
"periodSeconds": 1,
"successThreshold": 1,
"timeoutSeconds": 5,
}
docs = render_chart(
values={
"airflowVersion": "2.10.5",
"dags": {
"gitSync": {
"enabled": True,
"livenessProbe": livenessProbe,
"readinessProbe": readinessProbe,
},
},
},
show_only=["templates/scheduler/scheduler-deployment.yaml"],
)
container_search_result = jmespath.search(
"spec.template.spec.containers[?name == 'git-sync']", docs[0]
)
init_container_search_result = jmespath.search(
"spec.template.spec.initContainers[?name == 'git-sync-init']", docs[0]
)
assert "livenessProbe" in container_search_result[0]
assert "readinessProbe" in container_search_result[0]
assert "readinessProbe" not in init_container_search_result[0]
assert "readinessProbe" not in init_container_search_result[0]
assert livenessProbe == container_search_result[0]["livenessProbe"]
assert readinessProbe == container_search_result[0]["readinessProbe"]
42 changes: 42 additions & 0 deletions helm-tests/tests/helm_tests/other/test_git_sync_triggerer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,3 +76,45 @@ def test_validate_if_ssh_params_are_added_with_git_ssh_key(self):
"name": "git-sync-ssh-key",
"secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288},
} in jmespath.search("spec.template.spec.volumes", docs[0])

def test_liveliness_and_readiness_probes_are_configurable(self):
livenessProbe = {
"failureThreshold": 10,
"exec": {"command": ["/bin/true"]},
"initialDelaySeconds": 0,
"periodSeconds": 1,
"successThreshold": 1,
"timeoutSeconds": 5,
}
readinessProbe = {
"failureThreshold": 10,
"exec": {"command": ["/bin/true"]},
"initialDelaySeconds": 0,
"periodSeconds": 1,
"successThreshold": 1,
"timeoutSeconds": 5,
}
docs = render_chart(
values={
"dags": {
"gitSync": {
"enabled": True,
"livenessProbe": livenessProbe,
"readinessProbe": readinessProbe,
},
}
},
show_only=["templates/triggerer/triggerer-deployment.yaml"],
)
container_search_result = jmespath.search(
"spec.template.spec.containers[?name == 'git-sync']", docs[0]
)
init_container_search_result = jmespath.search(
"spec.template.spec.initContainers[?name == 'git-sync-init']", docs[0]
)
assert "livenessProbe" in container_search_result[0]
assert "readinessProbe" in container_search_result[0]
assert "readinessProbe" not in init_container_search_result[0]
assert "readinessProbe" not in init_container_search_result[0]
assert livenessProbe == container_search_result[0]["livenessProbe"]
assert readinessProbe == container_search_result[0]["readinessProbe"]
44 changes: 44 additions & 0 deletions helm-tests/tests/helm_tests/other/test_git_sync_webserver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,3 +196,47 @@ def test_validate_if_ssh_params_are_added_with_git_ssh_key(self):
"name": "git-sync-ssh-key",
"secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288},
} in jmespath.search("spec.template.spec.volumes", docs[0])

def test_liveliness_and_readiness_probes_are_configurable(self):
"""If Airflow < 2.0.0 - test git sync related containers, volume mounts & volumes are created."""
livenessProbe = {
"failureThreshold": 10,
"exec": {"command": ["/bin/true"]},
"initialDelaySeconds": 0,
"periodSeconds": 1,
"successThreshold": 1,
"timeoutSeconds": 5,
}
readinessProbe = {
"failureThreshold": 10,
"exec": {"command": ["/bin/true"]},
"initialDelaySeconds": 0,
"periodSeconds": 1,
"successThreshold": 1,
"timeoutSeconds": 5,
}
docs = render_chart(
values={
"airflowVersion": "1.10.14",
"dags": {
"gitSync": {
"enabled": True,
"livenessProbe": livenessProbe,
"readinessProbe": readinessProbe,
},
},
},
show_only=["templates/webserver/webserver-deployment.yaml"],
)
container_search_result = jmespath.search(
"spec.template.spec.containers[?name == 'git-sync']", docs[0]
)
init_container_search_result = jmespath.search(
"spec.template.spec.initContainers[?name == 'git-sync-init']", docs[0]
)
assert "livenessProbe" in container_search_result[0]
assert "readinessProbe" in container_search_result[0]
assert "readinessProbe" not in init_container_search_result[0]
assert "readinessProbe" not in init_container_search_result[0]
assert livenessProbe == container_search_result[0]["livenessProbe"]
assert readinessProbe == container_search_result[0]["readinessProbe"]
42 changes: 42 additions & 0 deletions helm-tests/tests/helm_tests/other/test_git_sync_worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,3 +201,45 @@ def test_container_lifecycle_hooks(self):
},
"preStop": {"exec": {"command": ["/bin/sh", "-c", "echo preStop handler > /git/message_start"]}},
}

def test_liveliness_and_readiness_probes_are_configurable(self):
livenessProbe = {
"failureThreshold": 10,
"exec": {"command": ["/bin/true"]},
"initialDelaySeconds": 0,
"periodSeconds": 1,
"successThreshold": 1,
"timeoutSeconds": 5,
}
readinessProbe = {
"failureThreshold": 10,
"exec": {"command": ["/bin/true"]},
"initialDelaySeconds": 0,
"periodSeconds": 1,
"successThreshold": 1,
"timeoutSeconds": 5,
}
docs = render_chart(
values={
"dags": {
"gitSync": {
"enabled": True,
"livenessProbe": livenessProbe,
"readinessProbe": readinessProbe,
},
}
},
show_only=["templates/workers/worker-deployment.yaml"],
)
container_search_result = jmespath.search(
"spec.template.spec.containers[?name == 'git-sync']", docs[0]
)
init_container_search_result = jmespath.search(
"spec.template.spec.initContainers[?name == 'git-sync-init']", docs[0]
)
assert "livenessProbe" in container_search_result[0]
assert "readinessProbe" in container_search_result[0]
assert "readinessProbe" not in init_container_search_result[0]
assert "readinessProbe" not in init_container_search_result[0]
assert livenessProbe == container_search_result[0]["livenessProbe"]
assert readinessProbe == container_search_result[0]["readinessProbe"]