build(docker-compose): setup health check for each airflow component

Author: Lee-W

docker-compose-dev.yml

@@ -8,6 +8,8 @@ x-docker-common: &docker-common
     - ./sqlite:/opt/airflow/sqlite
     # you can comment out the following line if you don't have service-account.json
     - ./service-account.json:/opt/airflow/service-account.json
+  environment:
+    AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-api-server:8080/execution/'
   logging:
     driver: json-file
     options:
@@ -21,7 +23,7 @@ services:
       - "8080:8080"
     command: api-server
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+      test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/version"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -35,6 +37,12 @@ services:
     <<: *docker-common
     container_name: airflow-dag-processor
     command: dag-processor
+    healthcheck:
+      test: ["CMD-SHELL", 'airflow jobs check --job-type DagProcessorJob --hostname "$${HOSTNAME}"']
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
     restart: always
     depends_on:
       airflow-init:
@@ -44,6 +52,12 @@ services:
     <<: *docker-common
     container_name: airflow-scheduler
     command: scheduler
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
     restart: always
     depends_on:
       airflow-init:
@@ -58,7 +72,6 @@ services:
         if [[ -n "$${_AIRFLOW_DB_MIGRATE=}" ]]; then
             airflow db migrate || true
         fi
-        exit 0
     restart: "no"
     environment:
       _AIRFLOW_DB_MIGRATE: 'false'

docker-compose.yml

@@ -18,7 +18,7 @@ services:
       - "8080:8080"
     command: api-server
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+      test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/version"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -32,6 +32,12 @@ services:
     <<: *docker-common
     container_name: airflow-dag-processor
     command: dag-processor
+    healthcheck:
+      test: ["CMD-SHELL", 'airflow jobs check --job-type DagProcessorJob --hostname "$${HOSTNAME}"']
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
     restart: always
     depends_on:
       airflow-init:
@@ -41,6 +47,12 @@ services:
     <<: *docker-common
     container_name: airflow-scheduler
     command: scheduler
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
     restart: always
     depends_on:
       airflow-init: