Skip to content

Commit 983979c

Browse files
committed
fix(airflow.cfg): update dag_processor and scheduler configs to decrease CPU loading
1 parent 51495a4 commit 983979c

File tree

1 file changed

+9
-1
lines changed

1 file changed

+9
-1
lines changed

airflow.cfg

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# The amount of parallelism as a setting to the executor. This defines
33
# the max number of task instances that should run simultaneously
44
# on this airflow installation
5-
parallelism = 256
5+
parallelism = 4
66

77
# The number of task instances allowed to run concurrently by the scheduler
88
max_active_tasks_per_dag = 64
@@ -20,6 +20,9 @@ dagbag_import_timeout = 600
2020

2121
auth_manager = airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
2222

23+
max_active_runs_per_dag = 2
24+
max_active_tasks_per_dag = 2
25+
2326
[dag_processor]
2427
# How long before timing out a DagFileProcessor, which processes a dag file
2528
dag_file_processor_timeout = 600
@@ -29,6 +32,8 @@ min_file_process_interval = 604800
2932

3033
parsing_processes = 1
3134

35+
refresh_interval = 600
36+
3237
[database]
3338
# The SqlAlchemy connection string to the metadata database.
3439
# SqlAlchemy supports many different database engine, more information
@@ -61,6 +66,9 @@ auth_backends = airflow.providers.fab.auth_manager.api.auth.backend.session
6166

6267
[scheduler]
6368
enable_health_check = True
69+
scheduler_heartbeat_sec = 10
70+
orphaned_tasks_check_interval = 600
71+
parsing_cleanup_interval = 600
6472

6573
# ref: https://airflow.apache.org/docs/apache-airflow/1.10.1/security.html#setting-up-google-authentication
6674
[google]

0 commit comments

Comments
 (0)