Merge branch 'master' into prod

Author: Lee-W

airflow.cfg

@@ -2,7 +2,7 @@
 # The amount of parallelism as a setting to the executor. This defines
 # the max number of task instances that should run simultaneously
 # on this airflow installation
-parallelism = 256
+parallelism = 4
 
 # The number of task instances allowed to run concurrently by the scheduler
 max_active_tasks_per_dag = 64
@@ -20,12 +20,19 @@ dagbag_import_timeout = 600
 
 auth_manager = airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
 
+max_active_runs_per_dag = 2
+max_active_tasks_per_dag = 2
+
 [dag_processor]
 # How long before timing out a DagFileProcessor, which processes a dag file
 dag_file_processor_timeout = 600
 # We don't really need it based on how we deploy dags.
-# But set it to a large enough 60 * 24 (1 day) to avoid high CPU usage
-min_file_process_interval = 1440
+# But set it to a large enough 60 * 60 * 24 * 7 (1 week) to avoid high CPU usage
+min_file_process_interval = 604800
+
+parsing_processes = 1
+
+refresh_interval = 600
 
 [database]
 # The SqlAlchemy connection string to the metadata database.
@@ -59,6 +66,9 @@ auth_backends = airflow.providers.fab.auth_manager.api.auth.backend.session
 
 [scheduler]
 enable_health_check = True
+scheduler_heartbeat_sec = 10
+orphaned_tasks_check_interval = 600
+parsing_cleanup_interval = 600
 
 # ref: https://airflow.apache.org/docs/apache-airflow/1.10.1/security.html#setting-up-google-authentication
 [google]