fix(airflow.cfg): set min_file_process_interval to a large enough time and lower api.workers to 1

api.wokers > 2 does not work for our limited machine resources

Author: Lee-W

airflow.cfg

@@ -23,6 +23,9 @@ auth_manager = airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManage
 [dag_processor]
 # How long before timing out a DagFileProcessor, which processes a dag file
 dag_file_processor_timeout = 600
+# We don't really need it based on how we deploy dags.
+# But set it to a large enough 60 * 24 (1 day) to avoid high CPU usage
+min_file_process_interval = 1440
 
 [database]
 # The SqlAlchemy connection string to the metadata database.
@@ -34,7 +37,8 @@ external_db_managers = airflow.providers.fab.auth_manager.models.db.FABDBManager
 
 [api]
 # Number of workers to run the Gunicorn web server
-workers = 2
+# WARNING:: DO NOT increase this number. Due to our limited resources, increasing it to 2 breaks API server
+workers = 1
 
 # Number of seconds the gunicorn webserver waits before timing out on a worker
 worker_timeout = 600