fix(airflow.cfg): migrate to airflow 3

remove airflow config that does not exist or the value is the same as default value

Author: Lee-W

airflow.cfg

@@ -1,8 +1,4 @@
 [core]
-# The folder where your airflow pipelines live, most likely a
-# subfolder in a code repository. This path must be absolute.
-dags_folder = /opt/airflow/dags
-
 # The amount of parallelism as a setting to the executor. This defines
 # the max number of task instances that should run simultaneously
 # on this airflow installation
@@ -16,73 +12,47 @@ max_active_tasks_per_dag = 64
 # environment
 load_examples = False
 
-# Where your Airflow plugins are stored
-plugins_folder = /opt/airflow/plugins
-
 # Secret key to save connection passwords in the db
 fernet_key = $FERNET_KEY
 
 # How long before timing out a python file import
 dagbag_import_timeout = 600
 
+[dag_processor]
 # How long before timing out a DagFileProcessor, which processes a dag file
 dag_file_processor_timeout = 600
 
-
 [database]
 # The SqlAlchemy connection string to the metadata database.
 # SqlAlchemy supports many different database engine, more information
 # their website
 sql_alchemy_conn = sqlite:////opt/airflow/sqlite/airflow.db
 
-
-[logging]
-# The folder where airflow should store its log files
-# This path must be absolute
-base_log_folder = /opt/airflow/logs
-
-# Log format for when Colored logs is enabled
-colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s
-
-# Format of Log line
-log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s
-
-dag_processor_manager_log_location = /opt/airflow/logs/dag_processor_manager/dag_processor_manager.log
-
-
 [api]
-# How to authenticate users of the API
-auth_backends = airflow.api.auth.backend.default
-
-
-[webserver]
-# Number of seconds the webserver waits before killing gunicorn master that doesn't respond
-web_server_master_timeout = 600
+# Number of workers to run the Gunicorn web server
+workers = 2
 
 # Number of seconds the gunicorn webserver waits before timing out on a worker
-web_server_worker_timeout = 600
+worker_timeout = 600
 
+# Expose the configuration file in the web server
+expose_config = True
+
+# TODO: move it to env var
 # Secret key used to run your flask app
 # It should be as random as possible
 secret_key = l\xba,\xc3\x023\xca\x04\xdb\xf2\xf7\xfa\xb8#\xee>
 
-# Number of workers to run the Gunicorn web server
-workers = 2
-
-# Expose the configuration file in the web server
-expose_config = True
+[api_auth]
+jwt_secret = $JWT_SECRET
 
-# Allow the UI to be rendered in a frame
-x_frame_enabled = True
+[fab]
 
 # Minutes of non-activity before logged out from UI
 # 0 means never get forcibly logged out
-force_log_out_after = 0
-
-authenticate = False
-auth_backend = airflow.api.auth.backend.default
-
+session_lifetime_minutes = 0
 
+# TODO: check whether we need it. I don't think we're using celery
 [celery]
 # The concurrency that will be used when starting workers with the
 # ``airflow celery worker`` command. This defines the number of task instances that
@@ -115,10 +85,8 @@ result_backend = db+postgresql://airflow:airflow@postgres/airflow
 
 
 [scheduler]
-child_process_log_directory = /opt/airflow/logs/scheduler
-
-# Format of the log_id, which is used to query for a given tasks logs
-log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}}
+# TODO: check whether it exists
+enable_health_check = True
 
 
 [kubernetes]
@@ -133,9 +101,6 @@ log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}}
 # The timeout is specified as [connect timeout, read timeout]
 kube_client_request_args = {{"_request_timeout" : [60,60] }}
 
-# Specifies the uid to run the first process of the worker pods containers as
-run_as_user =
-
 # ref: https://airflow.apache.org/docs/apache-airflow/1.10.1/security.html#setting-up-google-authentication
 [google]
 client_id = <check the doc above>