Merge pull request #159 from pycontw/upgrade-to-airflow-3

Upgrade to Airflow 3.0.2

Author: henry410213028

.gitignore

@@ -104,6 +104,10 @@ docs/_build/
 # PyBuilder
 target/
 
-# airflow sqlite db
-sqlite/
+# mkdocs
 site
+
+# airflow
+sqlite/
+logs/
+simple_auth_manager_passwords.json.generated

Dockerfile

@@ -1,4 +1,4 @@
-ARG AIRFLOW_VERSION=2.11.0
+ARG AIRFLOW_VERSION=3.0.2
 ARG PYTHON_VERSION=3.10
 ARG PLATFORM=linux
 
@@ -37,7 +37,9 @@ ENV PATH="/app/.venv/bin:$PATH"
 
 COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
 COPY --chown=airflow:root dags ${AIRFLOW_HOME}/dags
-# TODO: remove this and the patch files once upgrade to 2.11.1
-COPY --chown=airflow:root patch /app/.venv/lib/python3.10/site-packages/airflow/migrations
+# TODO: remove this and the patch files once upgrade to 3.0.3
+COPY --chown=airflow:root patch/utils.py /app/.venv/lib/python3.10/site-packages/airflow/migrations/utils.py
+COPY --chown=airflow:root patch/versions /app/.venv/lib/python3.10/site-packages/airflow/migrations/versions
+COPY --chown=airflow:root patch/serialized_objects.py /app/.venv/lib/python3.10/site-packages/airflow/serialization/serialized_objects.py
 
 ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.test

@@ -1,4 +1,4 @@
-ARG AIRFLOW_VERSION=2.11.0
+ARG AIRFLOW_VERSION=3.0.2
 ARG PYTHON_VERSION=3.10
 ARG PLATFORM=linux
 
@@ -37,8 +37,10 @@ ENV PATH="/app/.venv/bin:$PATH"
 
 COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
 COPY --chown=airflow:root dags ${AIRFLOW_HOME}/dags
-# TODO: remove this and the patch files once upgrade to 2.11.1
-COPY --chown=airflow:root patch /app/.venv/lib/python3.10/site-packages/airflow/migrations
+# TODO: remove this and the patch files once upgrade to 3.0.3
+COPY --chown=airflow:root patch/utils.py /app/.venv/lib/python3.10/site-packages/airflow/migrations/utils.py
+COPY --chown=airflow:root patch/versions /app/.venv/lib/python3.10/site-packages/airflow/migrations/versions
+COPY --chown=airflow:root patch/serialized_objects.py /app/.venv/lib/python3.10/site-packages/airflow/serialization/serialized_objects.py
 
 ENV AIRFLOW_TEST_MODE=True
 

airflow.cfg

@@ -1,8 +1,4 @@
 [core]
-# The folder where your airflow pipelines live, most likely a
-# subfolder in a code repository. This path must be absolute.
-dags_folder = /opt/airflow/dags
-
 # The amount of parallelism as a setting to the executor. This defines
 # the max number of task instances that should run simultaneously
 # on this airflow installation
@@ -16,73 +12,48 @@ max_active_tasks_per_dag = 64
 # environment
 load_examples = False
 
-# Where your Airflow plugins are stored
-plugins_folder = /opt/airflow/plugins
-
 # Secret key to save connection passwords in the db
 fernet_key = $FERNET_KEY
 
 # How long before timing out a python file import
 dagbag_import_timeout = 600
 
+auth_manager = airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
+
+[dag_processor]
 # How long before timing out a DagFileProcessor, which processes a dag file
 dag_file_processor_timeout = 600
 
-
 [database]
 # The SqlAlchemy connection string to the metadata database.
 # SqlAlchemy supports many different database engine, more information
 # their website
 sql_alchemy_conn = sqlite:////opt/airflow/sqlite/airflow.db
-
-
-[logging]
-# The folder where airflow should store its log files
-# This path must be absolute
-base_log_folder = /opt/airflow/logs
-
-# Log format for when Colored logs is enabled
-colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s
-
-# Format of Log line
-log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s
-
-dag_processor_manager_log_location = /opt/airflow/logs/dag_processor_manager/dag_processor_manager.log
+external_db_managers = airflow.providers.fab.auth_manager.models.db.FABDBManager
 
 
 [api]
-# How to authenticate users of the API
-auth_backends = airflow.api.auth.backend.default
-
-
-[webserver]
-# Number of seconds the webserver waits before killing gunicorn master that doesn't respond
-web_server_master_timeout = 600
-
-# Number of seconds the gunicorn webserver waits before timing out on a worker
-web_server_worker_timeout = 600
-
-# Secret key used to run your flask app
-# It should be as random as possible
-secret_key = l\xba,\xc3\x023\xca\x04\xdb\xf2\xf7\xfa\xb8#\xee>
-
 # Number of workers to run the Gunicorn web server
 workers = 2
 
+# Number of seconds the gunicorn webserver waits before timing out on a worker
+worker_timeout = 600
+
 # Expose the configuration file in the web server
 expose_config = True
 
-# Allow the UI to be rendered in a frame
-x_frame_enabled = True
-
-# Minutes of non-activity before logged out from UI
-# 0 means never get forcibly logged out
-force_log_out_after = 0
+# TODO: move it to env var
+# Secret key used to run your flask app
+# It should be as random as possible
+secret_key = l\xba,\xc3\x023\xca\x04\xdb\xf2\xf7\xfa\xb8#\xee>
 
-authenticate = False
-auth_backend = airflow.api.auth.backend.default
+[api_auth]
+jwt_secret = $JWT_SECRET
 
+[fab]
+auth_backends = airflow.providers.fab.auth_manager.api.auth.backend.session
 
+# TODO: check whether we need it. I don't think we're using celery
 [celery]
 # The concurrency that will be used when starting workers with the
 # ``airflow celery worker`` command. This defines the number of task instances that
@@ -115,10 +86,8 @@ result_backend = db+postgresql://airflow:airflow@postgres/airflow
 
 
 [scheduler]
-child_process_log_directory = /opt/airflow/logs/scheduler
-
-# Format of the log_id, which is used to query for a given tasks logs
-log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}}
+# TODO: check whether it exists
+enable_health_check = True
 
 
 [kubernetes]
@@ -133,9 +102,6 @@ log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}}
 # The timeout is specified as [connect timeout, read timeout]
 kube_client_request_args = {{"_request_timeout" : [60,60] }}
 
-# Specifies the uid to run the first process of the worker pods containers as
-run_as_user =
-
 # ref: https://airflow.apache.org/docs/apache-airflow/1.10.1/security.html#setting-up-google-authentication
 [google]
 client_id = <check the doc above>