From 16c66ad658c5a4e3b1b14982f871ab2d12b26371 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 25 Jun 2025 15:54:55 +0200 Subject: [PATCH 1/9] set INFO as default for airflow.task logging --- rust/operator-binary/src/product_logging.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 6bf99883..d0f00c18 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -114,6 +114,9 @@ for logger_name, logger_config in LOGGING_CONFIG['loggers'].items(): # otherwise DAGs cannot be loaded anymore. if logger_name != 'airflow.task': logger_config['propagate'] = True + # Adopting to airflows logging standards + if logger_name == 'airflow.task': + logger_config['level'] = logging.INFO LOGGING_CONFIG.setdefault('formatters', {{}}) LOGGING_CONFIG['formatters']['json'] = {{ From ed7d743d0ecaf77d604314f2db72029142ce5ae6 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 25 Jun 2025 16:01:51 +0200 Subject: [PATCH 2/9] Add documentation --- docs/modules/airflow/pages/usage-guide/logging.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/modules/airflow/pages/usage-guide/logging.adoc b/docs/modules/airflow/pages/usage-guide/logging.adoc index 260842be..1abaebf2 100644 --- a/docs/modules/airflow/pages/usage-guide/logging.adoc +++ b/docs/modules/airflow/pages/usage-guide/logging.adoc @@ -3,6 +3,8 @@ The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: +NOTE: airflow.task loglevel is set to `INFO` by default. + [source,yaml] ---- spec: From 8c04906f6353ff59b0a989635383ab439d7b6cb3 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 25 Jun 2025 16:02:41 +0200 Subject: [PATCH 3/9] Add example to docs --- docs/modules/airflow/pages/usage-guide/logging.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/modules/airflow/pages/usage-guide/logging.adoc b/docs/modules/airflow/pages/usage-guide/logging.adoc index 1abaebf2..3d5d3df3 100644 --- a/docs/modules/airflow/pages/usage-guide/logging.adoc +++ b/docs/modules/airflow/pages/usage-guide/logging.adoc @@ -28,6 +28,8 @@ spec: loggers: "airflow.processor": level: INFO + "airflow.task": + level: DEBUG schedulers: config: logging: From ef76f359df35827a26f4b787c798873a5ab3f9bc Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 25 Jun 2025 17:33:59 +0200 Subject: [PATCH 4/9] Using handler task to set log level, update comment --- rust/operator-binary/src/product_logging.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index d0f00c18..5000e7e1 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -114,9 +114,6 @@ for logger_name, logger_config in LOGGING_CONFIG['loggers'].items(): # otherwise DAGs cannot be loaded anymore. if logger_name != 'airflow.task': logger_config['propagate'] = True - # Adopting to airflows logging standards - if logger_name == 'airflow.task': - logger_config['level'] = logging.INFO LOGGING_CONFIG.setdefault('formatters', {{}}) LOGGING_CONFIG['formatters']['json'] = {{ @@ -136,6 +133,10 @@ LOGGING_CONFIG['handlers']['file'] = {{ 'backupCount': 1, }} +# The default behavior of airflow is to enforce log level 'INFO' on tasks. (https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#logging-level) +# TODO: Make task handler log level configurable through CRDs with default 'INFO'. +LOGGING_CONFIG['handlers']['task']['level'] = logging.INFO + LOGGING_CONFIG['root'] = {{ 'level': {root_log_level}, 'filters': ['mask_secrets'], From bb842e502f4c6ce4d1d9b52cd22c2804d6dc102c Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 25 Jun 2025 18:02:05 +0200 Subject: [PATCH 5/9] Moving handler to comment --- rust/operator-binary/src/product_logging.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 5000e7e1..7545f99e 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -114,6 +114,11 @@ for logger_name, logger_config in LOGGING_CONFIG['loggers'].items(): # otherwise DAGs cannot be loaded anymore. if logger_name != 'airflow.task': logger_config['propagate'] = True + # The default behavior of airflow is to enforce log level 'INFO' on tasks. (https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#logging-level) + # TODO: Make task handler log level configurable through CRDs with default 'INFO'. + # e.g. LOGGING_CONFIG['handlers']['task']['level'] = {{task_log_level}} + if 'task' in logger_config['handlers']: + logger_config['level'] = logging.INFO LOGGING_CONFIG.setdefault('formatters', {{}}) LOGGING_CONFIG['formatters']['json'] = {{ @@ -133,10 +138,6 @@ LOGGING_CONFIG['handlers']['file'] = {{ 'backupCount': 1, }} -# The default behavior of airflow is to enforce log level 'INFO' on tasks. (https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#logging-level) -# TODO: Make task handler log level configurable through CRDs with default 'INFO'. -LOGGING_CONFIG['handlers']['task']['level'] = logging.INFO - LOGGING_CONFIG['root'] = {{ 'level': {root_log_level}, 'filters': ['mask_secrets'], From 45bcae7b10e28b7c1eb2e05f8b06b426e5ab39ea Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 25 Jun 2025 18:08:21 +0200 Subject: [PATCH 6/9] update Changelog.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 992e1460..09a2d491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. - Add support for airflow `2.10.5` ([#625]). - Add experimental support for airflow `3.0.1` ([#630]). +- "airflow.task" logger defaults to log level 'INFO' instead of 'NOTSET' ([#649]). ### Changed @@ -50,6 +51,7 @@ [#630]: https://github.com/stackabletech/airflow-operator/pull/630 [#636]: https://github.com/stackabletech/airflow-operator/pull/636 [#645]: https://github.com/stackabletech/airflow-operator/pull/645 +[#649]: https://github.com/stackabletech/airflow-operator/pull/649 ## [25.3.0] - 2025-03-21 From cc013e376a57b109180b9ad88b9bb633f571c7ff Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 25 Jun 2025 18:09:53 +0200 Subject: [PATCH 7/9] fixing typo --- docs/modules/airflow/pages/usage-guide/logging.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/logging.adoc b/docs/modules/airflow/pages/usage-guide/logging.adoc index 3d5d3df3..95ba3a51 100644 --- a/docs/modules/airflow/pages/usage-guide/logging.adoc +++ b/docs/modules/airflow/pages/usage-guide/logging.adoc @@ -3,7 +3,7 @@ The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: -NOTE: airflow.task loglevel is set to `INFO` by default. +NOTE: airflow.task log level is set to `INFO` by default. [source,yaml] ---- From d2061bb58ebdd8937c425b97a3c8c504c477d62a Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Thu, 26 Jun 2025 10:22:43 +0200 Subject: [PATCH 8/9] Better documentation Co-authored-by: Siegfried Weber --- docs/modules/airflow/pages/usage-guide/logging.adoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/logging.adoc b/docs/modules/airflow/pages/usage-guide/logging.adoc index 95ba3a51..bd82ee39 100644 --- a/docs/modules/airflow/pages/usage-guide/logging.adoc +++ b/docs/modules/airflow/pages/usage-guide/logging.adoc @@ -3,7 +3,10 @@ The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: -NOTE: airflow.task log level is set to `INFO` by default. +NOTE: The `task` handler is responsible for showing the task logs in the UI. +Unfortunately, the log level of the `task` handler cannot be specified. +To avoid that all logs are emitted to the UI, the log level of the `airflow.task` logger is set explicitly to `INFO`. +You can change the log level as shown below. [source,yaml] ---- From a0378e86066043eead895e2679e1b677efadcc34 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 26 Jun 2025 13:04:56 +0200 Subject: [PATCH 9/9] More robust decision makeing --- rust/operator-binary/src/product_logging.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 7545f99e..1557baf6 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -117,7 +117,7 @@ for logger_name, logger_config in LOGGING_CONFIG['loggers'].items(): # The default behavior of airflow is to enforce log level 'INFO' on tasks. (https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#logging-level) # TODO: Make task handler log level configurable through CRDs with default 'INFO'. # e.g. LOGGING_CONFIG['handlers']['task']['level'] = {{task_log_level}} - if 'task' in logger_config['handlers']: + if 'handlers' in logger_config and 'task' in logger_config['handlers']: logger_config['level'] = logging.INFO LOGGING_CONFIG.setdefault('formatters', {{}})