Expose variables from Settings (#964)

* Expose settings

* Update imports

* Update correct expose settings

* Update with SecretStr

* Fix URI expose

* Add test and extend sanitize

* Update linting

* Add documentation and bump version

* update doc

* Fix tests

* update doc

* Update after review

* Add env variable to expose secrets

* Fix testing and linting

* fix test

* Fix linting

* Update comment

* Update after review

* Add docs

* Update after review

Author: Igoranze

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.0.4
+current_version = 4.1.0rc1
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(rc(?P<build>\d+))?

docs/reference-docs/app/settings_overview.md

@@ -0,0 +1,89 @@
+# Settings overview in Orchestrator
+
+You can use the `api/settings/overview` endpoint to get an overview of the settings that are used in the application.
+This endpoint provides a JSON response that contains the settings that are defined in the application. The settings are
+grouped by their names and sensitive values are masked for security reasons.
+Per default, the application settings are used to configure the application. The settings are defined in the
+`orchestrator.settings.py` module and can be used to configure the application.
+
+An example of the settings is shown below:
+
+```python
+from orchestrator.settings import BaseSettings
+
+
+class AppSettings(BaseSettings):
+    TESTING: bool = True
+    SESSION_SECRET: OrchSecretStr = "".join(secrets.choice(string.ascii_letters) for i in range(16))  # type: ignore
+    CORS_ORIGINS: str = "*"
+    ...
+    EXPOSE_SETTINGS: bool = False
+    EXPOSE_OAUTH_SETTINGS: bool = False
+
+
+app_settings = AppSettings()
+
+if app_settings.EXPOSE_SETTINGS:
+    expose_settings("app_settings", app_settings)  # type: ignore
+
+if app_settings.EXPOSE_OAUTH_SETTINGS:
+    expose_settings("oauth2lib_settings", oauth2lib_settings)  # type: ignore
+```
+
+What you see above is the default settings for the application. The settings are defined in the
+`orchestrator.settings.py` module and can be used to configure the application.
+The `EXPOSE_SETTINGS` and `EXPOSE_OAUTH_SETTINGS` flags are used to control whether the settings should be exposed via
+the `api/settings/overview` endpoint, the result looks like this:
+
+```json
+[
+  {
+    "name": "app_settings",
+    "variables": [
+      {
+        "env_name": "TESTING",
+        "env_value": false
+      },
+      {
+        "env_name": "SESSION_SECRET",
+        "env_value": "**********"
+      },
+      {
+        "env_name": "CORS_ORIGINS",
+        "env_value": "*"
+      }
+    ]
+  }
+]
+```
+
+The `app_settings` in the example above is a name of the settings class that is registered to be exposed.
+
+## Exposing your settings
+
+In order to expose your settings, you need to register them using the `expose_settings()` function. This function takes
+two arguments: the name of the settings class and the instance of the settings class.
+
+```python
+from orchestrator.settings import expose_settings, BaseSettings
+
+
+class MySettings(BaseSettings):
+    debug: bool = True
+
+
+my_settings = MySettings()
+
+expose_settings("my_settings", my_settings)
+```
+
+## Masking Secrets
+
+The following rules apply when exposing settings:
+
+### Rules for Masking Secrets
+
+- Keys containing `"password"` or `"secret"` in their names are masked.
+- `SecretStr` from `from pydantic import SecretStr` are masked.
+- `SecretStr` from `from orchestrator.utils.expose_settings import SecretStr` are masked.
+- `PostgresDsn` from `from pydantic import PostgresDsn` are masked.

mkdocs.yml

@@ -182,6 +182,7 @@ nav:
                 - App.py: reference-docs/app/app.md
                 - Python Version: reference-docs/python.md
                 - Scaling: reference-docs/app/scaling.md
+                - Settings: reference-docs/app/settings_overview.md
           # - Tasks:  reference-docs/tasks.md
           # - Tests: reference-docs/tests.md
           - Workflows:

orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "4.0.4"
+__version__ = "4.1.0rc1"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings

orchestrator/api/api_v1/endpoints/settings.py

@@ -22,11 +22,17 @@
 from oauth2_lib.fastapi import OIDCUserModel
 from orchestrator.api.error_handling import raise_status
 from orchestrator.db import EngineSettingsTable
-from orchestrator.schemas import EngineSettingsBaseSchema, EngineSettingsSchema, WorkerStatus
+from orchestrator.schemas import (
+    EngineSettingsBaseSchema,
+    EngineSettingsSchema,
+    WorkerStatus,
+)
 from orchestrator.security import authenticate
 from orchestrator.services import processes, settings
 from orchestrator.services.settings import generate_engine_global_status
+from orchestrator.services.settings_env_variables import get_all_exposed_settings
 from orchestrator.settings import ExecutorType, app_settings
+from orchestrator.utils.expose_settings import SettingsExposedSchema
 from orchestrator.utils.json import json_dumps
 from orchestrator.utils.redis import delete_keys_matching_pattern
 from orchestrator.utils.redis_client import create_redis_asyncio_client
@@ -169,3 +175,8 @@ def generate_engine_status_response(
     result = EngineSettingsSchema.model_validate(engine_settings)
     result.global_status = generate_engine_global_status(engine_settings)
     return result
+
+
+@router.get("/overview", response_model=list[SettingsExposedSchema])
+def get_exposed_settings() -> list[SettingsExposedSchema]:
+    return get_all_exposed_settings()

orchestrator/services/settings_env_variables.py

@@ -0,0 +1,68 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, Type
+
+from pydantic import SecretStr as PydanticSecretStr
+from pydantic_core import MultiHostUrl
+from pydantic_settings import BaseSettings
+
+from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
+from orchestrator.utils.expose_settings import SettingsEnvVariablesSchema, SettingsExposedSchema
+
+EXPOSED_ENV_SETTINGS_REGISTRY: Dict[str, Type[BaseSettings]] = {}
+MASK = "**********"
+
+
+def expose_settings(settings_name: str, base_settings: Type[BaseSettings]) -> Type[BaseSettings]:
+    """Decorator to register settings classes."""
+    EXPOSED_ENV_SETTINGS_REGISTRY[settings_name] = base_settings
+    return base_settings
+
+
+def mask_value(key: str, value: Any) -> Any:
+    key_lower = key.lower()
+
+    if "secret" in key_lower or "password" in key_lower:
+        # Mask sensitive information
+        return MASK
+
+    if isinstance(value, PydanticSecretStr):
+        # Need to convert SecretStr to str for serialization
+        return str(value)
+
+    if isinstance(value, OrchSecretStr):
+        return MASK
+
+    # PostgresDsn is just MultiHostUrl with extra metadata (annotations)
+    if isinstance(value, MultiHostUrl):
+        # Convert PostgresDsn to str for serialization
+        return MASK
+
+    return value
+
+
+def get_all_exposed_settings() -> list[SettingsExposedSchema]:
+    """Return all registered settings as dicts."""
+
+    def _get_settings_env_variables(base_settings: Type[BaseSettings]) -> list[SettingsEnvVariablesSchema]:
+        """Get environment variables from settings."""
+        return [
+            SettingsEnvVariablesSchema(env_name=key, env_value=mask_value(key, value))
+            for key, value in base_settings.model_dump().items()  # type: ignore
+        ]
+
+    return [
+        SettingsExposedSchema(name=name, variables=_get_settings_env_variables(base_settings))
+        for name, base_settings in EXPOSED_ENV_SETTINGS_REGISTRY.items()
+    ]

orchestrator/settings.py

@@ -20,6 +20,8 @@
 from pydantic_settings import BaseSettings
 
 from oauth2_lib.settings import oauth2lib_settings
+from orchestrator.services.settings_env_variables import expose_settings
+from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
 from pydantic_forms.types import strEnum
 
 
@@ -30,7 +32,7 @@ class ExecutorType(strEnum):
 
 class AppSettings(BaseSettings):
     TESTING: bool = True
-    SESSION_SECRET: str = "".join(secrets.choice(string.ascii_letters) for i in range(16))  # noqa: S311
+    SESSION_SECRET: OrchSecretStr = "".join(secrets.choice(string.ascii_letters) for i in range(16))  # type: ignore
     CORS_ORIGINS: str = "*"
     CORS_ALLOW_METHODS: list[str] = ["GET", "PUT", "PATCH", "POST", "DELETE", "OPTIONS", "HEAD"]
     CORS_ALLOW_HEADERS: list[str] = ["If-None-Match", "Authorization", "If-Match", "Content-Type"]
@@ -55,7 +57,7 @@ class AppSettings(BaseSettings):
     MAIL_PORT: int = 25
     MAIL_STARTTLS: bool = False
     CACHE_URI: RedisDsn = "redis://localhost:6379/0"  # type: ignore
-    CACHE_HMAC_SECRET: str | None = None  # HMAC signing key, used when pickling results in the cache
+    CACHE_HMAC_SECRET: OrchSecretStr | None = None  # HMAC signing key, used when pickling results in the cache
     REDIS_RETRY_COUNT: NonNegativeInt = Field(
         2, description="Number of retries for redis connection errors/timeouts, 0 to disable"
     )  # More info: https://redis-py.readthedocs.io/en/stable/retry.html
@@ -87,10 +89,17 @@ class AppSettings(BaseSettings):
     ENABLE_PROMETHEUS_METRICS_ENDPOINT: bool = False
     VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS: bool = False
     FILTER_BY_MODE: Literal["partial", "exact"] = "exact"
+    EXPOSE_SETTINGS: bool = False
+    EXPOSE_OAUTH_SETTINGS: bool = False
 
 
 app_settings = AppSettings()
 
 # Set oauth2lib_settings variables to the same (default) value of settings
 oauth2lib_settings.SERVICE_NAME = app_settings.SERVICE_NAME
 oauth2lib_settings.ENVIRONMENT = app_settings.ENVIRONMENT
+
+if app_settings.EXPOSE_SETTINGS:
+    expose_settings("app_settings", app_settings)  # type: ignore
+if app_settings.EXPOSE_OAUTH_SETTINGS:
+    expose_settings("oauth2lib_settings", oauth2lib_settings)  # type: ignore

orchestrator/utils/expose_settings.py

@@ -0,0 +1,45 @@
+# Copyright 2019-2025 SURF.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility module for exposing settings in a structured format.
+
+Unfortunately, this module needs to be imported from the utils and cannot be added to the schemas folder.
+This is due to circular import issues with the combination of schemas/settings.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel
+from pydantic_core import core_schema
+
+
+class SecretStr(str):
+    """A string that is treated as a secret, for example, passwords or API keys.
+
+    This class is used to indicate that the string should not be logged or displayed in plaintext.
+    """
+
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type, handler):  # type: ignore
+        # This method is used to define how the SecretStr type should be handled by Pydantic.
+        # With this implementation, it will fail validation.
+        return core_schema.no_info_plain_validator_function(cls)
+
+
+class SettingsEnvVariablesSchema(BaseModel):
+    env_name: str
+    env_value: Any
+
+
+class SettingsExposedSchema(BaseModel):
+    name: str
+    variables: list[SettingsEnvVariablesSchema]

test/unit_tests/api/test_settings.py

@@ -2,10 +2,13 @@
 from unittest import mock
 from unittest.mock import Mock
 
+from pydantic import SecretStr
+from pydantic_settings import BaseSettings
 from sqlalchemy.exc import SQLAlchemyError
 
 from orchestrator.db import db
 from orchestrator.services.settings import get_engine_settings
+from orchestrator.services.settings_env_variables import expose_settings, get_all_exposed_settings
 
 
 def test_get_engine_status(test_client):
@@ -74,3 +77,22 @@ def test_reset_search_index_error(test_client, generic_subscription_1, generic_s
         ex.attach_mock(session_execute_mock, "execute")
         response = test_client.post("/api/settings/search-index/reset")
         assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
+
+
+def test_get_exposed_settings(test_client):
+    class MySettings(BaseSettings):
+        db_password: SecretStr = "test_password"  # noqa: S105
+
+    my_settings = MySettings()
+    expose_settings("my_settings", my_settings)
+    assert len(get_all_exposed_settings()) == 1
+
+    response = test_client.get("/api/settings/overview")
+    assert response.status_code == HTTPStatus.OK
+
+    exposed_settings = response.json()
+
+    # Find the env_name db_password and ensure it is masked is **********
+    session_secret = next((var for var in exposed_settings[0]["variables"] if var["env_name"] == "db_password"), None)
+    assert session_secret is not None
+    assert session_secret["env_value"] == "**********"

test/unit_tests/services/test_expose_settings.py

@@ -0,0 +1,35 @@
+from pydantic import PostgresDsn
+from pydantic import SecretStr as PydanticSecretStr
+from pydantic_settings import BaseSettings
+
+from orchestrator.services.settings_env_variables import MASK, expose_settings, get_all_exposed_settings
+from orchestrator.utils.expose_settings import SecretStr as OrchSecretStr
+
+
+def test_expose_settings():
+
+    class MySettings(BaseSettings):
+        api_key: OrchSecretStr = "test_api_key"
+        db_password: PydanticSecretStr = "test_password"  # noqa: S105
+        debug_mode: bool = True
+        secret_test: str = "test_secret"  # noqa: S105
+        uri: PostgresDsn = "postgresql://user:password@localhost/dbname"
+
+    my_settings = MySettings()
+    expose_settings("my_settings", my_settings)
+
+    exposed_settings = get_all_exposed_settings()
+
+    assert len(exposed_settings) == 1
+    my_settings_index = 0
+
+    assert exposed_settings[my_settings_index].name == "my_settings"
+
+    assert len(exposed_settings[my_settings_index].variables) == 5
+
+    # Assert that sensitive values are masked
+    assert exposed_settings[my_settings_index].variables[0].env_value == MASK  # api_key
+    assert exposed_settings[my_settings_index].variables[1].env_value == MASK  # db_password
+    assert exposed_settings[my_settings_index].variables[2].env_value is True  # debug_mode
+    assert exposed_settings[my_settings_index].variables[3].env_value == MASK  # secret_test
+    assert exposed_settings[my_settings_index].variables[4].env_value == MASK  # uri