From fb7289744b6b71818bd3dace94816ec9239775ca Mon Sep 17 00:00:00 2001 From: Adhitya Ravi Date: Fri, 8 Aug 2025 16:22:58 +0200 Subject: [PATCH 1/6] feat: enable running cos-alerter in micro-service mode (api and dashboard) --- cos_alerter/config-defaults.yaml | 5 +++ cos_alerter/daemon.py | 70 ++++++++++++++++++++++++++------ cos_alerter/server.py | 39 +++++++++++++++--- 3 files changed, 96 insertions(+), 18 deletions(-) diff --git a/cos_alerter/config-defaults.yaml b/cos_alerter/config-defaults.yaml index 936e04f..0a47d1b 100644 --- a/cos_alerter/config-defaults.yaml +++ b/cos_alerter/config-defaults.yaml @@ -45,3 +45,8 @@ log_level: "info" # The address to listen on for http traffic. # Format HOST:PORT web_listen_addr: "0.0.0.0:8080" + +# Optional: Separate address for the dashboard UI. +# If not set, dashboard will be served on the same address as the API. +# Format HOST:PORT +# dashboard_listen_addr: "127.0.0.1:8081" diff --git a/cos_alerter/daemon.py b/cos_alerter/daemon.py index 8d73c68..bf5587d 100644 --- a/cos_alerter/daemon.py +++ b/cos_alerter/daemon.py @@ -16,7 +16,7 @@ from .alerter import AlerterState, config, send_test_notification, up_time from .logging import LEVELS, init_logging -from .server import app +from .server import create_app logger = logging.getLogger("cos_alerter.daemon") @@ -98,21 +98,65 @@ def main(run_for: Optional[int] = None, argv: List[str] = sys.argv): if not str(e) == "signal only works in main thread of the main interpreter": raise # pragma: no cover - # Start the web server. + # Start the web server(s). # Starting in a thread rather than a new process allows waitress to inherit the log level # from the daemon. It also facilitates communication over memory rather than files. # clear_untrusted_proxy_headers is set to suppress a DeprecationWarning. - server_thread = threading.Thread( - target=waitress.serve, - args=(app,), - kwargs={ - "clear_untrusted_proxy_headers": True, - "listen": config["web_listen_addr"], - }, - ) - server_thread.daemon = True # Makes this thread exit when the main thread exits. - logger.info("Starting the web server thread.") - server_thread.start() + + # Check if split mode is enabled (dashboard_listen_addr is configured) + try: + dashboard_addr = config["dashboard_listen_addr"] + except KeyError: + dashboard_addr = None + if dashboard_addr: + # Split mode: separate API and dashboard servers + logger.info("Starting microservice mode - API server on %s, dashboard on %s", + config["web_listen_addr"], dashboard_addr) + + # API server + api_app = create_app(include_api=True, include_dashboard=False) + api_server_thread = threading.Thread( + target=waitress.serve, + args=(api_app,), + kwargs={ + "clear_untrusted_proxy_headers": True, + "listen": config["web_listen_addr"], + }, + ) + api_server_thread.daemon = True + api_server_thread.start() + logger.info("Started API server thread on %s", config["web_listen_addr"]) + + # Dashboard server + dashboard_app = create_app(include_api=False, include_dashboard=True) + dashboard_server_thread = threading.Thread( + target=waitress.serve, + args=(dashboard_app,), + kwargs={ + "clear_untrusted_proxy_headers": True, + "listen": dashboard_addr, + }, + ) + dashboard_server_thread.daemon = True + dashboard_server_thread.start() + logger.info("Started dashboard server thread on %s", dashboard_addr) + + else: + # Combined mode: single server with both endpoints (backwards compatible) + logger.info("Starting monolith mode - App served on %s", + config["web_listen_addr"]) + app = create_app(include_api=True, include_dashboard=True) + server_thread = threading.Thread( + target=waitress.serve, + args=(app,), + kwargs={ + "clear_untrusted_proxy_headers": True, + "listen": config["web_listen_addr"], + }, + ) + server_thread.daemon = True + server_thread.start() + logger.info("Started server thread on %s", config["web_listen_addr"]) for clientid in config["watch"]["clients"]: client_thread = threading.Thread(target=client_loop, args=(clientid,)) diff --git a/cos_alerter/server.py b/cos_alerter/server.py index 398d40b..d8489b3 100644 --- a/cos_alerter/server.py +++ b/cos_alerter/server.py @@ -12,12 +12,39 @@ from .alerter import AlerterState, config, now_datetime -app = Flask(__name__) -metrics = PrometheusMetrics(app) logger = logging.getLogger(__name__) -@app.route("/", methods=["GET"]) +def create_app(include_api: bool = True, include_dashboard: bool = True) -> Flask: + """Create Flask app with specified endpoints. + + Args: + include_api: Whether to include the /alive API endpoint + include_dashboard: Whether to include the / dashboard endpoint + + Returns: + Flask application instance + """ + app = Flask(__name__) + metrics = PrometheusMetrics(app) + + if include_dashboard: + @app.route("/", methods=["GET"]) + def dashboard_route(): + return dashboard() + + if include_api: + @app.route("/alive", methods=["POST"]) + def alive_route(): + return alive() + + @app.before_request + def log_request_wrapper(): + return log_request() + + return app + + def dashboard(): """Endpoint for the COS Alerter dashboard.""" clients = [] @@ -40,7 +67,6 @@ def dashboard(): return render_template("dashboard.html", clients=clients) -@app.route("/alive", methods=["POST"]) def alive(): """Endpoint for Alertmanager instances to send their heartbeat alerts.""" # TODO Decide if we should validate the request. @@ -74,7 +100,6 @@ def alive(): return "Success!" -@app.before_request def log_request(): """Log every HTTP request.""" logger.info( @@ -82,3 +107,7 @@ def log_request(): request.method, request.url, ) + + +# Backwards compatibility - existing imports will still work +app = create_app() From 7cd2cccceacff5a21b646152e55cd4de3965c41d Mon Sep 17 00:00:00 2001 From: Adhitya Ravi Date: Tue, 12 Aug 2025 21:13:45 +0200 Subject: [PATCH 2/6] test: add tests for the new micro-service mode --- cos_alerter/daemon.py | 2 +- tests/test_daemon.py | 41 +++++++++++++++++++++++++++++++++++++++++ tests/test_server.py | 20 +++++++++++++++++++- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/cos_alerter/daemon.py b/cos_alerter/daemon.py index bf5587d..7e077a9 100644 --- a/cos_alerter/daemon.py +++ b/cos_alerter/daemon.py @@ -142,7 +142,7 @@ def main(run_for: Optional[int] = None, argv: List[str] = sys.argv): logger.info("Started dashboard server thread on %s", dashboard_addr) else: - # Combined mode: single server with both endpoints (backwards compatible) + # Combined mode: single server with both api and dashboard logger.info("Starting monolith mode - App served on %s", config["web_listen_addr"]) app = create_app(include_api=True, include_dashboard=True) diff --git a/tests/test_daemon.py b/tests/test_daemon.py index ebffc27..c23578a 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -85,3 +85,44 @@ def test_main(notify_mock, add_mock, mock_fs): def test_log_level_arg(mock_fs): main(run_for=0, argv=["cos-alerter", "--log-level", "DEBUG"]) assert logging.getLogger("cos_alerter").getEffectiveLevel() == logging.DEBUG + + +@unittest.mock.patch("cos_alerter.daemon.waitress.serve") +def test_microservice_mode_with_dashboard_addr(serve_mock, mock_fs): + # Add dashboard_listen_addr to config + with open("/etc/cos-alerter.yaml", "w") as f: + config_data = { + "watch": { + "down_interval": "4s", + "wait_for_first_connection": False, + "clients": { + "clientid1": { + "key": "822295b207a0b73dd4690b60a03c55599346d44aef3da4cf28c3296eadb98b2647ae18863cc3ae8ae5574191b60360858982fd8a8d176c0edf646ce6eee24ef9", + "name": "Instance Name 1", + }, + }, + }, + "notify": { + "destinations": DESTINATIONS, + "repeat_interval": "4s", + }, + "log_level": "info", + "web_listen_addr": "0.0.0.0:8080", + "dashboard_listen_addr": "127.0.0.1:8081", + } + f.write(yaml.dump(config_data)) + + config.reload() + + main(run_for=0, argv=["cos-alerter"]) + + # Check that waitress.serve was called twice (API and dashboard servers) + assert serve_mock.call_count == 2 + + +@unittest.mock.patch("cos_alerter.daemon.waitress.serve") +def test_monolith_mode_without_dashboard_addr(serve_mock, mock_fs): + main(run_for=0, argv=["cos-alerter"]) + + # Check that waitress.serve was called once (single server) + assert serve_mock.call_count == 1 diff --git a/tests/test_server.py b/tests/test_server.py index df7544b..0db3c12 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -9,7 +9,7 @@ from werkzeug.datastructures import MultiDict from cos_alerter.alerter import AlerterState, config -from cos_alerter.server import app +from cos_alerter.server import app, create_app PARAMS = {"clientid": "clientid1", "key": "clientkey1"} @@ -111,3 +111,21 @@ def test_multiple_key_values(flask_client, fake_fs, state_init): ) assert response.status_code == 400 assert len(response.data) > 0 + + +def test_create_app_api_only(fake_fs, state_init): + app_instance = create_app(include_api=True, include_dashboard=False) + client = app_instance.test_client() + + # Only API endpoint should be available + assert client.get("/").status_code == 404 + assert client.post("/alive", query_string=PARAMS).status_code == 200 + + +def test_create_app_dashboard_only(fake_fs, state_init): + app_instance = create_app(include_api=False, include_dashboard=True) + client = app_instance.test_client() + + # Only dashboard endpoint should be available + assert client.get("/").status_code == 200 + assert client.post("/alive", query_string=PARAMS).status_code == 404 From c5218700533aba0d4c073a0011995123dd3f3eeb Mon Sep 17 00:00:00 2001 From: Adhitya Ravi Date: Tue, 12 Aug 2025 21:53:47 +0200 Subject: [PATCH 3/6] test: remove thread mocking duplicate tests (also its causing alloc issues) --- tests/test_daemon.py | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/tests/test_daemon.py b/tests/test_daemon.py index c23578a..ebffc27 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -85,44 +85,3 @@ def test_main(notify_mock, add_mock, mock_fs): def test_log_level_arg(mock_fs): main(run_for=0, argv=["cos-alerter", "--log-level", "DEBUG"]) assert logging.getLogger("cos_alerter").getEffectiveLevel() == logging.DEBUG - - -@unittest.mock.patch("cos_alerter.daemon.waitress.serve") -def test_microservice_mode_with_dashboard_addr(serve_mock, mock_fs): - # Add dashboard_listen_addr to config - with open("/etc/cos-alerter.yaml", "w") as f: - config_data = { - "watch": { - "down_interval": "4s", - "wait_for_first_connection": False, - "clients": { - "clientid1": { - "key": "822295b207a0b73dd4690b60a03c55599346d44aef3da4cf28c3296eadb98b2647ae18863cc3ae8ae5574191b60360858982fd8a8d176c0edf646ce6eee24ef9", - "name": "Instance Name 1", - }, - }, - }, - "notify": { - "destinations": DESTINATIONS, - "repeat_interval": "4s", - }, - "log_level": "info", - "web_listen_addr": "0.0.0.0:8080", - "dashboard_listen_addr": "127.0.0.1:8081", - } - f.write(yaml.dump(config_data)) - - config.reload() - - main(run_for=0, argv=["cos-alerter"]) - - # Check that waitress.serve was called twice (API and dashboard servers) - assert serve_mock.call_count == 2 - - -@unittest.mock.patch("cos_alerter.daemon.waitress.serve") -def test_monolith_mode_without_dashboard_addr(serve_mock, mock_fs): - main(run_for=0, argv=["cos-alerter"]) - - # Check that waitress.serve was called once (single server) - assert serve_mock.call_count == 1 From 4cd6a3c03f2d76e1655e32d148c95d04687ad3ca Mon Sep 17 00:00:00 2001 From: Adhitya Ravi Date: Tue, 12 Aug 2025 21:55:42 +0200 Subject: [PATCH 4/6] chore: update changelog and version --- CHANGELOG.md | 3 +++ pyproject.toml | 2 +- rockcraft.yaml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef29f05..18b7b82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## CI - updates - Fixed deprecated job versions in the CI +## [0.10.0] - 2025-08-12 + +- Added microservice mode support - API and dashboard can now run on separate addresses (#46). ## [0.9.0] - 2024-05-30 diff --git a/pyproject.toml b/pyproject.toml index 105afcd..944a7f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "cos-alerter" -version = "0.9.0" +version = "0.10.0" authors = [ { name="Dylan Stephano-Shachter", email="dylan.stephano-shachter@canonical.com" } ] diff --git a/rockcraft.yaml b/rockcraft.yaml index 9c036a9..406d6da 100644 --- a/rockcraft.yaml +++ b/rockcraft.yaml @@ -1,7 +1,7 @@ name: cos-alerter summary: A liveness checker for self-monitoring. description: Receive regular pings from the cos stack and alert when they stop. -version: "0.9.0" +version: "0.10.0" base: ubuntu@22.04 license: Apache-2.0 platforms: From 1868656af0228fee2b429412251e11b1f19d1337 Mon Sep 17 00:00:00 2001 From: Adhitya Ravi Date: Wed, 13 Aug 2025 11:14:00 +0200 Subject: [PATCH 5/6] chore: fix linting and code style --- cos_alerter/daemon.py | 10 ++++++---- cos_alerter/server.py | 4 +++- tests/test_server.py | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cos_alerter/daemon.py b/cos_alerter/daemon.py index 7e077a9..9cb6c49 100644 --- a/cos_alerter/daemon.py +++ b/cos_alerter/daemon.py @@ -110,8 +110,11 @@ def main(run_for: Optional[int] = None, argv: List[str] = sys.argv): dashboard_addr = None if dashboard_addr: # Split mode: separate API and dashboard servers - logger.info("Starting microservice mode - API server on %s, dashboard on %s", - config["web_listen_addr"], dashboard_addr) + logger.info( + "Starting microservice mode - API server on %s, dashboard on %s", + config["web_listen_addr"], + dashboard_addr, + ) # API server api_app = create_app(include_api=True, include_dashboard=False) @@ -143,8 +146,7 @@ def main(run_for: Optional[int] = None, argv: List[str] = sys.argv): else: # Combined mode: single server with both api and dashboard - logger.info("Starting monolith mode - App served on %s", - config["web_listen_addr"]) + logger.info("Starting monolith mode - App served on %s", config["web_listen_addr"]) app = create_app(include_api=True, include_dashboard=True) server_thread = threading.Thread( target=waitress.serve, diff --git a/cos_alerter/server.py b/cos_alerter/server.py index d8489b3..95ef71f 100644 --- a/cos_alerter/server.py +++ b/cos_alerter/server.py @@ -26,14 +26,16 @@ def create_app(include_api: bool = True, include_dashboard: bool = True) -> Flas Flask application instance """ app = Flask(__name__) - metrics = PrometheusMetrics(app) + metrics = PrometheusMetrics(app) # noqa: F841 if include_dashboard: + @app.route("/", methods=["GET"]) def dashboard_route(): return dashboard() if include_api: + @app.route("/alive", methods=["POST"]) def alive_route(): return alive() diff --git a/tests/test_server.py b/tests/test_server.py index 0db3c12..d48133a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -116,7 +116,7 @@ def test_multiple_key_values(flask_client, fake_fs, state_init): def test_create_app_api_only(fake_fs, state_init): app_instance = create_app(include_api=True, include_dashboard=False) client = app_instance.test_client() - + # Only API endpoint should be available assert client.get("/").status_code == 404 assert client.post("/alive", query_string=PARAMS).status_code == 200 @@ -125,7 +125,7 @@ def test_create_app_api_only(fake_fs, state_init): def test_create_app_dashboard_only(fake_fs, state_init): app_instance = create_app(include_api=False, include_dashboard=True) client = app_instance.test_client() - + # Only dashboard endpoint should be available assert client.get("/").status_code == 200 assert client.post("/alive", query_string=PARAMS).status_code == 404 From 9af63f5f5c3e9f59c547bbe349d44439aa496e16 Mon Sep 17 00:00:00 2001 From: Adhitya Ravi Date: Mon, 25 Aug 2025 12:00:02 +0200 Subject: [PATCH 6/6] chore: Minor refactors --- CHANGELOG.md | 2 +- cos_alerter/alerter.py | 7 +++++++ cos_alerter/daemon.py | 28 +++++++++++----------------- cos_alerter/server.py | 4 ---- tests/test_server.py | 3 ++- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18b7b82..dca948d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed deprecated job versions in the CI ## [0.10.0] - 2025-08-12 -- Added microservice mode support - API and dashboard can now run on separate addresses (#46). +- API and dashboard can now run on separate addresses (#46). ## [0.9.0] - 2024-05-30 diff --git a/cos_alerter/alerter.py b/cos_alerter/alerter.py index b380be5..7377976 100644 --- a/cos_alerter/alerter.py +++ b/cos_alerter/alerter.py @@ -80,6 +80,13 @@ def reload(self): self.data["notify"]["repeat_interval"] ).total_seconds() + # if dashboard address key is missing, set it to None + dashboard_addr = None + try: + dashboard_addr = self.data["dashboard_listen_addr"] + except KeyError: + self.data["dashboard_listen_addr"] = dashboard_addr + # Static variables. We define them here so it is easy to expose them later as config # values if needed. base_dir = xdg_base_dirs.xdg_state_home() / "cos_alerter" diff --git a/cos_alerter/daemon.py b/cos_alerter/daemon.py index 9cb6c49..1474803 100644 --- a/cos_alerter/daemon.py +++ b/cos_alerter/daemon.py @@ -102,18 +102,16 @@ def main(run_for: Optional[int] = None, argv: List[str] = sys.argv): # Starting in a thread rather than a new process allows waitress to inherit the log level # from the daemon. It also facilitates communication over memory rather than files. # clear_untrusted_proxy_headers is set to suppress a DeprecationWarning. + # If dashboard_lister_addr exists, serve api and dashboard in their own respective addresses - # Check if split mode is enabled (dashboard_listen_addr is configured) - try: - dashboard_addr = config["dashboard_listen_addr"] - except KeyError: - dashboard_addr = None - if dashboard_addr: - # Split mode: separate API and dashboard servers + dashboard_listen_addr = config["dashboard_listen_addr"] + web_listen_addr = config["web_listen_addr"] + + if dashboard_listen_addr: logger.info( - "Starting microservice mode - API server on %s, dashboard on %s", + "Starting API server on %s, dashboard on %s", config["web_listen_addr"], - dashboard_addr, + dashboard_listen_addr, ) # API server @@ -123,12 +121,11 @@ def main(run_for: Optional[int] = None, argv: List[str] = sys.argv): args=(api_app,), kwargs={ "clear_untrusted_proxy_headers": True, - "listen": config["web_listen_addr"], + "listen": web_listen_addr, }, ) api_server_thread.daemon = True api_server_thread.start() - logger.info("Started API server thread on %s", config["web_listen_addr"]) # Dashboard server dashboard_app = create_app(include_api=False, include_dashboard=True) @@ -137,28 +134,25 @@ def main(run_for: Optional[int] = None, argv: List[str] = sys.argv): args=(dashboard_app,), kwargs={ "clear_untrusted_proxy_headers": True, - "listen": dashboard_addr, + "listen": dashboard_listen_addr, }, ) dashboard_server_thread.daemon = True dashboard_server_thread.start() - logger.info("Started dashboard server thread on %s", dashboard_addr) else: - # Combined mode: single server with both api and dashboard - logger.info("Starting monolith mode - App served on %s", config["web_listen_addr"]) + logger.info("Starting API server and dashboard on %s", config["web_listen_addr"]) app = create_app(include_api=True, include_dashboard=True) server_thread = threading.Thread( target=waitress.serve, args=(app,), kwargs={ "clear_untrusted_proxy_headers": True, - "listen": config["web_listen_addr"], + "listen": web_listen_addr, }, ) server_thread.daemon = True server_thread.start() - logger.info("Started server thread on %s", config["web_listen_addr"]) for clientid in config["watch"]["clients"]: client_thread = threading.Thread(target=client_loop, args=(clientid,)) diff --git a/cos_alerter/server.py b/cos_alerter/server.py index 95ef71f..b99bbf4 100644 --- a/cos_alerter/server.py +++ b/cos_alerter/server.py @@ -109,7 +109,3 @@ def log_request(): request.method, request.url, ) - - -# Backwards compatibility - existing imports will still work -app = create_app() diff --git a/tests/test_server.py b/tests/test_server.py index d48133a..66d054e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -9,13 +9,14 @@ from werkzeug.datastructures import MultiDict from cos_alerter.alerter import AlerterState, config -from cos_alerter.server import app, create_app +from cos_alerter.server import create_app PARAMS = {"clientid": "clientid1", "key": "clientkey1"} @pytest.fixture def flask_client(): + app = create_app() return app.test_client()