feat(API): Support read only replica configuration for Postgres via environment variables

.gitignore

@@ -44,6 +44,7 @@ junit-reports/
 
 # Cursor files
 .cursorignore
+.cursor/
 
 # Terraform
 .terraform*

api/.env.example

@@ -29,6 +29,9 @@ DJANGO_SENTRY_DSN=
 # If running django and celery on host, use 'localhost', else use 'postgres-db'
 POSTGRES_HOST=[localhost|postgres-db]
 POSTGRES_PORT=5432
+# If you are running a replica only for read queries. Defaults to the same value as POSTGRES_HOST and POSTGRES_PORT
+POSTGRES_HOST_READ_ONLY=[localhost|postgres-db]
+POSTGRES_PORT_READ_ONLY=5432
 POSTGRES_ADMIN_USER=prowler
 POSTGRES_ADMIN_PASSWORD=S3cret
 POSTGRES_USER=prowler_user

api/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to the **Prowler API** are documented in this file.
 
+## [v1.9.0] (Prowler UNRELEASED)
+
+### Added
+- Support for read only replicas in the database [(#7210)](https://github.com/prowler-cloud/prowler/pull/7210).
+
+---
+
 ## [v1.8.1] (Prowler v5.7.1)
 
 ### Fixed
@@ -33,7 +40,6 @@ All notable changes to the **Prowler API** are documented in this file.
 ## [v1.6.0] (Prowler v5.5.0)
 
 ### Added
-
 - Support for developing new integrations [(#7167)](https://github.com/prowler-cloud/prowler/pull/7167).
 - HTTP Security Headers [(#7289)](https://github.com/prowler-cloud/prowler/pull/7289).
 - New endpoint to get the compliance overviews metadata [(#7333)](https://github.com/prowler-cloud/prowler/pull/7333).
@@ -71,7 +77,6 @@ All notable changes to the **Prowler API** are documented in this file.
 - Fixed a race condition when deleting export files after the S3 upload [(#7172)](https://github.com/prowler-cloud/prowler/pull/7172).
 - Handled exception when a provider has no secret in test connection [(#7283)](https://github.com/prowler-cloud/prowler/pull/7283).
 
-
 ---
 
 ## [v1.5.0] (Prowler v5.4.0)

api/src/backend/api/db_router.py

@@ -3,27 +3,30 @@
 
 class MainRouter:
     default_db = "default"
+    default_read = "default_read"
+    prowler_user = "prowler_user"
     admin_db = "admin"
+    admin_read = "admin_read"
+
 
     def db_for_read(self, model, **hints):  # noqa: F841
         model_table_name = model._meta.db_table
-        if model_table_name.startswith("django_") or any(
-            model_table_name.startswith(f"{app}_") for app in ALLOWED_APPS
-        ):
-            return self.admin_db
-        return None
+        if any(model_table_name.startswith(f"{app}_") for app in ALLOWED_APPS):
+            return self.admin_read
+        return self.default_read
 
     def db_for_write(self, model, **hints):  # noqa: F841
         model_table_name = model._meta.db_table
         if any(model_table_name.startswith(f"{app}_") for app in ALLOWED_APPS):
             return self.admin_db
-        return None
+        return self.default_db
 
     def allow_migrate(self, db, app_label, model_name=None, **hints):  # noqa: F841
         return db == self.admin_db
 
     def allow_relation(self, obj1, obj2, **hints):  # noqa: F841
-        # Allow relations if both objects are in either "default" or "admin" db connectors
-        if {obj1._state.db, obj2._state.db} <= {self.default_db, self.admin_db}:
+        # Allow relations if both objects are using one of our defined connectors
+        allowed = {self.default_db, self.default_read, self.admin_db, self.admin_read, self.prowler_user}
+        if {obj1._state.db, obj2._state.db} <= allowed:
             return True
         return None

api/src/backend/api/rbac/permissions.py

@@ -30,7 +30,9 @@ def has_permission(self, request, view):
             return True
 
         user_roles = (
-            User.objects.using(MainRouter.admin_db).get(id=request.user.id).roles.all()
+            User.objects.using(MainRouter.admin_read)
+            .get(id=request.user.id)
+            .roles.all()
         )
         if not user_roles:
             return False

api/src/backend/api/tests/test_database.py

@@ -1,15 +1,17 @@
+from unittest.mock import patch
+
 import pytest
+from config.django.base import DATABASE_ROUTERS as PROD_DATABASE_ROUTERS
 from django.conf import settings
 from django.db.migrations.recorder import MigrationRecorder
 from django.db.utils import ConnectionRouter
 
 from api.db_router import MainRouter
 from api.rls import Tenant
-from config.django.base import DATABASE_ROUTERS as PROD_DATABASE_ROUTERS
-from unittest.mock import patch
 
 
 @patch("api.db_router.MainRouter.admin_db", new="admin")
+@patch("api.db_router.MainRouter.admin_read", new="admin_read")
 class TestMainDatabaseRouter:
     @pytest.fixture(scope="module")
     def router(self):
@@ -20,12 +22,12 @@ def router(self):
 
     @pytest.mark.parametrize("api_model", [Tenant])
     def test_router_api_models(self, api_model, router):
-        assert router.db_for_read(api_model) == "default"
+        assert router.db_for_read(api_model) == "prowler_user_read"
         assert router.db_for_write(api_model) == "default"
 
         assert router.allow_migrate_model(MainRouter.admin_db, api_model)
         assert not router.allow_migrate_model("default", api_model)
 
     def test_router_django_models(self, router):
-        assert router.db_for_read(MigrationRecorder.Migration) == MainRouter.admin_db
-        assert not router.db_for_read(MigrationRecorder.Migration) == "default"
+        assert router.db_for_read(MigrationRecorder.Migration) == MainRouter.admin_read
+        assert router.db_for_read(MigrationRecorder.Migration) != "default"

api/src/backend/api/utils.py

@@ -186,7 +186,7 @@ def validate_invitation(
     try:
         # Admin DB connector is used to bypass RLS protection since the invitation belongs to a tenant the user
         # is not a member of yet
-        invitation = Invitation.objects.using(MainRouter.admin_db).get(
+        invitation = Invitation.objects.using(MainRouter.admin_read).get(
             token=invitation_token, email=email
         )
     except Invitation.DoesNotExist:

api/src/backend/api/v1/views.py

@@ -2197,7 +2197,7 @@ def accept(self, request):
         )
 
         # Proceed with accepting the invitation
-        user = User.objects.using(MainRouter.admin_db).get(email=user_email)
+        user = User.objects.using(MainRouter.admin_read).get(email=user_email)
         membership = Membership.objects.using(MainRouter.admin_db).create(
             user=user,
             tenant=invitation.tenant,

api/src/backend/config/django/devel.py

@@ -14,6 +14,19 @@
         "HOST": env("POSTGRES_HOST", default="postgres-db"),
         "PORT": env("POSTGRES_PORT", default="5432"),
     },
+    "default_read": {
+        "ENGINE": "psqlextra.backend",
+        "NAME": env("POSTGRES_DB", default="prowler_db"),
+        "USER": env("POSTGRES_USER", default="prowler_user"),
+        "PASSWORD": env("POSTGRES_PASSWORD", default="prowler"),
+        "HOST": env(
+            "POSTGRES_HOST_READ_ONLY",
+            default=env("POSTGRES_HOST", default="postgres-db"),
+        ),
+        "PORT": env(
+            "POSTGRES_PORT_READ_ONLY", default=env("POSTGRES_PORT", default="5432")
+        ),
+    },
     "admin": {
         "ENGINE": "psqlextra.backend",
         "NAME": env("POSTGRES_DB", default="prowler_db"),
@@ -22,6 +35,19 @@
         "HOST": env("POSTGRES_HOST", default="postgres-db"),
         "PORT": env("POSTGRES_PORT", default="5432"),
     },
+    "admin_read": {
+        "ENGINE": "psqlextra.backend",
+        "NAME": env("POSTGRES_DB", default="prowler_db"),
+        "USER": env("POSTGRES_ADMIN_USER", default="prowler"),
+        "PASSWORD": env("POSTGRES_ADMIN_PASSWORD", default="S3cret"),
+        "HOST": env(
+            "POSTGRES_HOST_READ_ONLY",
+            default=env("POSTGRES_HOST", default="postgres-db"),
+        ),
+        "PORT": env(
+            "POSTGRES_PORT_READ_ONLY", default=env("POSTGRES_PORT", default="5432")
+        ),
+    },
 }
 DATABASES["default"] = DATABASES["prowler_user"]
 

api/src/backend/config/django/production.py

@@ -5,7 +5,6 @@
 ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"])
 
 # Database
-# TODO Use Django database routers https://docs.djangoproject.com/en/5.0/topics/db/multi-db/#automatic-database-routing
 DATABASES = {
     "prowler_user": {
         "ENGINE": "django.db.backends.postgresql",
@@ -15,6 +14,14 @@
         "HOST": env("POSTGRES_HOST"),
         "PORT": env("POSTGRES_PORT"),
     },
+    "default_read": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": env("POSTGRES_DB"),
+        "USER": env("POSTGRES_USER"),
+        "PASSWORD": env("POSTGRES_PASSWORD"),
+        "HOST": env("POSTGRES_HOST_READ_ONLY", default=env("POSTGRES_HOST")),
+        "PORT": env("POSTGRES_PORT_READ_ONLY", default=env("POSTGRES_PORT")),
+    },
     "admin": {
         "ENGINE": "psqlextra.backend",
         "NAME": env("POSTGRES_DB"),
@@ -23,5 +30,13 @@
         "HOST": env("POSTGRES_HOST"),
         "PORT": env("POSTGRES_PORT"),
     },
+    "admin_read": {
+        "ENGINE": "psqlextra.backend",
+        "NAME": env("POSTGRES_DB"),
+        "USER": env("POSTGRES_ADMIN_USER"),
+        "PASSWORD": env("POSTGRES_ADMIN_PASSWORD"),
+        "HOST": env("POSTGRES_HOST_READ_ONLY", default=env("POSTGRES_HOST")),
+        "PORT": env("POSTGRES_PORT_READ_ONLY", default=env("POSTGRES_PORT")),
+    },
 }
 DATABASES["default"] = DATABASES["prowler_user"]

api/src/backend/conftest.py

@@ -983,6 +983,7 @@ def pytest_configure(config):
     # Apply the mock before the test session starts. This is necessary to avoid admin error when running the
     # 0004_rbac_missing_admin_roles migration
     patch("api.db_router.MainRouter.admin_db", new="default").start()
+    patch("api.db_router.MainRouter.admin_read", new="default").start()
 
 
 def pytest_unconfigure(config):

api/src/backend/tasks/jobs/deletion.py

@@ -64,7 +64,7 @@ def delete_tenant(pk: str):
     """
     deletion_summary = {}
 
-    for provider in Provider.objects.using(MainRouter.admin_db).filter(tenant_id=pk):
+    for provider in Provider.objects.using(MainRouter.admin_read).filter(tenant_id=pk):
         summary = delete_provider(pk, provider.id)
         deletion_summary.update(summary)