feat: database-backed artifact blob storage

dev/build/migration-start.sh

@@ -3,4 +3,7 @@
 echo "Running Datatracker migrations..."
 ./ietf/manage.py migrate --settings=settings_local
 
+echo "Running Blobdb migrations ..."
+./ietf/manage.py migrate --settings=settings_local --database=blobdb
+
 echo "Done!"

dev/build/start.sh

@@ -2,7 +2,7 @@
 #
 # Environment config:
 #
-#  CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker)
+#  CONTAINER_ROLE - datatracker, celery, beat, migrations, or replicator (defaults to datatracker)
 #
 case "${CONTAINER_ROLE:-datatracker}" in
     auth)
@@ -20,6 +20,9 @@ case "${CONTAINER_ROLE:-datatracker}" in
     migrations)
         exec ./migration-start.sh
         ;;
+    replicator)
+        exec ./celery-start.sh --app=ietf worker --queues=blobdb --concurrency=1
+        ;;
     *)
         echo "Unknown role '${CONTAINER_ROLE}'"
         exit 255       

dev/celery/docker-init.sh

@@ -99,7 +99,7 @@ if [[ -n "${DEV_MODE}" ]]; then
             --recursive \
             --debounce-interval 5 \
             -- \
-            celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" &
+            celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" $@ &
   celery_pid=$!
 else
   celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" &

dev/deploy-to-container/settings_local.py

@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings import *   # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -81,22 +79,3 @@
 
 # OIDC configuration
 SITE_URL = 'https://__HOSTNAME__'
-
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            verify=False,
-            bucket_name=f"test-{storagename}",
-        ),
-    }

dev/diff/settings_local.py

@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings import *  # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -68,22 +66,3 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
-
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            verify=False,
-            bucket_name=f"test-{storagename}",
-        ),
-    }

dev/tests/settings_local.py

@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings import *  # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -67,22 +65,3 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
-
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            verify=False,
-            bucket_name=f"test-{storagename}",
-        ),
-    }

docker-compose.yml

@@ -86,13 +86,44 @@ services:
             - .:/workspace
             - app-assets:/assets
 
+    replicator:
+        build:
+            context: .
+            dockerfile: docker/celery.Dockerfile
+        init: true
+        environment:
+            CELERY_APP: ietf
+            CELERY_ROLE: worker
+            UPDATE_REQUIREMENTS_FROM: requirements.txt
+            DEV_MODE: "yes"
+        command:
+            - '--loglevel=INFO'
+            - '--queues=blobdb'
+            - '--concurrency=1'
+
+        depends_on:
+            - db
+        restart: unless-stopped
+        stop_grace_period: 1m
+        volumes:
+            - .:/workspace
+            - app-assets:/assets
+
     blobstore:
         image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
         restart: unless-stopped
         volumes:
             - "minio-data:/data"
 
-
+    blobdb:
+        image: postgres:16
+        restart: unless-stopped
+        environment:
+            POSTGRES_DB: blob
+            POSTGRES_USER: dt
+            POSTGRES_PASSWORD: abcd1234
+        volumes:
+            - blobdb-data:/var/lib/postgresql/data
 
 #  Celery Beat is a periodic task runner. It is not normally needed for development,
 #  but can be enabled by uncommenting the following.
@@ -118,3 +149,4 @@ volumes:
     postgresdb-data:
     app-assets:
     minio-data:
+    blobdb-data:

docker/configs/settings_local.py

@@ -2,12 +2,26 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings import *  # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
+from ietf.settings import (
+    ARTIFACT_STORAGE_NAMES,
+    STORAGES,
+    BLOBSTORAGE_MAX_ATTEMPTS,
+    BLOBSTORAGE_READ_TIMEOUT,
+    BLOBSTORAGE_CONNECT_TIMEOUT,
+)
 
 ALLOWED_HOSTS = ['*']
 
 from ietf.settings_postgresqldb import DATABASES  # pyflakes:ignore
+DATABASE_ROUTERS = ["ietf.blobdb.routers.BlobdbStorageRouter"]
+BLOBDB_DATABASE = "blobdb"
+BLOBDB_REPLICATION = {
+    "ENABLED": True,
+    "DEST_STORAGE_PATTERN": "r2-{bucket}",
+    "INCLUDE_BUCKETS": ARTIFACT_STORAGE_NAMES,
+    "EXCLUDE_BUCKETS": ["staging"],
+    "VERBOSE_LOGGING": True,
+}
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
@@ -39,25 +53,6 @@
 # DEV_TEMPLATE_CONTEXT_PROCESSORS = [
 #    'ietf.context_processors.sql_debug',
 # ]
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            verify=False,
-            bucket_name=storagename,
-        ),
-    }
-
 
 DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/'
 INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/'
@@ -80,3 +75,26 @@
 
 STATIC_IETF_ORG = "/_static"
 STATIC_IETF_ORG_INTERNAL = "http://static"
+
+
+# Blob replication storage for dev
+import botocore.config
+for storagename in ARTIFACT_STORAGE_NAMES:
+    replica_storagename = f"r2-{storagename}"
+    STORAGES[replica_storagename] = {
+        "BACKEND": "ietf.doc.storage.MetadataS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=f"{storagename}",
+        ),
+    }

docker/configs/settings_postgresqldb.py

@@ -7,4 +7,12 @@
         'USER': 'django',
         'PASSWORD': 'RkTkDPFnKpko',
     },
+    'blobdb': {
+        'HOST': 'blobdb',
+        'PORT': 5432,
+        'NAME': 'blob',
+        'ENGINE': 'django.db.backends.postgresql',
+        'USER': 'dt',
+        'PASSWORD': 'abcd1234',
+    },
 }

docker/scripts/app-configure-blobstore.py

@@ -7,7 +7,7 @@
 import os
 import sys
 
-from ietf.settings import MORE_STORAGE_NAMES
+from ietf.settings import ARTIFACT_STORAGE_NAMES
 
 
 def init_blobstore():
@@ -19,7 +19,7 @@ def init_blobstore():
         aws_session_token=None,
         config=botocore.config.Config(signature_version="s3v4"),
     )
-    for bucketname in MORE_STORAGE_NAMES:
+    for bucketname in ARTIFACT_STORAGE_NAMES:
         try:
             blobstore.create_bucket(
                 Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()

docker/scripts/app-init.sh

@@ -101,9 +101,11 @@ echo "Running initial checks..."
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local
 
 # Migrate, adjusting to what the current state of the underlying database might be:
-
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --fake-initial --settings=settings_local
 
+# Apply migrations to the blobdb database as well (most are skipped)
+/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local --database=blobdb
+
 if [ -z "$EDITOR_VSCODE" ]; then
     CODE=0
     python -m smtpd -n -c DebuggingServer localhost:2025 &

ietf/api/tests.py

@@ -50,6 +50,7 @@
     'ietf.secr.proceedings',
     'ietf.ipr',
     'ietf.status',
+    'ietf.blobdb',
 )
 
 class CustomApiTests(TestCase):

ietf/blobdb/admin.py

@@ -0,0 +1,31 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from django.contrib import admin
+from django.db.models.functions import Length
+from rangefilter.filters import DateRangeQuickSelectListFilterBuilder
+
+from .models import Blob
+
+
+@admin.register(Blob)
+class BlobAdmin(admin.ModelAdmin):
+    list_display = ["bucket", "name", "object_size", "modified", "mtime", "content_type"]
+    list_filter = [
+        "bucket",
+        "content_type",
+        ("modified", DateRangeQuickSelectListFilterBuilder()),
+        ("mtime", DateRangeQuickSelectListFilterBuilder()),
+    ]
+    search_fields = ["name"]
+    list_display_links = ["name"]
+
+    def get_queryset(self, request):
+        return (
+            super().get_queryset(request)
+            .defer("content")  # don't load this unless we want it
+            .annotate(object_size=Length("content"))  # accessed via object_size()
+        )
+
+    @admin.display(ordering="object_size")
+    def object_size(self, instance):
+        """Get the size of the object"""
+        return instance.object_size  # annotation added in get_queryset()

ietf/blobdb/apps.py

@@ -0,0 +1,30 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from django.apps import AppConfig
+
+
+class BlobdbConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "ietf.blobdb"
+
+    def ready(self):
+        """Initialize app once the registries / settings are populated"""
+        from django.conf import settings
+
+        # Validate that the DB is set up
+        db = get_blobdb()  # depends on settings.BLOBDB_DATABASE
+        if db is not None and db not in settings.DATABASES:
+            raise RuntimeError(
+                f"settings.BLOBDB_DATABASE is '{db}' but that is not present in settings.DATABASES"
+            )
+
+        # Validate replication settings
+        from .replication import validate_replication_settings
+
+        validate_replication_settings()
+
+
+def get_blobdb():
+    """Retrieve the blobdb setting from Django's settings"""
+    from django.conf import settings
+
+    return getattr(settings, "BLOBDB_DATABASE", None)

ietf/blobdb/factories.py

@@ -0,0 +1,13 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import factory
+
+from .models import Blob
+
+
+class BlobFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = Blob
+
+    name = factory.Faker("file_path")
+    bucket = factory.Faker("word")
+    content = factory.Faker("binary", length=32)  # careful, default length is 1e6