Webhook behavior customization and delivery records (#1338)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -13,6 +13,12 @@ v34.7.2 (unreleased)
   related to the Pipeline run.
   https://github.com/nexB/scancode.io/pull/1330
 
+- Expands on the existing WebhookSubscription model by adding a few fields to
+  configure the behavior of the Webhooks, and moves some of the fields to a new
+  WebhookDelivery model, which captures the results of a WebhookSubscription
+  "delivery".
+  https://github.com/nexB/scancode.io/issues/1325
+
 v34.7.1 (2024-07-15)
 --------------------
 

scanpipe/api/serializers.py

@@ -36,6 +36,7 @@
 from scanpipe.models import Project
 from scanpipe.models import ProjectMessage
 from scanpipe.models import Run
+from scanpipe.models import WebhookSubscription
 from scanpipe.pipes import count_group_by
 
 scanpipe_app = apps.get_app_config("scanpipe")
@@ -143,6 +144,18 @@ class Meta:
         ]
 
 
+class WebhookSubscriptionSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = WebhookSubscription
+        fields = [
+            "target_url",
+            "trigger_on_each_run",
+            "include_summary",
+            "include_results",
+            "is_active",
+        ]
+
+
 class ProjectSerializer(
     ExcludeFromListViewMixin,
     SerializerExcludeFieldsMixin,
@@ -158,6 +171,7 @@ class ProjectSerializer(
     execute_now = serializers.BooleanField(
         write_only=True,
         help_text="Execute pipeline now",
+        required=False,
     )
     upload_file = serializers.FileField(write_only=True, required=False)
     upload_file_tag = serializers.CharField(write_only=True, required=False)
@@ -167,6 +181,7 @@ class ProjectSerializer(
         style={"base_template": "textarea.html"},
     )
     webhook_url = serializers.CharField(write_only=True, required=False)
+    webhooks = WebhookSubscriptionSerializer(many=True, write_only=True, required=False)
     next_run = serializers.CharField(source="get_next_run", read_only=True)
     runs = RunSerializer(many=True, read_only=True)
     input_sources = InputSourceSerializer(
@@ -192,6 +207,7 @@ class Meta:
             "upload_file_tag",
             "input_urls",
             "webhook_url",
+            "webhooks",
             "created_date",
             "is_archived",
             "notes",
@@ -289,6 +305,7 @@ def create(self, validated_data):
         pipeline = validated_data.pop("pipeline", [])
         execute_now = validated_data.pop("execute_now", False)
         webhook_url = validated_data.pop("webhook_url", None)
+        webhooks = validated_data.pop("webhooks", [])
 
         project = super().create(validated_data)
 
@@ -298,12 +315,15 @@ def create(self, validated_data):
         for url in input_urls:
             project.add_input_source(download_url=url)
 
-        if webhook_url:
-            project.add_webhook_subscription(webhook_url)
-
         for pipeline_name in pipeline:
             project.add_pipeline(pipeline_name, execute_now)
 
+        if webhook_url:
+            project.add_webhook_subscription(target_url=webhook_url)
+
+        for webhook_data in webhooks:
+            project.add_webhook_subscription(**webhook_data)
+
         return project
 
 

scanpipe/migrations/0064_webhookdelivery.py

@@ -0,0 +1,37 @@
+# Generated by Django 5.0.7 on 2024-07-24 05:22
+
+import django.db.models.deletion
+import scanpipe.models
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0063_run_selected_steps'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='WebhookDelivery',
+            fields=[
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='UUID')),
+                ('target_url', models.URLField(help_text='Stores a copy of the Webhook target URL in case the subscription object is deleted.', max_length=1024, verbose_name='Target URL')),
+                ('sent_date', models.DateTimeField(auto_now_add=True, help_text='The date and time when the Webhook was sent.')),
+                ('payload', models.JSONField(blank=True, default=dict, help_text='The JSON payload that was sent to the target URL.')),
+                ('response_status_code', models.PositiveIntegerField(blank=True, help_text='The HTTP status code received in response to the Webhook request.', null=True)),
+                ('response_text', models.TextField(blank=True, help_text='The text response received from the target URL.')),
+                ('delivery_error', models.TextField(blank=True, help_text='Any error messages encountered during the Webhook delivery.')),
+                ('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='webhookdeliveries', to='scanpipe.project')),
+                ('run', models.ForeignKey(blank=True, editable=False, help_text='The Pipeline Run associated with this delivery.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='webhook_deliveries', to='scanpipe.run')),
+                ('webhook_subscription', models.ForeignKey(blank=True, editable=False, help_text='The Webhook subscription associated with this delivery.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deliveries', to='scanpipe.webhooksubscription')),
+            ],
+            options={
+                'verbose_name': 'webhook delivery',
+                'verbose_name_plural': 'webhook deliveries',
+                'ordering': ['-sent_date'],
+            },
+            bases=(scanpipe.models.UpdateMixin, models.Model),
+        ),
+    ]

scanpipe/migrations/0065_webhookdelivery_data.py

@@ -0,0 +1,42 @@
+# Generated by Django 5.0.7 on 2024-07-24 05:24
+
+from django.db import migrations
+
+
+def migrate_webhook_delivery_data(apps, schema_editor):
+    WebhookSubscription = apps.get_model("scanpipe", "WebhookSubscription")
+    WebhookDelivery = apps.get_model("scanpipe", "WebhookDelivery")
+
+    subscription_qs = WebhookSubscription.objects.all().select_related("project")
+
+    for subscription in subscription_qs:
+        project = subscription.project
+
+        # Asserting the run FK in case this project has a single run
+        project_runs = project.runs.all()
+        run = project_runs[0] if len(project_runs) == 1 else None
+
+        WebhookDelivery.objects.create(
+            project=project,
+            run=run,
+            webhook_subscription=subscription,
+            target_url=subscription.target_url,
+            sent_date=subscription.created_date,
+            payload={},
+            response_status_code=subscription.response_status_code,
+            response_text=subscription.response_text,
+            delivery_error=subscription.delivery_error,
+        )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0064_webhookdelivery"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            migrate_webhook_delivery_data,
+            reverse_code=migrations.RunPython.noop,
+        ),
+    ]

scanpipe/migrations/0066_alter_webhooksubscription_options_and_more.py

@@ -0,0 +1,59 @@
+# Generated by Django 5.0.7 on 2024-07-24 05:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0065_webhookdelivery_data'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='webhooksubscription',
+            options={'ordering': ['-created_date']},
+        ),
+        migrations.RemoveField(
+            model_name='webhooksubscription',
+            name='delivery_error',
+        ),
+        migrations.RemoveField(
+            model_name='webhooksubscription',
+            name='response_status_code',
+        ),
+        migrations.RemoveField(
+            model_name='webhooksubscription',
+            name='response_text',
+        ),
+        migrations.AddField(
+            model_name='webhooksubscription',
+            name='include_results',
+            field=models.BooleanField(default=False, help_text='Include the entire results data in the payload.'),
+        ),
+        migrations.AddField(
+            model_name='webhooksubscription',
+            name='include_summary',
+            field=models.BooleanField(default=False, help_text='Include the entire summary data in the payload.'),
+        ),
+        migrations.AddField(
+            model_name='webhooksubscription',
+            name='is_active',
+            field=models.BooleanField(default=True, help_text='Indicates whether the Webhook is currently active and should be triggered.'),
+        ),
+        migrations.AddField(
+            model_name='webhooksubscription',
+            name='trigger_on_each_run',
+            field=models.BooleanField(default=False, help_text='Trigger the Webhook after each individual pipeline run instead of only after all runs are completed.'),
+        ),
+        migrations.AlterField(
+            model_name='webhooksubscription',
+            name='created_date',
+            field=models.DateTimeField(auto_now_add=True, help_text='The date and time when the Webhook subscription was created.'),
+        ),
+        migrations.AlterField(
+            model_name='webhooksubscription',
+            name='target_url',
+            field=models.URLField(help_text='The URL to which the POST request will be sent when the Webhook is triggered.', max_length=1024, verbose_name='Target URL'),
+        ),
+    ]