Skip to content

Fixes #1246 and #1756 - Removed Hall of Fame task #1754

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Fixes #1246 and #1756 - Removed Hall of Fame task #1754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 179 additions & 0 deletions checks/migrations/0019_hall_of_fame_plus_triggers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
# Partly generated by Django 4.2.20 on 2025-05-25 15:24 together with manual RunSQL

from django.db import migrations, models
from django.db.models import Case, F, Q, Value, When
from django.db.models.functions import Greatest
from django.db.models.lookups import GreaterThan
import django.db.models.deletion
import pgtrigger.compiler
import pgtrigger.migrations


class Migration(migrations.Migration):
dependencies = [
("checks", "0018_domaintesttls_caa_records"),
]

operations = [
# Note db_index is False on the ForeignKey to prevent extra indices that are not needed
# AutoField has to be primary key in Django, to solve this manually alter SQL this field
# see https://github.com/django/django/blob/787f3130f751283140fe2be8188eb5299552232d/django/db/models/fields/__init__.py#L2801
migrations.CreateModel(
name="Fame",
fields=[
("id", models.IntegerField(serialize=False, verbose_name="ID")),
("domain", models.CharField(max_length=255, primary_key=True, serialize=False)),
(
"site_report",
models.ForeignKey(
db_index=False,
null=True,
on_delete=django.db.models.deletion.CASCADE,
to="checks.domaintestreport",
),
),
("site_report_timestamp", models.DateTimeField(null=True)),
(
"mail_report",
models.ForeignKey(
db_index=False,
null=True,
on_delete=django.db.models.deletion.CASCADE,
to="checks.mailtestreport",
),
),
("mail_report_timestamp", models.DateTimeField(null=True)),
],
),
migrations.AddIndex(
model_name="fame",
index=models.Index(
condition=models.Q(site_report_id__isnull=False),
fields=["-site_report_timestamp", "domain", "site_report_id"],
name="checks_fame_sites_idx",
),
),
migrations.AddIndex(
model_name="fame",
index=models.Index(
condition=models.Q(mail_report_id__isnull=False),
fields=["-mail_report_timestamp", "domain", "mail_report_id"],
name="checks_fame_mail_idx",
),
),
migrations.AddIndex(
model_name="fame",
index=models.Index(
models.OrderBy(
Greatest("site_report_timestamp", "mail_report_timestamp"),
descending=True,
),
"domain",
Case(
models.When(
GreaterThan(F("site_report_timestamp"), F("mail_report_timestamp")),
then=Value("s"),
),
default=Value("m"),
output_field=models.CharField(max_length=1),
),
Case(
When(
GreaterThan(F("site_report_timestamp"), F("mail_report_timestamp")),
then="site_report_id",
),
default="mail_report_id",
),
condition=Q(site_report_id__isnull=False) & Q(mail_report_id__isnull=False),
name="checks_fame_champions_idx",
),
),
pgtrigger.migrations.AddTrigger(
model_name="domaintestreport",
trigger=pgtrigger.compiler.Trigger(
name="update_fame_on_site_report",
sql=pgtrigger.compiler.UpsertTriggerSql(
func="""
IF NEW.score IS NULL THEN
-- DO NOTHING
ELSIF NEW.score = 100 THEN
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
VALUES (NEW.domain, NEW.id, NEW.timestamp, NULL, NULL)
ON CONFLICT (domain)
DO UPDATE SET site_report_id = NEW.id, site_report_timestamp = NEW.timestamp;
ELSE
MERGE INTO ONLY checks_fame c1
USING checks_fame c2 ON c1.domain = c2.domain AND c1.domain = NEW.domain
WHEN NOT MATCHED THEN
DO NOTHING
WHEN MATCHED AND c1.mail_report_id IS NOT NULL THEN
UPDATE SET site_report_id = NULL, site_report_timestamp = NULL
WHEN MATCHED AND c1.mail_report_id IS NULL THEN
DELETE;
END IF;
RETURN NEW;
""",
hash="b4f792b06123914de71b57669c202a19b04e9e9c",
operation='INSERT OR UPDATE OF "score"',
pgid="pgtrigger_update_fame_on_site_report_e4fdc",
table="checks_domaintestreport",
when="AFTER",
),
),
),
pgtrigger.migrations.AddTrigger(
model_name="mailtestreport",
trigger=pgtrigger.compiler.Trigger(
name="update_fame_on_mail_report",
sql=pgtrigger.compiler.UpsertTriggerSql(
func="""
IF NEW.score IS NULL THEN
-- DO NOTHING
ELSIF NEW.score = 100 THEN
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
VALUES (NEW.domain, NULL, NULL, NEW.id, NEW.timestamp)
ON CONFLICT (domain)
DO UPDATE SET mail_report_id = NEW.id, mail_report_timestamp = NEW.timestamp;
ELSE
MERGE INTO ONLY checks_fame c1
USING checks_fame c2 ON c1.domain = c2.domain AND c1.domain = NEW.domain
WHEN NOT MATCHED THEN
DO NOTHING
WHEN MATCHED AND c1.site_report_id IS NOT NULL THEN
UPDATE SET mail_report_id = NULL, mail_report_timestamp = NULL
WHEN MATCHED AND c1.site_report_id IS NULL THEN
DELETE;
END IF;
RETURN NEW;
""",
hash="707aefc7a83dd041dd815511f1d1cf7e8f84f944",
operation='INSERT OR UPDATE OF "score"',
pgid="pgtrigger_update_fame_on_mail_report_b3a27",
table="checks_mailtestreport",
when="AFTER",
),
),
),
migrations.RunSQL(
sql=[
'ALTER TABLE "checks_fame" ALTER COLUMN "id" ADD GENERATED BY DEFAULT AS IDENTITY;',
"""
WITH
site_fame AS (
SELECT domain, id AS site_report_id, timestamp AS site_report_timestamp FROM (
SELECT domain, score, id, timestamp, rank() OVER (PARTITION BY domain ORDER BY id DESC) FROM checks_domaintestreport
) alias WHERE rank = 1 AND score = 100),
mail_fame AS (
SELECT domain, id AS mail_report_id, timestamp AS mail_report_timestamp FROM (
SELECT domain, score, id, timestamp, rank() OVER (PARTITION BY domain ORDER BY id DESC) FROM checks_mailtestreport
) alias WHERE rank = 1 AND score = 100)
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
SELECT * FROM site_fame FULL OUTER JOIN mail_fame USING (domain);
""",
],
reverse_sql=[
'DELETE FROM "checks_fame";',
'ALTER TABLE "checks_fame" ALTER COLUMN "id" DROP IDENTITY;',
],
),
]
105 changes: 105 additions & 0 deletions checks/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,13 @@

from django.core.exceptions import SuspiciousFileOperation
from django.db import models, transaction
from django.db.models import Case, F, Q, Value, When
from django.db.models.functions import Greatest
from django.db.models.lookups import GreaterThan
from django.utils import timezone
from enumfields import Enum as LabelEnum
from enumfields import EnumField, EnumIntegerField
import pgtrigger


class ListField(models.TextField):
Expand Down Expand Up @@ -947,6 +951,34 @@ def __dir__(self):
class Meta:
app_label = "checks"

triggers = [
pgtrigger.Trigger(
name="update_fame_on_site_report",
when=pgtrigger.After,
operation=pgtrigger.Insert | pgtrigger.UpdateOf("score"),
func="""
IF NEW.score IS NULL THEN
-- DO NOTHING
ELSIF NEW.score = 100 THEN
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
VALUES (NEW.domain, NEW.id, NEW.timestamp, NULL, NULL)
ON CONFLICT (domain)
DO UPDATE SET site_report_id = NEW.id, site_report_timestamp = NEW.timestamp;
ELSE
MERGE INTO ONLY checks_fame c1
USING checks_fame c2 ON c1.domain = c2.domain AND c1.domain = NEW.domain
WHEN NOT MATCHED THEN
DO NOTHING
WHEN MATCHED AND c1.mail_report_id IS NOT NULL THEN
UPDATE SET site_report_id = NULL, site_report_timestamp = NULL
WHEN MATCHED AND c1.mail_report_id IS NULL THEN
DELETE;
END IF;
RETURN NEW;
""",
),
]


###
# Mail test
Expand Down Expand Up @@ -1093,6 +1125,79 @@ def __dir__(self):
class Meta:
app_label = "checks"

triggers = [
pgtrigger.Trigger(
name="update_fame_on_mail_report",
when=pgtrigger.After,
operation=pgtrigger.Insert | pgtrigger.UpdateOf("score"),
func="""
IF NEW.score IS NULL THEN
-- DO NOTHING
ELSIF NEW.score = 100 THEN
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
VALUES (NEW.domain, NULL, NULL, NEW.id, NEW.timestamp)
ON CONFLICT (domain)
DO UPDATE SET mail_report_id = NEW.id, mail_report_timestamp = NEW.timestamp;
ELSE
MERGE INTO ONLY checks_fame c1
USING checks_fame c2 ON c1.domain = c2.domain AND c1.domain = NEW.domain
WHEN NOT MATCHED THEN
DO NOTHING
WHEN MATCHED AND c1.site_report_id IS NOT NULL THEN
UPDATE SET mail_report_id = NULL, mail_report_timestamp = NULL
WHEN MATCHED AND c1.site_report_id IS NULL THEN
DELETE;
END IF;
RETURN NEW;
""",
),
]


class Fame(models.Model):
id = models.IntegerField(serialize=False, verbose_name="ID")
domain = models.CharField(max_length=255, primary_key=True, serialize=False)
site_report = models.ForeignKey(DomainTestReport, null=True, on_delete=models.CASCADE, db_index=False)
site_report_timestamp = models.DateTimeField(null=True)
mail_report = models.ForeignKey(MailTestReport, null=True, on_delete=models.CASCADE, db_index=False)
mail_report_timestamp = models.DateTimeField(null=True)

def __dir__(self):
return ["domain", "site_report", "site_report_timestamp", "mail_report", "mail_report_timestamp"]

class Meta:
app_label = "checks"

indexes = [
models.Index(
condition=Q(site_report_id__isnull=False),
fields=["-site_report_timestamp", "domain", "site_report_id"],
name="checks_fame_sites_idx",
),
models.Index(
condition=Q(mail_report_id__isnull=False),
fields=["-mail_report_timestamp", "domain", "mail_report_id"],
name="checks_fame_mail_idx",
),
# TODO: is there a way to alias/annotate the expressions?
# (so psql `\d checks_fame_champions_idx` looks nice)
models.Index(
Greatest("site_report_timestamp", "mail_report_timestamp").desc(),
"domain",
Case(
When(GreaterThan(F("site_report_timestamp"), F("mail_report_timestamp")), then=Value("s")),
default=Value("m"),
output_field=models.CharField(max_length=1),
),
Case(
When(GreaterThan(F("site_report_timestamp"), F("mail_report_timestamp")), then="site_report_id"),
default="mail_report_id",
),
condition=Q(site_report_id__isnull=False) & Q(mail_report_id__isnull=False),
name="checks_fame_champions_idx",
),
]


class BatchUser(models.Model):
"""
Expand Down
Loading
Loading