Skip to content

Commit 88f8a89

Browse files
bwbroersmabwbroersma
committed
Fix #1246 - Improve Hall of Fame task
1 parent cef300d commit 88f8a89

File tree

2 files changed

+143
-89
lines changed

2 files changed

+143
-89
lines changed

‎checks/migrations/0019_hof.py

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
import django.db.models.deletion
2+
import django.utils.timezone
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
dependencies = [
8+
("checks", "0018_domaintesttls_caa_records"),
9+
]
10+
11+
operations = [
12+
# Note db_index is False on the ForeignKey to prevent extra indices that are not needed
13+
migrations.CreateModel(
14+
name="Fame",
15+
fields=[
16+
("id", models.IntegerField()),
17+
("domain", models.CharField(primary_key=True)),
18+
(
19+
"site_report",
20+
models.ForeignKey(
21+
null=True,
22+
on_delete=django.db.models.deletion.CASCADE,
23+
to="checks.DomainTestReport",
24+
db_index=False,
25+
),
26+
),
27+
("site_report_timestamp", models.DateTimeField(null=True)),
28+
(
29+
"mail_report",
30+
models.ForeignKey(
31+
null=True,
32+
on_delete=django.db.models.deletion.CASCADE,
33+
to="checks.MailTestReport",
34+
db_index=False,
35+
),
36+
),
37+
("mail_report_timestamp", models.DateTimeField(null=True)),
38+
],
39+
options={
40+
"abstract": False,
41+
},
42+
),
43+
# AutoField has to be primary key in Django, to solve this manually alter SQL this field
44+
# see https://github.com/django/django/blob/787f3130f751283140fe2be8188eb5299552232d/django/db/models/fields/__init__.py#L2801
45+
migrations.RunSQL(
46+
sql=[
47+
'ALTER TABLE "checks_fame" ALTER COLUMN "id" ADD GENERATED BY DEFAULT AS IDENTITY;',
48+
"""
49+
CREATE OR REPLACE FUNCTION fn_update_site_fame()
50+
RETURNS TRIGGER AS $$
51+
BEGIN
52+
IF NEW.score = 100 THEN
53+
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
54+
VALUES (NEW.domain, NEW.id, NEW.timestamp, NULL, NULL)
55+
ON CONFLICT (domain)
56+
DO UPDATE SET site_report_id = NEW.id, site_report_timestamp = NEW.timestamp;
57+
ELSE
58+
DELETE FROM checks_fame WHERE domain = NEW.domain AND mail_report_id IS NULL;
59+
UPDATE checks_fame SET site_report_id = NULL, site_report_timestamp = NULL WHERE domain = NEW.domain;
60+
END IF;
61+
RETURN NEW;
62+
END;
63+
$$ LANGUAGE plpgsql;
64+
""",
65+
"""
66+
CREATE OR REPLACE FUNCTION fn_update_mail_fame()
67+
RETURNS TRIGGER AS $$
68+
BEGIN
69+
IF NEW.score = 100 THEN
70+
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
71+
VALUES (NEW.domain, NULL, NULL, NEW.id, NEW.timestamp)
72+
ON CONFLICT (domain)
73+
DO UPDATE SET mail_report_id = NEW.id, mail_report_timestamp = NEW.timestamp;
74+
ELSE
75+
DELETE FROM checks_fame WHERE domain = NEW.domain AND site_report_id IS NULL;
76+
UPDATE checks_fame SET mail_report_id = NULL, mail_report_timestamp = NULL WHERE domain = NEW.domain;
77+
END IF;
78+
RETURN NEW;
79+
END;
80+
$$ LANGUAGE plpgsql;
81+
""",
82+
"""
83+
CREATE TRIGGER tr_update_site_fame
84+
AFTER INSERT ON checks_domaintestreport
85+
FOR EACH ROW EXECUTE PROCEDURE fn_update_site_fame();
86+
""",
87+
"""
88+
CREATE TRIGGER tr_update_mail_fame
89+
AFTER INSERT ON checks_mailtestreport
90+
FOR EACH ROW EXECUTE PROCEDURE fn_update_mail_fame();
91+
""",
92+
"""
93+
WITH
94+
site_fame AS (
95+
SELECT domain, id AS site_report_id, timestamp AS site_report_timestamp FROM (
96+
SELECT domain, score, id, timestamp, rank() OVER (PARTITION BY domain ORDER BY id DESC) FROM checks_domaintestreport
97+
) alias WHERE rank = 1 AND score = 100),
98+
mail_fame AS (
99+
SELECT domain, id AS mail_report_id, timestamp AS mail_report_timestamp FROM (
100+
SELECT domain, score, id, timestamp, rank() OVER (PARTITION BY domain ORDER BY id DESC) FROM checks_mailtestreport
101+
) alias WHERE rank = 1 AND score = 100)
102+
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
103+
SELECT * FROM site_fame FULL OUTER JOIN mail_fame USING (domain);
104+
""",
105+
],
106+
reverse_sql=[
107+
'ALTER TABLE "checks_fame" ALTER COLUMN "id" DROP IDENTITY;',
108+
'DROP TRIGGER IF EXISTS "tr_update_mail_fame" ON "checks_mailtestreport";',
109+
'DROP TRIGGER IF EXISTS "tr_update_site_fame" ON "checks_domaintestreport";',
110+
'DROP FUNCTION IF EXISTS "fn_update_mail_fame";',
111+
'DROP FUNCTION IF EXISTS "fn_update_site_fame";',
112+
'DELETE FROM "checks_fame";',
113+
],
114+
state_operations=[
115+
migrations.AlterField("Fame", "id", models.AutoField(primary_key=False)),
116+
],
117+
),
118+
]

‎checks/tasks/update.py

Lines changed: 25 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from django.core.cache import cache
66
from django.db import transaction
77

8-
from checks.models import DomainTestReport, MailTestReport
8+
from checks.models import Fame
99
from interface import redis_id
1010
from interface.batch import util
1111

@@ -19,85 +19,16 @@ def __init__(self, domain):
1919
self.web_permalink = None
2020
self.mail_timestamp = None
2121
self.mail_permalink = None
22-
self.mail_nomx = None
2322

2423
def __str__(self):
2524
return f"""------- {self.domain}
2625
web_timestamp: {self.web_timestamp}
2726
web_permalink: {self.web_permalink}
2827
mail_timestamp: {self.mail_timestamp}
2928
mail_permalink: {self.mail_permalink}
30-
mail_nomx: {self.mail_nomx}
3129
"""
3230

3331

34-
def _create_hof_entry(hof, domain_name):
35-
"""
36-
Create an entry in the Hall of Fame.
37-
38-
"""
39-
if domain_name in hof:
40-
return hof[domain_name]
41-
hof[domain_name] = HOFEntry(domain_name)
42-
return hof[domain_name]
43-
44-
45-
def _update_web_entry(hof, domain_name, report_id, timestamp):
46-
"""
47-
Update a web entry in the Hall of Fame.
48-
49-
"""
50-
entry = _create_hof_entry(hof, domain_name)
51-
entry.web_timestamp = timestamp
52-
entry.web_permalink = f"/site/{domain_name}/{report_id}/"
53-
54-
55-
def _update_mail_entry(hof, domain_name, report_id, timestamp):
56-
"""
57-
Update a mail entry in the Hall of Fame.
58-
59-
"""
60-
entry = _create_hof_entry(hof, domain_name)
61-
entry.mail_timestamp = timestamp
62-
entry.mail_permalink = f"/mail/{domain_name}/{report_id}/"
63-
report = MailTestReport.objects.get(id=report_id)
64-
ipv6_report = report.ipv6.report
65-
if not isinstance(ipv6_report, dict):
66-
return
67-
entry.mail_nomx = ipv6_report["mx_aaaa"]["verdict"] == "detail mail ipv6 mx-AAAA verdict other"
68-
69-
70-
def _populate_HOF(hof, model, entry_creation):
71-
"""
72-
Find entries that qualify for the Hall of Fame.
73-
74-
"""
75-
previousname = None
76-
previousscore = 0
77-
previoustimestamp = None
78-
previousreportid = None
79-
for report in model.objects.all().order_by("domain", "timestamp"):
80-
if previousname != report.domain and previousname is not None:
81-
if previousscore >= 100:
82-
entry_creation(hof, previousname, previousreportid, previoustimestamp)
83-
previousname = report.domain
84-
previousscore = report.score or 0
85-
previoustimestamp = report.timestamp
86-
previousreportid = report.id
87-
88-
else:
89-
report_score = report.score or 0
90-
if report_score != previousscore:
91-
previoustimestamp = report.timestamp
92-
previousname = report.domain
93-
previousreportid = report.id
94-
previousscore = report_score
95-
96-
# Last domain name.
97-
if previousscore >= 100:
98-
entry_creation(hof, previousname, previousreportid, previoustimestamp)
99-
100-
10132
@transaction.atomic
10233
def _update_hof():
10334
"""
@@ -108,28 +39,33 @@ def _update_hof():
10839
test scored 100%.
10940
11041
"""
111-
hof = dict()
112-
for model, entry_creation in ((DomainTestReport, _update_web_entry), (MailTestReport, _update_mail_entry)):
113-
_populate_HOF(hof, model, entry_creation)
114-
11542
champions = []
11643
web = []
11744
mail = []
118-
for entry in hof.values():
119-
is_web = False
120-
is_mail = False
121-
if entry.web_permalink:
122-
web.append({"permalink": entry.web_permalink, "domain": entry.domain, "timestamp": entry.web_timestamp})
123-
is_web = True
124-
if entry.mail_permalink:
125-
mail.append({"permalink": entry.mail_permalink, "domain": entry.domain, "timestamp": entry.mail_timestamp})
126-
is_mail = True
127-
if is_web and is_mail:
128-
timestamp = entry.mail_timestamp
129-
permalink = entry.mail_permalink
130-
if entry.web_timestamp > entry.mail_timestamp:
131-
timestamp = entry.web_timestamp
132-
permalink = entry.web_permalink
45+
46+
for entry in Fame.objects.all().iterator():
47+
if entry.site_report_id is not None:
48+
web.append(
49+
{
50+
"permalink": f"/site/{entry.domain}/{entry.site_report_id}/",
51+
"domain": entry.domain,
52+
"timestamp": entry.site_report_timestamp,
53+
}
54+
)
55+
if entry.mail_report_id is not None:
56+
mail.append(
57+
{
58+
"permalink": f"/mail/{entry.domain}/{entry.mail_report_id}/",
59+
"domain": entry.domain,
60+
"timestamp": entry.mail_report_timestamp,
61+
}
62+
)
63+
if entry.site_report_id is not None and entry.mail_report_id is not None:
64+
timestamp = entry.mail_report_timestamp
65+
permalink = f"/mail/{entry.domain}/{entry.mail_report_id}/"
66+
if entry.site_report_timestamp > entry.mail_report_timestamp:
67+
timestamp = entry.site_report_timestamp
68+
permalink = f"/site/{entry.domain}/{entry.site_report_id}/"
13369
champions.append({"permalink": permalink, "domain": entry.domain, "timestamp": timestamp})
13470
champions = sorted(champions, key=lambda x: x["timestamp"], reverse=True)
13571
web = sorted(web, key=lambda x: x["timestamp"], reverse=True)

0 commit comments

Comments
 (0)