Skip to content

Commit c47adae

Browse files
hsong-rhhsong-rh
authored
feat: Add a system command to purge log records (#1133)
1 parent dbab226 commit c47adae

File tree

2 files changed

+394
-0
lines changed

2 files changed

+394
-0
lines changed

‎src/aap_eda/core/management/commands/purge_log_records.py

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Copyright 2024 Red Hat, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from datetime import datetime
16+
from typing import Union
17+
18+
from django.core.management.base import (
19+
BaseCommand,
20+
CommandError,
21+
CommandParser,
22+
)
23+
from django.db import transaction
24+
from django.db.models import Q
25+
26+
from aap_eda.core import models
27+
28+
29+
class Command(BaseCommand):
30+
"""Purge the logs from a rulebook process."""
31+
32+
help = (
33+
"Purge log records from rulebook processes. "
34+
"If activation ids or names are not specified, "
35+
"all log records older than the cutoff date will be purged."
36+
)
37+
38+
def add_arguments(self, parser: CommandParser) -> None:
39+
parser.add_argument(
40+
"--activation-ids",
41+
nargs="+",
42+
type=int,
43+
dest="activation-ids",
44+
help=(
45+
"Specify the activation ids which you want to clear their "
46+
"records (e.g., ActivationID1 ActivationID2)"
47+
),
48+
)
49+
parser.add_argument(
50+
"--activation-names",
51+
nargs="+",
52+
type=str,
53+
dest="activation-names",
54+
help=(
55+
"Specify the activation names which you want to clear their "
56+
"records (e.g., ActivationName1 ActivationName2)"
57+
),
58+
)
59+
parser.add_argument(
60+
"--date",
61+
dest="date",
62+
action="store",
63+
required=True,
64+
help=(
65+
"Purge records older than this date from the database. "
66+
"The cutoff date in YYYY-MM-DD format"
67+
),
68+
)
69+
70+
def purge_log_records(
71+
self, ids: list[int], names: list[str], cutoff_timestamp: datetime
72+
) -> None:
73+
new_instance_logs = []
74+
if not bool(ids) and not bool(names):
75+
instances = models.RulebookProcess.objects.all()
76+
else:
77+
instances = models.RulebookProcess.objects.filter(
78+
Q(activation__id__in=ids) | Q(activation__name__in=names),
79+
)
80+
81+
if not instances.exists():
82+
self.stdout.write(
83+
self.style.SUCCESS("No records has been found for purging.")
84+
)
85+
return
86+
87+
for instance in instances:
88+
new_instance_logs.append(
89+
self.clean_instance_logs(instance, cutoff_timestamp)
90+
)
91+
92+
new_instance_logs = [
93+
item for item in new_instance_logs if item is not None
94+
]
95+
if len(new_instance_logs) > 0:
96+
models.RulebookProcessLog.objects.bulk_create(new_instance_logs)
97+
98+
self.stdout.write(
99+
self.style.SUCCESS(
100+
"Log records older than "
101+
f"{cutoff_timestamp.strftime('%Y-%m-%d')} are purged."
102+
)
103+
)
104+
105+
def clean_instance_logs(
106+
self,
107+
instance: models.RulebookProcess,
108+
cutoff_timestamp: datetime,
109+
) -> Union[models.RulebookProcessLog, None]:
110+
log_records = models.RulebookProcessLog.objects.filter(
111+
activation_instance=instance,
112+
log_timestamp__lt=int(cutoff_timestamp.timestamp()),
113+
)
114+
if not log_records.exists():
115+
return
116+
117+
log_records.delete()
118+
dt = f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
119+
120+
return models.RulebookProcessLog(
121+
log=(
122+
"All log records older than "
123+
f"{cutoff_timestamp.strftime('%Y-%m-%d')} are purged at {dt}."
124+
),
125+
activation_instance_id=instance.id,
126+
log_timestamp=int(cutoff_timestamp.timestamp()),
127+
)
128+
129+
@transaction.atomic
130+
def handle(self, *args, **options):
131+
input_ids = options.get("activation-ids") or []
132+
input_names = options.get("activation-names") or []
133+
cutoff_date = options.get("date")
134+
135+
try:
136+
ts = datetime.strptime(cutoff_date, "%Y-%m-%d")
137+
except ValueError as e:
138+
raise CommandError(f"{e}") from e
139+
140+
self.purge_log_records(
141+
ids=input_ids, names=input_names, cutoff_timestamp=ts
142+
)

‎tests/integration/commands/test_purge_log_records.py

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
# Copyright 2024 Red Hat, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from datetime import timedelta
16+
17+
import pytest
18+
from django.core.management import call_command
19+
from django.core.management.base import CommandError
20+
from django.utils import timezone
21+
22+
from aap_eda.core import enums, models
23+
24+
25+
@pytest.fixture
26+
def prepare_log_records(
27+
default_decision_environment: models.DecisionEnvironment,
28+
default_project: models.Project,
29+
default_rulebook: models.Rulebook,
30+
default_extra_var_data: str,
31+
default_organization: models.Organization,
32+
default_user: models.User,
33+
) -> list[models.Activation]:
34+
activation_30_days_ago = models.Activation.objects.create(
35+
name="activation-30-days-ago",
36+
description="Activation 30 days ago",
37+
decision_environment=default_decision_environment,
38+
project=default_project,
39+
rulebook=default_rulebook,
40+
extra_var=default_extra_var_data,
41+
organization=default_organization,
42+
user=default_user,
43+
log_level="debug",
44+
)
45+
46+
instances_30_days_ago = models.RulebookProcess.objects.bulk_create(
47+
[
48+
models.RulebookProcess(
49+
name="activation-30-days-ago-instance-1",
50+
activation=activation_30_days_ago,
51+
git_hash=default_project.git_hash,
52+
status=enums.ActivationStatus.STOPPED,
53+
status_message=enums.ACTIVATION_STATUS_MESSAGE_MAP[
54+
enums.ActivationStatus.STOPPED
55+
],
56+
organization=default_organization,
57+
),
58+
models.RulebookProcess(
59+
name="activation-30-days-ago-instance-2",
60+
activation=activation_30_days_ago,
61+
git_hash=default_project.git_hash,
62+
status=enums.ActivationStatus.FAILED,
63+
status_message=enums.ACTIVATION_STATUS_MESSAGE_MAP[
64+
enums.ActivationStatus.FAILED
65+
],
66+
organization=default_organization,
67+
),
68+
]
69+
)
70+
71+
activation_10_days_ago = models.Activation.objects.create(
72+
name="activation-10-days-ago",
73+
description="Activation 10 days ago",
74+
decision_environment=default_decision_environment,
75+
project=default_project,
76+
rulebook=default_rulebook,
77+
extra_var=default_extra_var_data,
78+
organization=default_organization,
79+
user=default_user,
80+
log_level="debug",
81+
)
82+
83+
instances_10_days_ago = models.RulebookProcess.objects.bulk_create(
84+
[
85+
models.RulebookProcess(
86+
name="activation-10-days-ago-instance-1",
87+
activation=activation_10_days_ago,
88+
git_hash=default_project.git_hash,
89+
status=enums.ActivationStatus.COMPLETED,
90+
status_message=enums.ACTIVATION_STATUS_MESSAGE_MAP[
91+
enums.ActivationStatus.COMPLETED
92+
],
93+
organization=default_organization,
94+
),
95+
models.RulebookProcess(
96+
name="activation-10-days-ago-instance-2",
97+
activation=activation_10_days_ago,
98+
git_hash=default_project.git_hash,
99+
status=enums.ActivationStatus.RUNNING,
100+
status_message=enums.ACTIVATION_STATUS_MESSAGE_MAP[
101+
enums.ActivationStatus.RUNNING
102+
],
103+
organization=default_organization,
104+
),
105+
]
106+
)
107+
108+
log_timestamp_10_days_ago = timezone.now() - timedelta(days=10)
109+
log_timestamp_30_days_ago = timezone.now() - timedelta(days=30)
110+
111+
models.RulebookProcessLog.objects.bulk_create(
112+
[
113+
models.RulebookProcessLog(
114+
log="activation-instance-30-days-ago-log-1",
115+
activation_instance=instances_30_days_ago[0],
116+
log_timestamp=int(log_timestamp_30_days_ago.timestamp()),
117+
),
118+
models.RulebookProcessLog(
119+
log="activation-instance-30-days-ago-log-2",
120+
activation_instance=instances_30_days_ago[0],
121+
log_timestamp=int(log_timestamp_30_days_ago.timestamp()),
122+
),
123+
models.RulebookProcessLog(
124+
log="activation-instance-30-days-ago-log-3",
125+
activation_instance=instances_30_days_ago[1],
126+
log_timestamp=int(log_timestamp_30_days_ago.timestamp()),
127+
),
128+
models.RulebookProcessLog(
129+
log="activation-instance-30-days-ago-log-4",
130+
activation_instance=instances_30_days_ago[1],
131+
log_timestamp=int(log_timestamp_30_days_ago.timestamp()),
132+
),
133+
models.RulebookProcessLog(
134+
log="activation-instance-10-days-ago-log-1",
135+
activation_instance=instances_10_days_ago[0],
136+
log_timestamp=int(log_timestamp_10_days_ago.timestamp()),
137+
),
138+
models.RulebookProcessLog(
139+
log="activation-instance-10-days-ago-log-2",
140+
activation_instance=instances_10_days_ago[0],
141+
log_timestamp=int(log_timestamp_10_days_ago.timestamp()),
142+
),
143+
models.RulebookProcessLog(
144+
log="activation-instance-10-days-ago-log-3",
145+
activation_instance=instances_10_days_ago[1],
146+
log_timestamp=int(log_timestamp_10_days_ago.timestamp()),
147+
),
148+
models.RulebookProcessLog(
149+
log="activation-instance-10-days-ago-log-4",
150+
activation_instance=instances_10_days_ago[1],
151+
log_timestamp=int(log_timestamp_10_days_ago.timestamp()),
152+
),
153+
]
154+
)
155+
156+
return [activation_30_days_ago, activation_10_days_ago]
157+
158+
159+
@pytest.mark.django_db
160+
def test_purge_log_records_missing_required_params():
161+
with pytest.raises(
162+
CommandError,
163+
match="Error: the following arguments are required: --date",
164+
):
165+
call_command("purge_log_records")
166+
167+
168+
@pytest.mark.django_db
169+
def test_purge_log_records_with_nonexist_activation(capsys):
170+
args = ("--activation-ids", "42", "--date", "2024-10-01")
171+
call_command("purge_log_records", args)
172+
captured = capsys.readouterr()
173+
174+
assert captured.out == "No records has been found for purging.\n"
175+
176+
args = ("--activation-name", "na", "--date", "2024-10-01")
177+
call_command("purge_log_records", args)
178+
captured = capsys.readouterr()
179+
assert captured.out == "No records has been found for purging.\n"
180+
181+
182+
@pytest.mark.parametrize(
183+
"identifiers, cutoff_days",
184+
[("ids", 15), ("names", 5), ("none", 15), ("none", 5)],
185+
)
186+
@pytest.mark.django_db
187+
def test_purge_log_records(
188+
prepare_log_records, capsys, identifiers, cutoff_days
189+
):
190+
activations = prepare_log_records
191+
192+
assert models.RulebookProcessLog.objects.count() == 8
193+
194+
command = "purge_log_records"
195+
196+
ts = timezone.now() - timedelta(days=cutoff_days)
197+
date_str = ts.strftime("%Y-%m-%d")
198+
199+
if identifiers == "ids":
200+
args = (
201+
"--activation-ids",
202+
activations[0].id,
203+
activations[1].id,
204+
"--date",
205+
date_str,
206+
)
207+
elif identifiers == "names":
208+
args = (
209+
"--activation-names",
210+
activations[0].name,
211+
activations[1].name,
212+
"--date",
213+
date_str,
214+
)
215+
else:
216+
args = (
217+
"--date",
218+
date_str,
219+
)
220+
221+
call_command(command, args)
222+
223+
captured = capsys.readouterr()
224+
225+
if cutoff_days < 10:
226+
assert models.RulebookProcessLog.objects.count() == 4
227+
228+
for log_record in models.RulebookProcessLog.objects.all():
229+
assert (
230+
f"All log records older than {date_str} are purged at"
231+
in log_record.log
232+
)
233+
else:
234+
assert models.RulebookProcessLog.objects.count() == 6
235+
236+
for log_record in models.RulebookProcessLog.objects.filter(
237+
activation_instance__activation=activations[0]
238+
):
239+
assert (
240+
f"All log records older than {date_str} are purged at"
241+
in log_record.log
242+
)
243+
244+
for log_record in models.RulebookProcessLog.objects.filter(
245+
activation_instance__activation=activations[1]
246+
):
247+
assert (
248+
f"All log records older than {date_str} are purged at"
249+
not in log_record.log
250+
)
251+
252+
assert f"Log records older than {date_str} are purged." in captured.out

0 commit comments

Comments
 (0)