diff --git a/MPCAutofill/MPCAutofill/settings.py b/MPCAutofill/MPCAutofill/settings.py index a8f232c9b..2f60733a0 100755 --- a/MPCAutofill/MPCAutofill/settings.py +++ b/MPCAutofill/MPCAutofill/settings.py @@ -217,7 +217,7 @@ PATREON_URL = env("PATREON_URL", default="") # Sentry -if len(sys.argv) >= 2 and sys.argv[1] != "runserver": +if len(sys.argv) >= 2 and sys.argv[1] != "runserver" and env("DJANGO_DEBUG", default=False) is False: sentry_sdk.init( dsn="https://4d29db1957fb9b3153aaba66e776b01f@o4505848489246720.ingest.sentry.io/4505848491540480", integrations=[DjangoIntegration()], diff --git a/MPCAutofill/cardpicker/admin.py b/MPCAutofill/cardpicker/admin.py index 5f3a97a8b..23dd36ffb 100644 --- a/MPCAutofill/cardpicker/admin.py +++ b/MPCAutofill/cardpicker/admin.py @@ -12,7 +12,7 @@ class AdminTag(admin.ModelAdmin[Tag]): @admin.register(Card) class AdminCard(admin.ModelAdmin[Card]): - list_display = ("identifier", "name", "source", "dpi", "date", "tags") + list_display = ("identifier", "name", "source", "dpi", "date_created", "tags") search_fields = ("identifier", "name") diff --git a/MPCAutofill/cardpicker/documents.py b/MPCAutofill/cardpicker/documents.py index c2984e924..48aba693a 100644 --- a/MPCAutofill/cardpicker/documents.py +++ b/MPCAutofill/cardpicker/documents.py @@ -18,7 +18,8 @@ class CardSearch(Document): searchq_precise = fields.TextField(attr="searchq", analyzer=precise_analyser) searchq_keyword = fields.KeywordField(attr="searchq") card_type = fields.KeywordField() - date = fields.DateField() + date_created = fields.DateField() + date_modified = fields.DateField() language = fields.TextField(analyzer=precise_analyser) # case insensitivity is one less thing which can go wrong tags = fields.KeywordField() # all elasticsearch fields support arrays by default diff --git a/MPCAutofill/cardpicker/management/commands/update_database.py b/MPCAutofill/cardpicker/management/commands/update_database.py index 8a50c7bb0..4040b3781 100755 --- a/MPCAutofill/cardpicker/management/commands/update_database.py +++ b/MPCAutofill/cardpicker/management/commands/update_database.py @@ -1,7 +1,6 @@ import time from typing import Any, Optional -from django.core import management from django.core.management.base import BaseCommand from cardpicker.models import Source @@ -23,6 +22,4 @@ def handle(self, *args: Any, **kwargs: str) -> None: drive: Optional[str] = kwargs.get("drive", None) t0 = time.time() update_database(source_key=drive) - management.call_command("search_index", "--rebuild", "-f") - print("") log_hours_minutes_seconds_elapsed(t0) diff --git a/MPCAutofill/cardpicker/migrations/0040_rename_date_card_date_created.py b/MPCAutofill/cardpicker/migrations/0040_rename_date_card_date_created.py new file mode 100644 index 000000000..7b432d6fd --- /dev/null +++ b/MPCAutofill/cardpicker/migrations/0040_rename_date_card_date_created.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.14 on 2025-05-16 09:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("cardpicker", "0039_remove_card_searchq_keyword"), + ] + + operations = [ + migrations.RenameField( + model_name="card", + old_name="date", + new_name="date_created", + ), + ] diff --git a/MPCAutofill/cardpicker/migrations/0041_card_date_modified.py b/MPCAutofill/cardpicker/migrations/0041_card_date_modified.py new file mode 100644 index 000000000..66090d043 --- /dev/null +++ b/MPCAutofill/cardpicker/migrations/0041_card_date_modified.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.14 on 2025-05-16 21:15 + +import datetime + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("cardpicker", "0040_rename_date_card_date_created"), + ] + + operations = [ + migrations.AddField( + model_name="card", + name="date_modified", + field=models.DateTimeField(default=datetime.datetime.now), + ), + migrations.RunSQL( + sql="UPDATE cardpicker_card SET date_modified = date_created", reverse_sql=migrations.RunSQL.noop + ), + ] diff --git a/MPCAutofill/cardpicker/migrations/0042_remove_projectmember_projectmember_unique_and_more.py b/MPCAutofill/cardpicker/migrations/0042_remove_projectmember_projectmember_unique_and_more.py new file mode 100644 index 000000000..7a4965259 --- /dev/null +++ b/MPCAutofill/cardpicker/migrations/0042_remove_projectmember_projectmember_unique_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.14 on 2025-05-18 02:22 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("cardpicker", "0041_card_date_modified"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="projectmember", + name="projectmember_unique", + ), + migrations.RemoveField( + model_name="projectmember", + name="card_id", + ), + migrations.AddField( + model_name="projectmember", + name="card", + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to="cardpicker.card" + ), + ), + migrations.AddConstraint( + model_name="projectmember", + constraint=models.UniqueConstraint(fields=("card", "project", "slot", "face"), name="projectmember_unique"), + ), + ] diff --git a/MPCAutofill/cardpicker/models.py b/MPCAutofill/cardpicker/models.py index 5e5414078..dfc216b6f 100755 --- a/MPCAutofill/cardpicker/models.py +++ b/MPCAutofill/cardpicker/models.py @@ -193,7 +193,8 @@ class Card(models.Model): dpi = models.IntegerField(default=0) searchq = models.CharField(max_length=200) extension = models.CharField(max_length=200) - date = models.DateTimeField(default=datetime.now) + date_created = models.DateTimeField(default=datetime.now) + date_modified = models.DateTimeField(default=datetime.now) size = models.IntegerField() tags = ArrayField(models.CharField(max_length=20), default=list, blank=True) # null=True is just for admin panel language = models.CharField(max_length=5) @@ -205,7 +206,7 @@ def __str__(self) -> str: f"{self.name} " f"[Type: {self.card_type}, " f"Identifier: {self.identifier}, " - f"Uploaded: {self.date.strftime('%d/%m/%Y')}, " + f"Uploaded: {self.date_created.strftime('%d/%m/%Y')}, " f"Priority: {self.priority}]" ) @@ -225,7 +226,8 @@ def serialise(self) -> SerialisedCard: dpi=self.dpi, searchq=self.searchq, extension=self.extension, - date=dateformat.format(self.date, DATE_FORMAT), + dateCreated=dateformat.format(self.date_created, DATE_FORMAT), + dateModified=dateformat.format(self.date_modified, DATE_FORMAT), size=self.size, downloadLink=self.get_download_link() or "", smallThumbnailUrl=self.get_small_thumbnail_url() or "", @@ -361,8 +363,18 @@ def set_project_members(self, records: dict[str, dict[str, list[dict[str, Any]]] if (card_identifier := record.get("card_identifier"), None) is not None: card_identifiers.add(card_identifier) + card_identifiers_to_pk: dict[str, Card] = { + x.identifier: x for x in Card.objects.filter(identifier__in=card_identifiers) + } members: list[ProjectMember] = [ - ProjectMember(card_id=value.get("card_identifier", None), slot=value["slot"], query=query, face=face) + ProjectMember( + card=card_identifiers_to_pk[card_identifier] + if (card_identifier := value.get("card_identifier", None)) is not None + else None, + slot=value["slot"], + query=query, + face=face, + ) for face in Faces if (face_members := records.get(face, None)) is not None for query, values in face_members.items() @@ -388,19 +400,22 @@ def __str__(self) -> str: class ProjectMember(models.Model): - card_id = models.CharField(max_length=200, null=True, blank=True) + card = models.ForeignKey(to=Card, on_delete=models.SET_NULL, null=True, blank=True) project = models.ForeignKey(to=Project, on_delete=models.CASCADE) query = models.CharField(max_length=200) slot = models.IntegerField() face = models.CharField(max_length=5, choices=Faces.choices, default=Faces.FRONT) class Meta: - constraints = [ - models.UniqueConstraint(fields=["card_id", "project", "slot", "face"], name="projectmember_unique") - ] + constraints = [models.UniqueConstraint(fields=["card", "project", "slot", "face"], name="projectmember_unique")] def to_dict(self) -> dict[str, Any]: - return {"card_identifier": self.card_id, "query": self.query, "slot": self.slot, "face": self.face} + return { + "card_identifier": self.card.identifier if self.card else None, + "query": self.query, + "slot": self.slot, + "face": self.face, + } __all__ = [ diff --git a/MPCAutofill/cardpicker/schema_types.py b/MPCAutofill/cardpicker/schema_types.py index e4f7fc76d..cd112d08c 100644 --- a/MPCAutofill/cardpicker/schema_types.py +++ b/MPCAutofill/cardpicker/schema_types.py @@ -232,9 +232,12 @@ class SourceType(str, Enum): class Card(BaseModel): cardType: CardType - date: str + dateCreated: str """Created date - formatted by backend""" + dateModified: str + """Modified date - formatted by backend""" + downloadLink: str dpi: int extension: str @@ -258,7 +261,8 @@ class Card(BaseModel): def from_dict(obj: Any) -> "Card": assert isinstance(obj, dict) cardType = CardType(obj.get("cardType")) - date = from_str(obj.get("date")) + dateCreated = from_str(obj.get("dateCreated")) + dateModified = from_str(obj.get("dateModified")) downloadLink = from_str(obj.get("downloadLink")) dpi = from_int(obj.get("dpi")) extension = from_str(obj.get("extension")) @@ -279,7 +283,8 @@ def from_dict(obj: Any) -> "Card": sourceType = from_union([SourceType, from_none], obj.get("sourceType")) return Card( cardType, - date, + dateCreated, + dateModified, downloadLink, dpi, extension, @@ -303,7 +308,8 @@ def from_dict(obj: Any) -> "Card": def to_dict(self) -> dict: result: dict = {} result["cardType"] = to_enum(CardType, self.cardType) - result["date"] = from_str(self.date) + result["dateCreated"] = from_str(self.dateCreated) + result["dateModified"] = from_str(self.dateModified) result["downloadLink"] = from_str(self.downloadLink) result["dpi"] = from_int(self.dpi) result["extension"] = from_str(self.extension) diff --git a/MPCAutofill/cardpicker/search/search_functions.py b/MPCAutofill/cardpicker/search/search_functions.py index 038673330..3e6ad3bdc 100644 --- a/MPCAutofill/cardpicker/search/search_functions.py +++ b/MPCAutofill/cardpicker/search/search_functions.py @@ -204,8 +204,8 @@ def retrieve_cardback_identifiers(search_settings: SearchSettings) -> list[str]: def get_new_cards_paginator(source: Source) -> Paginator[QuerySet[Card]]: now = timezone.now() cards = Card.objects.filter( - source=source, date__lt=now, date__gte=now - dt.timedelta(days=NEW_CARDS_DAYS) - ).order_by("-date", "name") + source=source, date_created__lt=now, date_created__gte=now - dt.timedelta(days=NEW_CARDS_DAYS) + ).order_by("-date_created", "name") return Paginator(cards, NEW_CARDS_PAGE_SIZE) # type: ignore # TODO: `_SupportsPagination` diff --git a/MPCAutofill/cardpicker/sources/api.py b/MPCAutofill/cardpicker/sources/api.py index 0aacf742e..8dff9b538 100644 --- a/MPCAutofill/cardpicker/sources/api.py +++ b/MPCAutofill/cardpicker/sources/api.py @@ -70,6 +70,7 @@ class Image: name: str size: int created_time: dt.datetime + modified_time: dt.datetime height: int folder: Folder diff --git a/MPCAutofill/cardpicker/sources/source_types.py b/MPCAutofill/cardpicker/sources/source_types.py index cb86df2bd..44a0e31f8 100644 --- a/MPCAutofill/cardpicker/sources/source_types.py +++ b/MPCAutofill/cardpicker/sources/source_types.py @@ -5,6 +5,8 @@ from tqdm import tqdm from django.db.models import TextChoices +from django.utils.dateparse import parse_datetime +from django.utils.timezone import now from django.utils.translation import gettext_lazy from cardpicker.schema_types import SourceType as SchemaSourceType @@ -151,7 +153,7 @@ def get_all_images_inside_folder(folder: Folder) -> list[Image]: "mimeType contains 'image/jpeg') and " f"'{folder.id}' in parents", fields="nextPageToken, files(" - "id, name, trashed, size, parents, createdTime, imageMediaMetadata" + "id, name, trashed, size, parents, createdTime, modifiedTime, imageMediaMetadata" ")", pageSize=500, pageToken=page_token, @@ -167,7 +169,8 @@ def get_all_images_inside_folder(folder: Folder) -> list[Image]: Image( id=item["id"], name=item["name"], - created_time=item["createdTime"], + created_time=parse_datetime(item["createdTime"]) or now(), + modified_time=parse_datetime(item["modifiedTime"]) or now(), folder=folder, height=item["imageMediaMetadata"]["height"], size=int(item["size"]), diff --git a/MPCAutofill/cardpicker/sources/update_database.py b/MPCAutofill/cardpicker/sources/update_database.py index e9967456b..18056da5b 100644 --- a/MPCAutofill/cardpicker/sources/update_database.py +++ b/MPCAutofill/cardpicker/sources/update_database.py @@ -9,6 +9,7 @@ from django.db import transaction from cardpicker.constants import DEFAULT_LANGUAGE, MAX_SIZE_MB +from cardpicker.documents import CardSearch from cardpicker.models import Card, CardTypes, Source from cardpicker.search.sanitisation import to_searchable from cardpicker.sources.api import Folder, Image @@ -112,7 +113,8 @@ def transform_images_into_objects(source: Source, images: list[Image], tags: Tag dpi=dpi, searchq=searchable_name, # search-friendly card name extension=extension, - date=image.created_time, + date_created=image.created_time, + date_modified=image.modified_time, size=image.size, tags=list(extracted_tags), language=(language or DEFAULT_LANGUAGE).alpha_2.upper(), @@ -143,10 +145,59 @@ def transform_images_into_objects(source: Source, images: list[Image], tags: Tag def bulk_sync_objects(source: Source, cards: list[Card]) -> None: print(f"Synchronising objects to database for source {TEXT_BOLD}{source.name}{TEXT_END}...", end="", flush=True) t0 = time.time() - with transaction.atomic(): # django-bulk-sync is crushingly slow with postgres - Card.objects.filter(source=source).delete() - Card.objects.bulk_create(cards) - print(f" and done! That took {TEXT_BOLD}{(time.time() - t0):.2f}{TEXT_END} seconds.") + + incoming = {card.identifier: card for card in cards} + incoming_ids = set(incoming.keys()) + existing = {card.identifier: card for card in Card.objects.filter(source=source)} + existing_ids = set(existing.keys()) + + created = [incoming[identifier] for identifier in incoming_ids - existing_ids] + updated: list[Card] = [] + for identifier in incoming_ids & existing_ids: + if incoming[identifier].date_modified > existing[identifier].date_modified: + incoming[identifier].pk = existing[identifier].pk # this must be explicitly set for bulk_update. + updated.append(incoming[identifier]) + deleted_ids = existing_ids - incoming_ids + deleted = [existing[identifier] for identifier in deleted_ids] + + with transaction.atomic(): + if created: + Card.objects.bulk_create(created) + CardSearch().update(list(created), action="index") + if updated: + Card.objects.bulk_update( + updated, + # update every field except for `identifier` + [ + "card_type", + "name", + "priority", + "source", + "source_verbose", + "folder_location", + "dpi", + "searchq", + "extension", + "date_created", + "date_modified", + "size", + "tags", + "language", + ], + batch_size=1000, + ) + # as per this thread https://github.com/django-es/django-elasticsearch-dsl/issues/224#issuecomment-551955511 + # action type "index" is used for indexing new objects as well as updating existing objects + CardSearch().update(list(updated), action="index") + if deleted_ids: + Card.objects.filter(identifier__in=deleted_ids).delete() + CardSearch().update(list(deleted), action="delete") + print( + f" and done! That took {TEXT_BOLD}{(time.time() - t0):.2f}{TEXT_END} seconds.\n" + f"Created {TEXT_BOLD}{len(created)}{TEXT_END}, " + f"updated {TEXT_BOLD}{len(updated)}{TEXT_END}, " + f"and deleted {TEXT_BOLD}{len(deleted_ids)}{TEXT_END} cards." + ) def update_database_for_source(source: Source, source_type: Type[SourceType], root_folder: Folder, tags: Tags) -> None: diff --git a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr index b498e2e96..88f937dab 100644 --- a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr +++ b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr @@ -192,7 +192,8 @@ 'CARD': dict({ 'Brainstorm': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -215,7 +216,8 @@ }), 'Island': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -238,7 +240,8 @@ }), 'Island (William Bradford)': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -261,7 +264,8 @@ }), 'Pást in Flames': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -289,7 +293,8 @@ 'TOKEN': dict({ 'Goblin': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -394,7 +399,8 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -417,7 +423,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -440,7 +447,8 @@ }), dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -463,7 +471,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -486,7 +495,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -509,7 +519,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -546,7 +557,8 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -571,7 +583,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -653,7 +666,8 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -676,7 +690,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -699,7 +714,8 @@ }), dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -722,7 +738,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -745,7 +762,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -768,7 +786,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -800,7 +819,8 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -823,7 +843,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -846,7 +867,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -870,7 +892,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -893,7 +916,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -927,7 +951,8 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -952,7 +977,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -1326,7 +1352,8 @@ 'results': dict({ '17fopRCNRge72U8Hac8pApHZtEalx5kHy': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -1349,7 +1376,8 @@ }), '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1391,7 +1419,8 @@ 'results': dict({ '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1423,7 +1452,8 @@ 'results': dict({ '17fopRCNRge72U8Hac8pApHZtEalx5kHy': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -1446,7 +1476,8 @@ }), '1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -1469,7 +1500,8 @@ }), '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1511,7 +1543,8 @@ 'results': dict({ '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2175,7 +2208,8 @@ 'cards': list([ dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2198,7 +2232,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2223,7 +2258,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2257,7 +2293,8 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2280,7 +2317,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -2303,7 +2341,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -2326,7 +2365,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -2349,7 +2389,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -2372,7 +2413,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -2395,7 +2437,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -2418,7 +2461,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -2443,7 +2487,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -2467,7 +2512,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -2500,7 +2546,8 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2523,7 +2570,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -2546,7 +2594,8 @@ }), dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2569,7 +2618,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -2592,7 +2642,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -2615,7 +2666,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -2638,7 +2690,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -2661,7 +2714,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -2684,7 +2738,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -2709,7 +2764,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -2733,7 +2789,8 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -2756,7 +2813,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2781,7 +2839,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2815,7 +2874,8 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2848,7 +2908,8 @@ 'cards': list([ dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2882,7 +2943,8 @@ 'cards': list([ dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2915,7 +2977,8 @@ 'cards': list([ dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2940,7 +3003,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2974,7 +3038,8 @@ 'cards': list([ dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2998,7 +3063,8 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', diff --git a/MPCAutofill/cardpicker/tests/conftest.py b/MPCAutofill/cardpicker/tests/conftest.py index 3a739eeda..171dfc446 100644 --- a/MPCAutofill/cardpicker/tests/conftest.py +++ b/MPCAutofill/cardpicker/tests/conftest.py @@ -132,7 +132,7 @@ def brainstorm(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.BRAINSTORM.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -147,7 +147,7 @@ def island(example_drive_1) -> Card: source=example_drive_1, priority=7, size=Cards.ISLAND.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -162,7 +162,7 @@ def island_classical(example_drive_1) -> Card: source=example_drive_1, priority=6, size=Cards.ISLAND_CLASSICAL.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), language="FR", ) @@ -178,7 +178,7 @@ def mountain(example_drive_1) -> Card: source=example_drive_1, priority=7, size=Cards.MOUNTAIN.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -193,7 +193,7 @@ def simple_cube(example_drive_1, tag_in_data, another_tag_in_data) -> Card: source=example_drive_1, priority=17, size=Cards.SIMPLE_CUBE.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), tags=[tag_in_data.name, another_tag_in_data.name], language="DE", ) @@ -210,7 +210,7 @@ def simple_lotus(example_drive_2, tag_in_data) -> Card: source=example_drive_2, priority=7, size=Cards.SIMPLE_LOTUS.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), tags=[tag_in_data.name], language="EN", ) @@ -227,7 +227,7 @@ def huntmaster_of_the_fells(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.HUNTMASTER_OF_THE_FELLS.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -242,7 +242,7 @@ def ravager_of_the_fells(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.RAVAGER_OF_THE_FELLS.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -257,7 +257,7 @@ def past_in_flames_1(example_drive_1, tag_in_data) -> Card: source=example_drive_1, priority=2, size=Cards.PAST_IN_FLAMES_1.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), tags=[tag_in_data.name], language="EN", ) @@ -274,7 +274,7 @@ def past_in_flames_2(example_drive_2, tag_in_data, another_tag_in_data) -> Card: source=example_drive_2, priority=2, size=Cards.PAST_IN_FLAMES_2.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), tags=[tag_in_data.name, another_tag_in_data.name], language="DE", ) @@ -291,7 +291,7 @@ def delver_of_secrets(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.DELVER_OF_SECRETS.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -306,7 +306,7 @@ def insectile_aberration(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.INSECTILE_ABERRATION.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -321,7 +321,7 @@ def goblin(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.GOBLIN.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) diff --git a/MPCAutofill/cardpicker/tests/factories.py b/MPCAutofill/cardpicker/tests/factories.py index ed1eb875d..acd83ccf2 100644 --- a/MPCAutofill/cardpicker/tests/factories.py +++ b/MPCAutofill/cardpicker/tests/factories.py @@ -19,6 +19,7 @@ class SourceFactory(factory.django.DjangoModelFactory): class Meta: model = models.Source + identifier = factory.Sequence(lambda n: f"source_{n}") key = factory.Sequence(lambda n: f"source_{n}") name = factory.Sequence(lambda n: f"Source {n}") source_type = models.SourceTypeChoices.GOOGLE_DRIVE @@ -32,7 +33,8 @@ class Meta: model = models.Card card_type = models.CardTypes.CARD - date = factory.LazyFunction(lambda: dt.datetime(2023, 1, 1)) # for snapshot consistency + date_created = factory.LazyFunction(lambda: dt.datetime(2023, 1, 1)) # for snapshot consistency + date_modified = factory.LazyAttribute(lambda o: o.date_created) identifier = factory.Sequence(lambda n: f"card_{n}") name = factory.Sequence(lambda n: f"Card {n}") priority = factory.Sequence(lambda n: n) diff --git a/MPCAutofill/cardpicker/tests/test_sources.py b/MPCAutofill/cardpicker/tests/test_sources.py index 1f8738597..eb228523d 100644 --- a/MPCAutofill/cardpicker/tests/test_sources.py +++ b/MPCAutofill/cardpicker/tests/test_sources.py @@ -1,18 +1,24 @@ import datetime as dt +import freezegun import pytest +from django.core import management +from django.utils.timezone import make_aware, make_naive + +from cardpicker.documents import CardSearch from cardpicker.models import Card from cardpicker.sources.api import Folder, Image -from cardpicker.sources.update_database import update_database +from cardpicker.sources.update_database import bulk_sync_objects, update_database from cardpicker.tags import Tags +from cardpicker.tests import factories + +DEFAULT_DATE = dt.datetime(2023, 1, 1) class TestAPI: # region constants - DEFAULT_DATE = dt.datetime(2023, 1, 1) - FOLDER_A = Folder(id="a", name="Folder A", parent=None) FOLDER_B = Folder(id="b", name="Folder B", parent=FOLDER_A) FOLDER_C = Folder(id="c", name="Folder C [NSFW]", parent=FOLDER_B) @@ -26,47 +32,156 @@ class TestAPI: FOLDER_Z = Folder(id="z", name="Folder z [Full Art", parent=None) FOLDER_FRENCH = Folder(id="french", name="{FR} Folder", parent=None) - IMAGE_A = Image(id="a", name="Image A.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A) - IMAGE_B = Image(id="b", name="Image B [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A) - IMAGE_C = Image(id="b", name="Image C.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_C) + IMAGE_A = Image( + id="a", + name="Image A.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, + ) + IMAGE_B = Image( + id="b", + name="Image B [NSFW].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, + ) + IMAGE_C = Image( + id="b", + name="Image C.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_C, + ) IMAGE_D = Image( - id="b", name="Image D [NSFW, full art].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_C + id="b", + name="Image D [NSFW, full art].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_C, ) IMAGE_E = Image( - id="e", name="Image E [invalid tag.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="e", + name="Image E [invalid tag.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_F = Image( - id="F", name="Image F [NSFW, tag in data].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="F", + name="Image F [NSFW, tag in data].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_G = Image( - id="G", name="Image G [NSFW] (John Doe).png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="G", + name="Image G [NSFW] (John Doe).png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_H = Image( - id="H", name="Image H [A, NSFW, B] (John Doe).png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="H", + name="Image H [A, NSFW, B] (John Doe).png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, + ) + IMAGE_I = Image( + id="I", + name="Image A.I.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) - IMAGE_I = Image(id="I", name="Image A.I.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A) IMAGE_J = Image( - id="J", name="Image J [Child Tag].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="J", + name="Image J [Child Tag].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_K = Image( - id="K", name="Image K [Grandchild Tag].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="K", + name="Image K [Grandchild Tag].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, + ) + IMAGE_L = Image( + id="L", + name="Image L [NSFW].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_D, ) - IMAGE_L = Image(id="L", name="Image L [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_D) IMAGE_FRENCH = Image( - id="french", name="French.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH + id="french", + name="French.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_FRENCH, ) IMAGE_ENGLISH = Image( - id="english", name="{EN} English.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH + id="english", + name="{EN} English.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_FRENCH, + ) + IMAGE_NSFW = Image( + id="nsfw", + name="NSFW [NSFW].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) - IMAGE_NSFW = Image(id="nsfw", name="NSFW [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A) IMAGE_DOUBLE_NSFW = Image( - id="double nsfw", name="NSFW (NSFW) [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="double nsfw", + name="NSFW (NSFW) [NSFW].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_IMPLICITLY_FRENCH = Image( id="implicitly_french", name="Implicitly French.png", size=1, created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH, ) @@ -75,6 +190,7 @@ class TestAPI: name="{EN} Explicitly English.png", size=1, created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH, ) @@ -237,7 +353,6 @@ def test_comprehensive_snapshot(self, snapshot, django_settings, elasticsearch, update_database() assert list(Card.objects.all().order_by("identifier")) == snapshot(name="cards") - @pytest.mark.skip("we turned off upsert at time of writing because it's extremely slow with postgres") def test_upsert(self, django_settings, elasticsearch, all_sources): update_database() pk_to_identifier_1 = {x.pk: x.identifier for x in Card.objects.all()} @@ -245,4 +360,87 @@ def test_upsert(self, django_settings, elasticsearch, all_sources): pk_to_identifier_2 = {x.pk: x.identifier for x in Card.objects.all()} assert pk_to_identifier_1 == pk_to_identifier_2 + @pytest.mark.parametrize( + "existing_cards, incoming_cards", + [ + pytest.param( + [], + [], + id="no changes to empty database", + ), + pytest.param( + [("existing", "Existing Card", DEFAULT_DATE)], + [("existing", "Existing Card", DEFAULT_DATE)], + id="no changes to populated database", + ), + pytest.param( + [], + [("created", "Created Card", DEFAULT_DATE)], + id="create one card", + ), + pytest.param( + [("updated", "Card to Update", DEFAULT_DATE)], + [("updated", "Updated Card", DEFAULT_DATE + dt.timedelta(days=1))], + id="update one card", + ), + pytest.param( + [("deleted", "Card to Delete", DEFAULT_DATE)], + [], + id="delete one card", + ), + pytest.param( + [("updated", "Card to Update", DEFAULT_DATE), ("deleted", "Card to Delete", DEFAULT_DATE)], + [ + ("created", "Created Card", DEFAULT_DATE), + ("updated", "Updated Card", DEFAULT_DATE + dt.timedelta(days=1)), + ], + id="create + update + delete", + ), + pytest.param( + [("existing", "Existing Card", DEFAULT_DATE)], + [("existing", "Existing Card", DEFAULT_DATE), ("created", "Created Card", DEFAULT_DATE)], + id="create one card while another card exists and is not modified", + ), + ], + ) + @freezegun.freeze_time(DEFAULT_DATE) + def test_bulk_sync_objects(self, django_settings, elasticsearch, example_drive_1, existing_cards, incoming_cards): + # arrange - set up database and elasticsearch according to `existing_cards` + source = factories.SourceFactory() + for (identifier, searchq, date_modified) in existing_cards: + factories.CardFactory( + identifier=identifier, + searchq=searchq, + date_created=make_aware(DEFAULT_DATE), + date_modified=make_aware(date_modified), + source=source, + ) + management.call_command("search_index", "--rebuild", "-f") + + # act + bulk_sync_objects( + source=source, + cards=[ + Card( + identifier=identifier, + searchq=searchq, + date_created=make_aware(DEFAULT_DATE), + date_modified=make_aware(date_modified), + source=source, + # not strictly relevant for this test, but values for these non-nullable fields are required. + size=0, + ) + for (identifier, searchq, date_modified) in incoming_cards + ], + ) + + # assert - database and elasticsearch should now match `incoming_cards` + assert {(card.identifier, card.searchq, make_naive(card.date_modified)) for card in Card.objects.all()} == set( + incoming_cards + ) + assert { + (result.identifier, result.searchq_keyword, make_naive(result.date_modified)) + for result in CardSearch().search().scan() + } == set(incoming_cards) + # endregion diff --git a/MPCAutofill/cardpicker/views.py b/MPCAutofill/cardpicker/views.py index 03f6ffd71..ede8850b7 100644 --- a/MPCAutofill/cardpicker/views.py +++ b/MPCAutofill/cardpicker/views.py @@ -164,8 +164,8 @@ def post_explore_search(request: HttpRequest) -> HttpResponse: sort: dict[str, dict[str, str]] = { SortBy.nameAscending: {"searchq_keyword": {"order": "asc"}}, SortBy.nameDescending: {"searchq_keyword": {"order": "desc"}}, - SortBy.dateAscending: {"date": {"order": "asc"}, "searchq_keyword": {"order": "asc"}}, - SortBy.dateDescending: {"date": {"order": "desc"}, "searchq_keyword": {"order": "asc"}}, + SortBy.dateAscending: {"date_created": {"order": "asc"}, "searchq_keyword": {"order": "asc"}}, + SortBy.dateDescending: {"date_created": {"order": "desc"}, "searchq_keyword": {"order": "asc"}}, }[explore_search_request.sortBy] s = get_search( diff --git a/frontend/src/common/schema_types.ts b/frontend/src/common/schema_types.ts index 385d673f6..8adacf060 100644 --- a/frontend/src/common/schema_types.ts +++ b/frontend/src/common/schema_types.ts @@ -122,7 +122,11 @@ export interface Card { /** * Created date - formatted by backend */ - date: string; + dateCreated: string; + /** + * Modified date - formatted by backend + */ + dateModified: string; downloadLink: string; dpi: number; extension: string; @@ -984,7 +988,8 @@ const typeMap: any = { Card: o( [ { json: "cardType", js: "cardType", typ: r("CardType") }, - { json: "date", js: "date", typ: "" }, + { json: "dateCreated", js: "dateCreated", typ: "" }, + { json: "dateModified", js: "dateModified", typ: "" }, { json: "downloadLink", js: "downloadLink", typ: "" }, { json: "dpi", js: "dpi", typ: 0 }, { json: "extension", js: "extension", typ: "" }, diff --git a/frontend/src/common/test-constants.ts b/frontend/src/common/test-constants.ts index 34d35dab6..66ffc8841 100644 --- a/frontend/src/common/test-constants.ts +++ b/frontend/src/common/test-constants.ts @@ -84,7 +84,8 @@ export const cardDocument1: CardDocument = { dpi: 1200, searchq: "card 1", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -107,7 +108,8 @@ export const cardDocument2: CardDocument = { dpi: 1200, searchq: "card 2", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -130,7 +132,8 @@ export const cardDocument3: CardDocument = { dpi: 1200, searchq: "card 3", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -153,7 +156,8 @@ export const cardDocument4: CardDocument = { dpi: 1200, searchq: "card 4", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -176,7 +180,8 @@ export const cardDocument5: CardDocument = { dpi: 1200, searchq: "card 5", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -199,7 +204,8 @@ export const cardDocument6: CardDocument = { dpi: 1200, searchq: "card 6", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", diff --git a/frontend/src/features/card/Card.tsx b/frontend/src/features/card/Card.tsx index 97d300ff7..d5b50d0aa 100644 --- a/frontend/src/features/card/Card.tsx +++ b/frontend/src/features/card/Card.tsx @@ -477,7 +477,7 @@ export function DatedCard({ cardDocument }: { cardDocument: CardDocument }) { diff --git a/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx b/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx index 5c9563e1a..98143fc84 100644 --- a/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx +++ b/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx @@ -122,7 +122,8 @@ export function CardDetailedViewModal({ ), ], ["Resolution", `${cardDocument.dpi} DPI`], - ["Date Created", cardDocument.date], + ["Date Created", cardDocument.dateCreated], + ["Date Modified", cardDocument.dateModified], ["File Size", imageSizeToMBString(cardDocument.size, 2)], ]} hover={true} diff --git a/frontend/src/features/cardDetailedView/__snapshots__/CardDetailedViewModal.test.tsx.snap b/frontend/src/features/cardDetailedView/__snapshots__/CardDetailedViewModal.test.tsx.snap index c11b38abf..cac68d62d 100644 --- a/frontend/src/features/cardDetailedView/__snapshots__/CardDetailedViewModal.test.tsx.snap +++ b/frontend/src/features/cardDetailedView/__snapshots__/CardDetailedViewModal.test.tsx.snap @@ -220,6 +220,19 @@ exports[`the html structure of a CardDetailedViewModal 1`] = ` 1st January, 2000 + + + Date Modified + + + 1st January, 2000 + +