From 3df61a77471aa01797cc1b72fb840093311fcdf3 Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sun, 18 May 2025 07:42:01 +1000 Subject: [PATCH 01/11] only initialise sentry sdk when not running in debug --- MPCAutofill/MPCAutofill/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MPCAutofill/MPCAutofill/settings.py b/MPCAutofill/MPCAutofill/settings.py index a8f232c9..2f60733a 100755 --- a/MPCAutofill/MPCAutofill/settings.py +++ b/MPCAutofill/MPCAutofill/settings.py @@ -217,7 +217,7 @@ PATREON_URL = env("PATREON_URL", default="") # Sentry -if len(sys.argv) >= 2 and sys.argv[1] != "runserver": +if len(sys.argv) >= 2 and sys.argv[1] != "runserver" and env("DJANGO_DEBUG", default=False) is False: sentry_sdk.init( dsn="https://4d29db1957fb9b3153aaba66e776b01f@o4505848489246720.ingest.sentry.io/4505848491540480", integrations=[DjangoIntegration()], From bd4da3f1d00995c67af4796966eb04b4d49de8ab Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Fri, 16 May 2025 19:10:20 +1000 Subject: [PATCH 02/11] backend: Card.date -> Card.date_created --- MPCAutofill/cardpicker/admin.py | 2 +- MPCAutofill/cardpicker/documents.py | 2 +- .../0040_rename_date_card_date_created.py | 18 +++++++++++++ MPCAutofill/cardpicker/models.py | 6 ++--- .../cardpicker/search/search_functions.py | 4 +-- .../cardpicker/sources/update_database.py | 2 +- MPCAutofill/cardpicker/tests/conftest.py | 26 +++++++++---------- MPCAutofill/cardpicker/tests/factories.py | 2 +- MPCAutofill/cardpicker/views.py | 4 +-- 9 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 MPCAutofill/cardpicker/migrations/0040_rename_date_card_date_created.py diff --git a/MPCAutofill/cardpicker/admin.py b/MPCAutofill/cardpicker/admin.py index 5f3a97a8..23dd36ff 100644 --- a/MPCAutofill/cardpicker/admin.py +++ b/MPCAutofill/cardpicker/admin.py @@ -12,7 +12,7 @@ class AdminTag(admin.ModelAdmin[Tag]): @admin.register(Card) class AdminCard(admin.ModelAdmin[Card]): - list_display = ("identifier", "name", "source", "dpi", "date", "tags") + list_display = ("identifier", "name", "source", "dpi", "date_created", "tags") search_fields = ("identifier", "name") diff --git a/MPCAutofill/cardpicker/documents.py b/MPCAutofill/cardpicker/documents.py index c2984e92..7fda7bff 100644 --- a/MPCAutofill/cardpicker/documents.py +++ b/MPCAutofill/cardpicker/documents.py @@ -18,7 +18,7 @@ class CardSearch(Document): searchq_precise = fields.TextField(attr="searchq", analyzer=precise_analyser) searchq_keyword = fields.KeywordField(attr="searchq") card_type = fields.KeywordField() - date = fields.DateField() + date_created = fields.DateField() language = fields.TextField(analyzer=precise_analyser) # case insensitivity is one less thing which can go wrong tags = fields.KeywordField() # all elasticsearch fields support arrays by default diff --git a/MPCAutofill/cardpicker/migrations/0040_rename_date_card_date_created.py b/MPCAutofill/cardpicker/migrations/0040_rename_date_card_date_created.py new file mode 100644 index 00000000..7b432d6f --- /dev/null +++ b/MPCAutofill/cardpicker/migrations/0040_rename_date_card_date_created.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.14 on 2025-05-16 09:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("cardpicker", "0039_remove_card_searchq_keyword"), + ] + + operations = [ + migrations.RenameField( + model_name="card", + old_name="date", + new_name="date_created", + ), + ] diff --git a/MPCAutofill/cardpicker/models.py b/MPCAutofill/cardpicker/models.py index 5e541407..84b5898a 100755 --- a/MPCAutofill/cardpicker/models.py +++ b/MPCAutofill/cardpicker/models.py @@ -193,7 +193,7 @@ class Card(models.Model): dpi = models.IntegerField(default=0) searchq = models.CharField(max_length=200) extension = models.CharField(max_length=200) - date = models.DateTimeField(default=datetime.now) + date_created = models.DateTimeField(default=datetime.now) size = models.IntegerField() tags = ArrayField(models.CharField(max_length=20), default=list, blank=True) # null=True is just for admin panel language = models.CharField(max_length=5) @@ -205,7 +205,7 @@ def __str__(self) -> str: f"{self.name} " f"[Type: {self.card_type}, " f"Identifier: {self.identifier}, " - f"Uploaded: {self.date.strftime('%d/%m/%Y')}, " + f"Uploaded: {self.date_created.strftime('%d/%m/%Y')}, " f"Priority: {self.priority}]" ) @@ -225,7 +225,7 @@ def serialise(self) -> SerialisedCard: dpi=self.dpi, searchq=self.searchq, extension=self.extension, - date=dateformat.format(self.date, DATE_FORMAT), + date=dateformat.format(self.date_created, DATE_FORMAT), size=self.size, downloadLink=self.get_download_link() or "", smallThumbnailUrl=self.get_small_thumbnail_url() or "", diff --git a/MPCAutofill/cardpicker/search/search_functions.py b/MPCAutofill/cardpicker/search/search_functions.py index 03867333..3e6ad3bd 100644 --- a/MPCAutofill/cardpicker/search/search_functions.py +++ b/MPCAutofill/cardpicker/search/search_functions.py @@ -204,8 +204,8 @@ def retrieve_cardback_identifiers(search_settings: SearchSettings) -> list[str]: def get_new_cards_paginator(source: Source) -> Paginator[QuerySet[Card]]: now = timezone.now() cards = Card.objects.filter( - source=source, date__lt=now, date__gte=now - dt.timedelta(days=NEW_CARDS_DAYS) - ).order_by("-date", "name") + source=source, date_created__lt=now, date_created__gte=now - dt.timedelta(days=NEW_CARDS_DAYS) + ).order_by("-date_created", "name") return Paginator(cards, NEW_CARDS_PAGE_SIZE) # type: ignore # TODO: `_SupportsPagination` diff --git a/MPCAutofill/cardpicker/sources/update_database.py b/MPCAutofill/cardpicker/sources/update_database.py index e9967456..941d67b5 100644 --- a/MPCAutofill/cardpicker/sources/update_database.py +++ b/MPCAutofill/cardpicker/sources/update_database.py @@ -112,7 +112,7 @@ def transform_images_into_objects(source: Source, images: list[Image], tags: Tag dpi=dpi, searchq=searchable_name, # search-friendly card name extension=extension, - date=image.created_time, + date_created=image.created_time, size=image.size, tags=list(extracted_tags), language=(language or DEFAULT_LANGUAGE).alpha_2.upper(), diff --git a/MPCAutofill/cardpicker/tests/conftest.py b/MPCAutofill/cardpicker/tests/conftest.py index 3a739eed..171dfc44 100644 --- a/MPCAutofill/cardpicker/tests/conftest.py +++ b/MPCAutofill/cardpicker/tests/conftest.py @@ -132,7 +132,7 @@ def brainstorm(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.BRAINSTORM.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -147,7 +147,7 @@ def island(example_drive_1) -> Card: source=example_drive_1, priority=7, size=Cards.ISLAND.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -162,7 +162,7 @@ def island_classical(example_drive_1) -> Card: source=example_drive_1, priority=6, size=Cards.ISLAND_CLASSICAL.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), language="FR", ) @@ -178,7 +178,7 @@ def mountain(example_drive_1) -> Card: source=example_drive_1, priority=7, size=Cards.MOUNTAIN.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -193,7 +193,7 @@ def simple_cube(example_drive_1, tag_in_data, another_tag_in_data) -> Card: source=example_drive_1, priority=17, size=Cards.SIMPLE_CUBE.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), tags=[tag_in_data.name, another_tag_in_data.name], language="DE", ) @@ -210,7 +210,7 @@ def simple_lotus(example_drive_2, tag_in_data) -> Card: source=example_drive_2, priority=7, size=Cards.SIMPLE_LOTUS.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), tags=[tag_in_data.name], language="EN", ) @@ -227,7 +227,7 @@ def huntmaster_of_the_fells(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.HUNTMASTER_OF_THE_FELLS.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -242,7 +242,7 @@ def ravager_of_the_fells(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.RAVAGER_OF_THE_FELLS.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -257,7 +257,7 @@ def past_in_flames_1(example_drive_1, tag_in_data) -> Card: source=example_drive_1, priority=2, size=Cards.PAST_IN_FLAMES_1.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), tags=[tag_in_data.name], language="EN", ) @@ -274,7 +274,7 @@ def past_in_flames_2(example_drive_2, tag_in_data, another_tag_in_data) -> Card: source=example_drive_2, priority=2, size=Cards.PAST_IN_FLAMES_2.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), tags=[tag_in_data.name, another_tag_in_data.name], language="DE", ) @@ -291,7 +291,7 @@ def delver_of_secrets(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.DELVER_OF_SECRETS.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -306,7 +306,7 @@ def insectile_aberration(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.INSECTILE_ABERRATION.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) @@ -321,7 +321,7 @@ def goblin(example_drive_1) -> Card: source=example_drive_1, priority=2, size=Cards.GOBLIN.value.size, - date=dt.datetime(2023, 1, 1), + date_created=dt.datetime(2023, 1, 1), ) diff --git a/MPCAutofill/cardpicker/tests/factories.py b/MPCAutofill/cardpicker/tests/factories.py index ed1eb875..69d3d11d 100644 --- a/MPCAutofill/cardpicker/tests/factories.py +++ b/MPCAutofill/cardpicker/tests/factories.py @@ -32,7 +32,7 @@ class Meta: model = models.Card card_type = models.CardTypes.CARD - date = factory.LazyFunction(lambda: dt.datetime(2023, 1, 1)) # for snapshot consistency + date_created = factory.LazyFunction(lambda: dt.datetime(2023, 1, 1)) # for snapshot consistency identifier = factory.Sequence(lambda n: f"card_{n}") name = factory.Sequence(lambda n: f"Card {n}") priority = factory.Sequence(lambda n: n) diff --git a/MPCAutofill/cardpicker/views.py b/MPCAutofill/cardpicker/views.py index 03f6ffd7..ede8850b 100644 --- a/MPCAutofill/cardpicker/views.py +++ b/MPCAutofill/cardpicker/views.py @@ -164,8 +164,8 @@ def post_explore_search(request: HttpRequest) -> HttpResponse: sort: dict[str, dict[str, str]] = { SortBy.nameAscending: {"searchq_keyword": {"order": "asc"}}, SortBy.nameDescending: {"searchq_keyword": {"order": "desc"}}, - SortBy.dateAscending: {"date": {"order": "asc"}, "searchq_keyword": {"order": "asc"}}, - SortBy.dateDescending: {"date": {"order": "desc"}, "searchq_keyword": {"order": "asc"}}, + SortBy.dateAscending: {"date_created": {"order": "asc"}, "searchq_keyword": {"order": "asc"}}, + SortBy.dateDescending: {"date_created": {"order": "desc"}, "searchq_keyword": {"order": "asc"}}, }[explore_search_request.sortBy] s = get_search( From 2971837bd17a30949374f0a3a2aaa744c43cd35b Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sat, 17 May 2025 07:18:48 +1000 Subject: [PATCH 03/11] backend: add Card.date_modified field --- MPCAutofill/cardpicker/documents.py | 1 + .../migrations/0041_card_date_modified.py | 23 +++ MPCAutofill/cardpicker/models.py | 1 + MPCAutofill/cardpicker/sources/api.py | 1 + .../cardpicker/sources/source_types.py | 3 +- .../cardpicker/sources/update_database.py | 1 + MPCAutofill/cardpicker/tests/factories.py | 1 + MPCAutofill/cardpicker/tests/test_sources.py | 142 ++++++++++++++++-- 8 files changed, 156 insertions(+), 17 deletions(-) create mode 100644 MPCAutofill/cardpicker/migrations/0041_card_date_modified.py diff --git a/MPCAutofill/cardpicker/documents.py b/MPCAutofill/cardpicker/documents.py index 7fda7bff..48aba693 100644 --- a/MPCAutofill/cardpicker/documents.py +++ b/MPCAutofill/cardpicker/documents.py @@ -19,6 +19,7 @@ class CardSearch(Document): searchq_keyword = fields.KeywordField(attr="searchq") card_type = fields.KeywordField() date_created = fields.DateField() + date_modified = fields.DateField() language = fields.TextField(analyzer=precise_analyser) # case insensitivity is one less thing which can go wrong tags = fields.KeywordField() # all elasticsearch fields support arrays by default diff --git a/MPCAutofill/cardpicker/migrations/0041_card_date_modified.py b/MPCAutofill/cardpicker/migrations/0041_card_date_modified.py new file mode 100644 index 00000000..66090d04 --- /dev/null +++ b/MPCAutofill/cardpicker/migrations/0041_card_date_modified.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.14 on 2025-05-16 21:15 + +import datetime + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("cardpicker", "0040_rename_date_card_date_created"), + ] + + operations = [ + migrations.AddField( + model_name="card", + name="date_modified", + field=models.DateTimeField(default=datetime.datetime.now), + ), + migrations.RunSQL( + sql="UPDATE cardpicker_card SET date_modified = date_created", reverse_sql=migrations.RunSQL.noop + ), + ] diff --git a/MPCAutofill/cardpicker/models.py b/MPCAutofill/cardpicker/models.py index 84b5898a..79e3fb6b 100755 --- a/MPCAutofill/cardpicker/models.py +++ b/MPCAutofill/cardpicker/models.py @@ -194,6 +194,7 @@ class Card(models.Model): searchq = models.CharField(max_length=200) extension = models.CharField(max_length=200) date_created = models.DateTimeField(default=datetime.now) + date_modified = models.DateTimeField(default=datetime.now) size = models.IntegerField() tags = ArrayField(models.CharField(max_length=20), default=list, blank=True) # null=True is just for admin panel language = models.CharField(max_length=5) diff --git a/MPCAutofill/cardpicker/sources/api.py b/MPCAutofill/cardpicker/sources/api.py index 0aacf742..8dff9b53 100644 --- a/MPCAutofill/cardpicker/sources/api.py +++ b/MPCAutofill/cardpicker/sources/api.py @@ -70,6 +70,7 @@ class Image: name: str size: int created_time: dt.datetime + modified_time: dt.datetime height: int folder: Folder diff --git a/MPCAutofill/cardpicker/sources/source_types.py b/MPCAutofill/cardpicker/sources/source_types.py index cb86df2b..6d8c9b7e 100644 --- a/MPCAutofill/cardpicker/sources/source_types.py +++ b/MPCAutofill/cardpicker/sources/source_types.py @@ -151,7 +151,7 @@ def get_all_images_inside_folder(folder: Folder) -> list[Image]: "mimeType contains 'image/jpeg') and " f"'{folder.id}' in parents", fields="nextPageToken, files(" - "id, name, trashed, size, parents, createdTime, imageMediaMetadata" + "id, name, trashed, size, parents, createdTime, modifiedTime, imageMediaMetadata" ")", pageSize=500, pageToken=page_token, @@ -168,6 +168,7 @@ def get_all_images_inside_folder(folder: Folder) -> list[Image]: id=item["id"], name=item["name"], created_time=item["createdTime"], + modified_time=item["modifiedTime"], folder=folder, height=item["imageMediaMetadata"]["height"], size=int(item["size"]), diff --git a/MPCAutofill/cardpicker/sources/update_database.py b/MPCAutofill/cardpicker/sources/update_database.py index 941d67b5..9f4543d9 100644 --- a/MPCAutofill/cardpicker/sources/update_database.py +++ b/MPCAutofill/cardpicker/sources/update_database.py @@ -113,6 +113,7 @@ def transform_images_into_objects(source: Source, images: list[Image], tags: Tag searchq=searchable_name, # search-friendly card name extension=extension, date_created=image.created_time, + date_modified=image.modified_time, size=image.size, tags=list(extracted_tags), language=(language or DEFAULT_LANGUAGE).alpha_2.upper(), diff --git a/MPCAutofill/cardpicker/tests/factories.py b/MPCAutofill/cardpicker/tests/factories.py index 69d3d11d..c8be4ca1 100644 --- a/MPCAutofill/cardpicker/tests/factories.py +++ b/MPCAutofill/cardpicker/tests/factories.py @@ -33,6 +33,7 @@ class Meta: card_type = models.CardTypes.CARD date_created = factory.LazyFunction(lambda: dt.datetime(2023, 1, 1)) # for snapshot consistency + date_modified = factory.LazyAttribute(lambda o: o.date_created) identifier = factory.Sequence(lambda n: f"card_{n}") name = factory.Sequence(lambda n: f"Card {n}") priority = factory.Sequence(lambda n: n) diff --git a/MPCAutofill/cardpicker/tests/test_sources.py b/MPCAutofill/cardpicker/tests/test_sources.py index 1f873859..4638448c 100644 --- a/MPCAutofill/cardpicker/tests/test_sources.py +++ b/MPCAutofill/cardpicker/tests/test_sources.py @@ -26,47 +26,156 @@ class TestAPI: FOLDER_Z = Folder(id="z", name="Folder z [Full Art", parent=None) FOLDER_FRENCH = Folder(id="french", name="{FR} Folder", parent=None) - IMAGE_A = Image(id="a", name="Image A.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A) - IMAGE_B = Image(id="b", name="Image B [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A) - IMAGE_C = Image(id="b", name="Image C.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_C) + IMAGE_A = Image( + id="a", + name="Image A.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, + ) + IMAGE_B = Image( + id="b", + name="Image B [NSFW].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, + ) + IMAGE_C = Image( + id="b", + name="Image C.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_C, + ) IMAGE_D = Image( - id="b", name="Image D [NSFW, full art].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_C + id="b", + name="Image D [NSFW, full art].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_C, ) IMAGE_E = Image( - id="e", name="Image E [invalid tag.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="e", + name="Image E [invalid tag.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_F = Image( - id="F", name="Image F [NSFW, tag in data].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="F", + name="Image F [NSFW, tag in data].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_G = Image( - id="G", name="Image G [NSFW] (John Doe).png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="G", + name="Image G [NSFW] (John Doe).png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_H = Image( - id="H", name="Image H [A, NSFW, B] (John Doe).png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="H", + name="Image H [A, NSFW, B] (John Doe).png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, + ) + IMAGE_I = Image( + id="I", + name="Image A.I.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) - IMAGE_I = Image(id="I", name="Image A.I.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A) IMAGE_J = Image( - id="J", name="Image J [Child Tag].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="J", + name="Image J [Child Tag].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_K = Image( - id="K", name="Image K [Grandchild Tag].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="K", + name="Image K [Grandchild Tag].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, + ) + IMAGE_L = Image( + id="L", + name="Image L [NSFW].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_D, ) - IMAGE_L = Image(id="L", name="Image L [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_D) IMAGE_FRENCH = Image( - id="french", name="French.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH + id="french", + name="French.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_FRENCH, ) IMAGE_ENGLISH = Image( - id="english", name="{EN} English.png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH + id="english", + name="{EN} English.png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_FRENCH, + ) + IMAGE_NSFW = Image( + id="nsfw", + name="NSFW [NSFW].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) - IMAGE_NSFW = Image(id="nsfw", name="NSFW [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A) IMAGE_DOUBLE_NSFW = Image( - id="double nsfw", name="NSFW (NSFW) [NSFW].png", size=1, created_time=DEFAULT_DATE, height=1, folder=FOLDER_A + id="double nsfw", + name="NSFW (NSFW) [NSFW].png", + size=1, + created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, + height=1, + folder=FOLDER_A, ) IMAGE_IMPLICITLY_FRENCH = Image( id="implicitly_french", name="Implicitly French.png", size=1, created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH, ) @@ -75,6 +184,7 @@ class TestAPI: name="{EN} Explicitly English.png", size=1, created_time=DEFAULT_DATE, + modified_time=DEFAULT_DATE, height=1, folder=FOLDER_FRENCH, ) From c6df6db24ed20631658e598d6caacdbd1210e3c1 Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sat, 17 May 2025 15:00:02 +1000 Subject: [PATCH 04/11] backend: only apply diffs to database in source crawl --- .../cardpicker/sources/source_types.py | 6 ++- .../cardpicker/sources/update_database.py | 41 +++++++++++++++++-- MPCAutofill/cardpicker/tests/test_sources.py | 1 - 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/MPCAutofill/cardpicker/sources/source_types.py b/MPCAutofill/cardpicker/sources/source_types.py index 6d8c9b7e..44a0e31f 100644 --- a/MPCAutofill/cardpicker/sources/source_types.py +++ b/MPCAutofill/cardpicker/sources/source_types.py @@ -5,6 +5,8 @@ from tqdm import tqdm from django.db.models import TextChoices +from django.utils.dateparse import parse_datetime +from django.utils.timezone import now from django.utils.translation import gettext_lazy from cardpicker.schema_types import SourceType as SchemaSourceType @@ -167,8 +169,8 @@ def get_all_images_inside_folder(folder: Folder) -> list[Image]: Image( id=item["id"], name=item["name"], - created_time=item["createdTime"], - modified_time=item["modifiedTime"], + created_time=parse_datetime(item["createdTime"]) or now(), + modified_time=parse_datetime(item["modifiedTime"]) or now(), folder=folder, height=item["imageMediaMetadata"]["height"], size=int(item["size"]), diff --git a/MPCAutofill/cardpicker/sources/update_database.py b/MPCAutofill/cardpicker/sources/update_database.py index 9f4543d9..f97e816b 100644 --- a/MPCAutofill/cardpicker/sources/update_database.py +++ b/MPCAutofill/cardpicker/sources/update_database.py @@ -144,9 +144,44 @@ def transform_images_into_objects(source: Source, images: list[Image], tags: Tag def bulk_sync_objects(source: Source, cards: list[Card]) -> None: print(f"Synchronising objects to database for source {TEXT_BOLD}{source.name}{TEXT_END}...", end="", flush=True) t0 = time.time() - with transaction.atomic(): # django-bulk-sync is crushingly slow with postgres - Card.objects.filter(source=source).delete() - Card.objects.bulk_create(cards) + + incoming = {card.identifier: card for card in cards} + incoming_ids = set(incoming.keys()) + existing = {card.identifier: card for card in Card.objects.filter(source=source)} + existing_ids = set(existing.keys()) + + created = [incoming[identifier] for identifier in incoming_ids - existing_ids] + updated = [ + incoming[identifier] + for identifier in (incoming_ids & existing_ids) + if incoming[identifier].date_modified > existing[identifier].date_modified + ] + deleted_ids = existing_ids - incoming_ids + + with transaction.atomic(): + Card.objects.bulk_create(created) + Card.objects.bulk_update( + updated, + # update every field except for `identifier` + [ + "card_type", + "name", + "priority", + "source", + "source_verbose", + "folder_location", + "dpi", + "searchq", + "extension", + "date_created", + "date_modified", + "size", + "tags", + "language", + ], + batch_size=1000, + ) + Card.objects.filter(identifier__in=deleted_ids).delete() print(f" and done! That took {TEXT_BOLD}{(time.time() - t0):.2f}{TEXT_END} seconds.") diff --git a/MPCAutofill/cardpicker/tests/test_sources.py b/MPCAutofill/cardpicker/tests/test_sources.py index 4638448c..da225629 100644 --- a/MPCAutofill/cardpicker/tests/test_sources.py +++ b/MPCAutofill/cardpicker/tests/test_sources.py @@ -347,7 +347,6 @@ def test_comprehensive_snapshot(self, snapshot, django_settings, elasticsearch, update_database() assert list(Card.objects.all().order_by("identifier")) == snapshot(name="cards") - @pytest.mark.skip("we turned off upsert at time of writing because it's extremely slow with postgres") def test_upsert(self, django_settings, elasticsearch, all_sources): update_database() pk_to_identifier_1 = {x.pk: x.identifier for x in Card.objects.all()} From 3405aec734d293a32597888a4f474d6f143ad9cc Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sat, 17 May 2025 15:21:09 +1000 Subject: [PATCH 05/11] schemas, frontend: Card.date -> Card.dateCreated --- MPCAutofill/cardpicker/models.py | 2 +- MPCAutofill/cardpicker/schema_types.py | 8 +- .../cardpicker/sources/update_database.py | 15 +- .../tests/__snapshots__/test_views.ambr | 132 +++++++++--------- frontend/src/common/schema_types.ts | 4 +- frontend/src/common/test-constants.ts | 12 +- frontend/src/features/card/Card.tsx | 2 +- .../CardDetailedViewModal.tsx | 2 +- frontend/src/features/ui/DynamicLogo.tsx | 2 +- schemas/schemas/Card.json | 4 +- 10 files changed, 93 insertions(+), 90 deletions(-) diff --git a/MPCAutofill/cardpicker/models.py b/MPCAutofill/cardpicker/models.py index 79e3fb6b..14d10485 100755 --- a/MPCAutofill/cardpicker/models.py +++ b/MPCAutofill/cardpicker/models.py @@ -226,7 +226,7 @@ def serialise(self) -> SerialisedCard: dpi=self.dpi, searchq=self.searchq, extension=self.extension, - date=dateformat.format(self.date_created, DATE_FORMAT), + dateCreated=dateformat.format(self.date_created, DATE_FORMAT), size=self.size, downloadLink=self.get_download_link() or "", smallThumbnailUrl=self.get_small_thumbnail_url() or "", diff --git a/MPCAutofill/cardpicker/schema_types.py b/MPCAutofill/cardpicker/schema_types.py index e4f7fc76..6275b59a 100644 --- a/MPCAutofill/cardpicker/schema_types.py +++ b/MPCAutofill/cardpicker/schema_types.py @@ -232,7 +232,7 @@ class SourceType(str, Enum): class Card(BaseModel): cardType: CardType - date: str + dateCreated: str """Created date - formatted by backend""" downloadLink: str @@ -258,7 +258,7 @@ class Card(BaseModel): def from_dict(obj: Any) -> "Card": assert isinstance(obj, dict) cardType = CardType(obj.get("cardType")) - date = from_str(obj.get("date")) + dateCreated = from_str(obj.get("dateCreated")) downloadLink = from_str(obj.get("downloadLink")) dpi = from_int(obj.get("dpi")) extension = from_str(obj.get("extension")) @@ -279,7 +279,7 @@ def from_dict(obj: Any) -> "Card": sourceType = from_union([SourceType, from_none], obj.get("sourceType")) return Card( cardType, - date, + dateCreated, downloadLink, dpi, extension, @@ -303,7 +303,7 @@ def from_dict(obj: Any) -> "Card": def to_dict(self) -> dict: result: dict = {} result["cardType"] = to_enum(CardType, self.cardType) - result["date"] = from_str(self.date) + result["dateCreated"] = from_str(self.dateCreated) result["downloadLink"] = from_str(self.downloadLink) result["dpi"] = from_int(self.dpi) result["extension"] = from_str(self.extension) diff --git a/MPCAutofill/cardpicker/sources/update_database.py b/MPCAutofill/cardpicker/sources/update_database.py index f97e816b..7c75ab37 100644 --- a/MPCAutofill/cardpicker/sources/update_database.py +++ b/MPCAutofill/cardpicker/sources/update_database.py @@ -151,11 +151,11 @@ def bulk_sync_objects(source: Source, cards: list[Card]) -> None: existing_ids = set(existing.keys()) created = [incoming[identifier] for identifier in incoming_ids - existing_ids] - updated = [ - incoming[identifier] - for identifier in (incoming_ids & existing_ids) - if incoming[identifier].date_modified > existing[identifier].date_modified - ] + updated: list[Card] = [] + for identifier in incoming_ids & existing_ids: + if incoming[identifier].date_modified > existing[identifier].date_modified: + incoming[identifier].pk = existing[identifier].pk # this must be explicitly set for bulk_update. + updated.append(incoming[identifier]) deleted_ids = existing_ids - incoming_ids with transaction.atomic(): @@ -182,7 +182,10 @@ def bulk_sync_objects(source: Source, cards: list[Card]) -> None: batch_size=1000, ) Card.objects.filter(identifier__in=deleted_ids).delete() - print(f" and done! That took {TEXT_BOLD}{(time.time() - t0):.2f}{TEXT_END} seconds.") + print( + f" and done! That took {TEXT_BOLD}{(time.time() - t0):.2f}{TEXT_END} seconds.\n" + f"Created {len(created)}, updated {len(updated)}, and deleted {len(deleted_ids)} cards." + ) def update_database_for_source(source: Source, source_type: Type[SourceType], root_folder: Folder, tags: Tags) -> None: diff --git a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr index b498e2e9..3e44df44 100644 --- a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr +++ b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr @@ -192,7 +192,7 @@ 'CARD': dict({ 'Brainstorm': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -215,7 +215,7 @@ }), 'Island': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -238,7 +238,7 @@ }), 'Island (William Bradford)': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -261,7 +261,7 @@ }), 'Pást in Flames': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -289,7 +289,7 @@ 'TOKEN': dict({ 'Goblin': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -394,7 +394,7 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -417,7 +417,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -440,7 +440,7 @@ }), dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -463,7 +463,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -486,7 +486,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -509,7 +509,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -546,7 +546,7 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -571,7 +571,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -653,7 +653,7 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -676,7 +676,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -699,7 +699,7 @@ }), dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -722,7 +722,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -745,7 +745,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -768,7 +768,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -800,7 +800,7 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -823,7 +823,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -846,7 +846,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -870,7 +870,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -893,7 +893,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -927,7 +927,7 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -952,7 +952,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -1326,7 +1326,7 @@ 'results': dict({ '17fopRCNRge72U8Hac8pApHZtEalx5kHy': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -1349,7 +1349,7 @@ }), '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1391,7 +1391,7 @@ 'results': dict({ '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1423,7 +1423,7 @@ 'results': dict({ '17fopRCNRge72U8Hac8pApHZtEalx5kHy': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -1446,7 +1446,7 @@ }), '1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy': dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -1469,7 +1469,7 @@ }), '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1511,7 +1511,7 @@ 'results': dict({ '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2175,7 +2175,7 @@ 'cards': list([ dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2198,7 +2198,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2223,7 +2223,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2257,7 +2257,7 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2280,7 +2280,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -2303,7 +2303,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -2326,7 +2326,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -2349,7 +2349,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -2372,7 +2372,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -2395,7 +2395,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -2418,7 +2418,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -2443,7 +2443,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -2467,7 +2467,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -2500,7 +2500,7 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2523,7 +2523,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -2546,7 +2546,7 @@ }), dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2569,7 +2569,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -2592,7 +2592,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -2615,7 +2615,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -2638,7 +2638,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -2661,7 +2661,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -2684,7 +2684,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -2709,7 +2709,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -2733,7 +2733,7 @@ }), dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -2756,7 +2756,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2781,7 +2781,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2815,7 +2815,7 @@ 'cards': list([ dict({ 'cardType': 'CARD', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2848,7 +2848,7 @@ 'cards': list([ dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2882,7 +2882,7 @@ 'cards': list([ dict({ 'cardType': 'TOKEN', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2915,7 +2915,7 @@ 'cards': list([ dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2940,7 +2940,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2974,7 +2974,7 @@ 'cards': list([ dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2998,7 +2998,7 @@ }), dict({ 'cardType': 'CARDBACK', - 'date': '1st January, 2023', + 'dateCreated': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', diff --git a/frontend/src/common/schema_types.ts b/frontend/src/common/schema_types.ts index 385d673f..9f9d57c5 100644 --- a/frontend/src/common/schema_types.ts +++ b/frontend/src/common/schema_types.ts @@ -122,7 +122,7 @@ export interface Card { /** * Created date - formatted by backend */ - date: string; + dateCreated: string; downloadLink: string; dpi: number; extension: string; @@ -984,7 +984,7 @@ const typeMap: any = { Card: o( [ { json: "cardType", js: "cardType", typ: r("CardType") }, - { json: "date", js: "date", typ: "" }, + { json: "dateCreated", js: "dateCreated", typ: "" }, { json: "downloadLink", js: "downloadLink", typ: "" }, { json: "dpi", js: "dpi", typ: 0 }, { json: "extension", js: "extension", typ: "" }, diff --git a/frontend/src/common/test-constants.ts b/frontend/src/common/test-constants.ts index 34d35dab..8d3412aa 100644 --- a/frontend/src/common/test-constants.ts +++ b/frontend/src/common/test-constants.ts @@ -84,7 +84,7 @@ export const cardDocument1: CardDocument = { dpi: 1200, searchq: "card 1", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -107,7 +107,7 @@ export const cardDocument2: CardDocument = { dpi: 1200, searchq: "card 2", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -130,7 +130,7 @@ export const cardDocument3: CardDocument = { dpi: 1200, searchq: "card 3", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -153,7 +153,7 @@ export const cardDocument4: CardDocument = { dpi: 1200, searchq: "card 4", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -176,7 +176,7 @@ export const cardDocument5: CardDocument = { dpi: 1200, searchq: "card 5", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -199,7 +199,7 @@ export const cardDocument6: CardDocument = { dpi: 1200, searchq: "card 6", extension: "png", - date: "1st January, 2000", // formatted by backend + dateCreated: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", diff --git a/frontend/src/features/card/Card.tsx b/frontend/src/features/card/Card.tsx index 97d300ff..d5b50d0a 100644 --- a/frontend/src/features/card/Card.tsx +++ b/frontend/src/features/card/Card.tsx @@ -477,7 +477,7 @@ export function DatedCard({ cardDocument }: { cardDocument: CardDocument }) { diff --git a/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx b/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx index 5c9563e1..a06162dc 100644 --- a/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx +++ b/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx @@ -122,7 +122,7 @@ export function CardDetailedViewModal({ ), ], ["Resolution", `${cardDocument.dpi} DPI`], - ["Date Created", cardDocument.date], + ["Date Created", cardDocument.dateCreated], ["File Size", imageSizeToMBString(cardDocument.size, 2)], ]} hover={true} diff --git a/frontend/src/features/ui/DynamicLogo.tsx b/frontend/src/features/ui/DynamicLogo.tsx index 308a9835..c5ce0c47 100644 --- a/frontend/src/features/ui/DynamicLogo.tsx +++ b/frontend/src/features/ui/DynamicLogo.tsx @@ -220,7 +220,7 @@ const SampleCardDocument: CardDocument = { dpi: 300, searchq: "", extension: "png", - date: "1st January, 2000", + dateCreated: "1st January, 2000", downloadLink: "", size: 1, smallThumbnailUrl: "/logo-blank.png", diff --git a/schemas/schemas/Card.json b/schemas/schemas/Card.json index b7baba44..a156b7e3 100644 --- a/schemas/schemas/Card.json +++ b/schemas/schemas/Card.json @@ -42,7 +42,7 @@ "extension": { "type": "string" }, - "date": { + "dateCreated": { "description": "Created date - formatted by backend", "type": "string" }, @@ -80,7 +80,7 @@ "dpi", "searchq", "extension", - "date", + "dateCreated", "downloadLink", "size", "smallThumbnailUrl", From 7885fdda7d0ef0e046ac2b7be66c842af84045c5 Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sat, 17 May 2025 15:27:49 +1000 Subject: [PATCH 06/11] schemas, frontend: add Card.dateModified field --- MPCAutofill/cardpicker/models.py | 1 + MPCAutofill/cardpicker/schema_types.py | 6 ++ .../tests/__snapshots__/test_views.ambr | 66 +++++++++++++++++++ frontend/src/common/schema_types.ts | 5 ++ frontend/src/common/test-constants.ts | 6 ++ schemas/schemas/Card.json | 5 ++ 6 files changed, 89 insertions(+) diff --git a/MPCAutofill/cardpicker/models.py b/MPCAutofill/cardpicker/models.py index 14d10485..28787c95 100755 --- a/MPCAutofill/cardpicker/models.py +++ b/MPCAutofill/cardpicker/models.py @@ -227,6 +227,7 @@ def serialise(self) -> SerialisedCard: searchq=self.searchq, extension=self.extension, dateCreated=dateformat.format(self.date_created, DATE_FORMAT), + dateModified=dateformat.format(self.date_modified, DATE_FORMAT), size=self.size, downloadLink=self.get_download_link() or "", smallThumbnailUrl=self.get_small_thumbnail_url() or "", diff --git a/MPCAutofill/cardpicker/schema_types.py b/MPCAutofill/cardpicker/schema_types.py index 6275b59a..cd112d08 100644 --- a/MPCAutofill/cardpicker/schema_types.py +++ b/MPCAutofill/cardpicker/schema_types.py @@ -235,6 +235,9 @@ class Card(BaseModel): dateCreated: str """Created date - formatted by backend""" + dateModified: str + """Modified date - formatted by backend""" + downloadLink: str dpi: int extension: str @@ -259,6 +262,7 @@ def from_dict(obj: Any) -> "Card": assert isinstance(obj, dict) cardType = CardType(obj.get("cardType")) dateCreated = from_str(obj.get("dateCreated")) + dateModified = from_str(obj.get("dateModified")) downloadLink = from_str(obj.get("downloadLink")) dpi = from_int(obj.get("dpi")) extension = from_str(obj.get("extension")) @@ -280,6 +284,7 @@ def from_dict(obj: Any) -> "Card": return Card( cardType, dateCreated, + dateModified, downloadLink, dpi, extension, @@ -304,6 +309,7 @@ def to_dict(self) -> dict: result: dict = {} result["cardType"] = to_enum(CardType, self.cardType) result["dateCreated"] = from_str(self.dateCreated) + result["dateModified"] = from_str(self.dateModified) result["downloadLink"] = from_str(self.downloadLink) result["dpi"] = from_int(self.dpi) result["extension"] = from_str(self.extension) diff --git a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr index 3e44df44..88f937da 100644 --- a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr +++ b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr @@ -193,6 +193,7 @@ 'Brainstorm': dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -216,6 +217,7 @@ 'Island': dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -239,6 +241,7 @@ 'Island (William Bradford)': dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -262,6 +265,7 @@ 'Pást in Flames': dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -290,6 +294,7 @@ 'Goblin': dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -395,6 +400,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -418,6 +424,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -441,6 +448,7 @@ dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -464,6 +472,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -487,6 +496,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -510,6 +520,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -547,6 +558,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -572,6 +584,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -654,6 +667,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -677,6 +691,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -700,6 +715,7 @@ dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -723,6 +739,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -746,6 +763,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -769,6 +787,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -801,6 +820,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -824,6 +844,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -847,6 +868,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -871,6 +893,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -894,6 +917,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -928,6 +952,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -953,6 +978,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -1327,6 +1353,7 @@ '17fopRCNRge72U8Hac8pApHZtEalx5kHy': dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -1350,6 +1377,7 @@ '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1392,6 +1420,7 @@ '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1424,6 +1453,7 @@ '17fopRCNRge72U8Hac8pApHZtEalx5kHy': dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -1447,6 +1477,7 @@ '1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy': dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -1470,6 +1501,7 @@ '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -1512,6 +1544,7 @@ '1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA': dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2176,6 +2209,7 @@ dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2199,6 +2233,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2224,6 +2259,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2258,6 +2294,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2281,6 +2318,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -2304,6 +2342,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -2327,6 +2366,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -2350,6 +2390,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -2373,6 +2414,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -2396,6 +2438,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -2419,6 +2462,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -2444,6 +2488,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -2468,6 +2513,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -2501,6 +2547,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2524,6 +2571,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=17fopRCNRge72U8Hac8pApHZtEalx5kHy&export=download', 'dpi': 600, 'extension': 'png', @@ -2547,6 +2595,7 @@ dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2570,6 +2619,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1991MWCur9NdAFi-tQQD5YbQj2oqV_WRy&export=download', 'dpi': 600, 'extension': 'png', @@ -2593,6 +2643,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1mO73GTYlieP0kiZEkF58pJSrZTmC9lNh&export=download', 'dpi': 600, 'extension': 'png', @@ -2616,6 +2667,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1HsvTYs1jFGe1c8U1PnNZ9aB8jkAW7KU0&export=download', 'dpi': 600, 'extension': 'png', @@ -2639,6 +2691,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1IDtqSjJ4Yo45AnNA4SplOiN7ewibifMa&export=download', 'dpi': 600, 'extension': 'png', @@ -2662,6 +2715,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1-dcs0FEE05MTGiYbKqs9HnRdhXkgtIJG&export=download', 'dpi': 600, 'extension': 'png', @@ -2685,6 +2739,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU&export=download', 'dpi': 400, 'extension': 'png', @@ -2710,6 +2765,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1&export=download', 'dpi': 600, 'extension': 'png', @@ -2734,6 +2790,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1lv8WC1Xf1qxA7VHSc8jOtT5up6FwaBPH&export=download', 'dpi': 600, 'extension': 'png', @@ -2757,6 +2814,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2782,6 +2840,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2816,6 +2875,7 @@ dict({ 'cardType': 'CARD', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5&export=download', 'dpi': 600, 'extension': 'png', @@ -2849,6 +2909,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2883,6 +2944,7 @@ dict({ 'cardType': 'TOKEN', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1V5E0avDmNyEUuFfYwx3nA05aj-1HY0rA&export=download', 'dpi': 600, 'extension': 'png', @@ -2916,6 +2978,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', @@ -2941,6 +3004,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2975,6 +3039,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1oigI6wz0zA--pNMuExKTs40kBNH6VRP_&export=download', 'dpi': 300, 'extension': 'png', @@ -2999,6 +3064,7 @@ dict({ 'cardType': 'CARDBACK', 'dateCreated': '1st January, 2023', + 'dateModified': '1st January, 2023', 'downloadLink': 'https://drive.google.com/uc?id=1JtXL6Ca9nQkvhwZZRR9ZuKA9_DzsFf1V&export=download', 'dpi': 300, 'extension': 'png', diff --git a/frontend/src/common/schema_types.ts b/frontend/src/common/schema_types.ts index 9f9d57c5..8adacf06 100644 --- a/frontend/src/common/schema_types.ts +++ b/frontend/src/common/schema_types.ts @@ -123,6 +123,10 @@ export interface Card { * Created date - formatted by backend */ dateCreated: string; + /** + * Modified date - formatted by backend + */ + dateModified: string; downloadLink: string; dpi: number; extension: string; @@ -985,6 +989,7 @@ const typeMap: any = { [ { json: "cardType", js: "cardType", typ: r("CardType") }, { json: "dateCreated", js: "dateCreated", typ: "" }, + { json: "dateModified", js: "dateModified", typ: "" }, { json: "downloadLink", js: "downloadLink", typ: "" }, { json: "dpi", js: "dpi", typ: 0 }, { json: "extension", js: "extension", typ: "" }, diff --git a/frontend/src/common/test-constants.ts b/frontend/src/common/test-constants.ts index 8d3412aa..66ffc884 100644 --- a/frontend/src/common/test-constants.ts +++ b/frontend/src/common/test-constants.ts @@ -85,6 +85,7 @@ export const cardDocument1: CardDocument = { searchq: "card 1", extension: "png", dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -108,6 +109,7 @@ export const cardDocument2: CardDocument = { searchq: "card 2", extension: "png", dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -131,6 +133,7 @@ export const cardDocument3: CardDocument = { searchq: "card 3", extension: "png", dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -154,6 +157,7 @@ export const cardDocument4: CardDocument = { searchq: "card 4", extension: "png", dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -177,6 +181,7 @@ export const cardDocument5: CardDocument = { searchq: "card 5", extension: "png", dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", @@ -200,6 +205,7 @@ export const cardDocument6: CardDocument = { searchq: "card 6", extension: "png", dateCreated: "1st January, 2000", // formatted by backend + dateModified: "1st January, 2000", // formatted by backend downloadLink: "", size: 10_000_000, smallThumbnailUrl: "", diff --git a/schemas/schemas/Card.json b/schemas/schemas/Card.json index a156b7e3..f44aaf2d 100644 --- a/schemas/schemas/Card.json +++ b/schemas/schemas/Card.json @@ -46,6 +46,10 @@ "description": "Created date - formatted by backend", "type": "string" }, + "dateModified": { + "description": "Modified date - formatted by backend", + "type": "string" + }, "downloadLink": { "type": "string" }, @@ -81,6 +85,7 @@ "searchq", "extension", "dateCreated", + "dateModified", "downloadLink", "size", "smallThumbnailUrl", From 3e73609b736b34282b4e3edd25bd11f5e890cf30 Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sat, 17 May 2025 15:37:24 +1000 Subject: [PATCH 07/11] frontend: show Card.dateModified in detailed view --- .../cardDetailedView/CardDetailedViewModal.tsx | 1 + .../CardDetailedViewModal.test.tsx.snap | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx b/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx index a06162dc..98143fc8 100644 --- a/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx +++ b/frontend/src/features/cardDetailedView/CardDetailedViewModal.tsx @@ -123,6 +123,7 @@ export function CardDetailedViewModal({ ], ["Resolution", `${cardDocument.dpi} DPI`], ["Date Created", cardDocument.dateCreated], + ["Date Modified", cardDocument.dateModified], ["File Size", imageSizeToMBString(cardDocument.size, 2)], ]} hover={true} diff --git a/frontend/src/features/cardDetailedView/__snapshots__/CardDetailedViewModal.test.tsx.snap b/frontend/src/features/cardDetailedView/__snapshots__/CardDetailedViewModal.test.tsx.snap index c11b38ab..cac68d62 100644 --- a/frontend/src/features/cardDetailedView/__snapshots__/CardDetailedViewModal.test.tsx.snap +++ b/frontend/src/features/cardDetailedView/__snapshots__/CardDetailedViewModal.test.tsx.snap @@ -220,6 +220,19 @@ exports[`the html structure of a CardDetailedViewModal 1`] = ` 1st January, 2000 + + + Date Modified + + + 1st January, 2000 + + Date: Sun, 18 May 2025 07:43:01 +1000 Subject: [PATCH 08/11] backend: only apply diffs to elasticsearch in source crawl --- .../management/commands/update_database.py | 3 - .../cardpicker/sources/update_database.py | 60 +++++++++++-------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/MPCAutofill/cardpicker/management/commands/update_database.py b/MPCAutofill/cardpicker/management/commands/update_database.py index 8a50c7bb..4040b378 100755 --- a/MPCAutofill/cardpicker/management/commands/update_database.py +++ b/MPCAutofill/cardpicker/management/commands/update_database.py @@ -1,7 +1,6 @@ import time from typing import Any, Optional -from django.core import management from django.core.management.base import BaseCommand from cardpicker.models import Source @@ -23,6 +22,4 @@ def handle(self, *args: Any, **kwargs: str) -> None: drive: Optional[str] = kwargs.get("drive", None) t0 = time.time() update_database(source_key=drive) - management.call_command("search_index", "--rebuild", "-f") - print("") log_hours_minutes_seconds_elapsed(t0) diff --git a/MPCAutofill/cardpicker/sources/update_database.py b/MPCAutofill/cardpicker/sources/update_database.py index 7c75ab37..18056da5 100644 --- a/MPCAutofill/cardpicker/sources/update_database.py +++ b/MPCAutofill/cardpicker/sources/update_database.py @@ -9,6 +9,7 @@ from django.db import transaction from cardpicker.constants import DEFAULT_LANGUAGE, MAX_SIZE_MB +from cardpicker.documents import CardSearch from cardpicker.models import Card, CardTypes, Source from cardpicker.search.sanitisation import to_searchable from cardpicker.sources.api import Folder, Image @@ -157,34 +158,45 @@ def bulk_sync_objects(source: Source, cards: list[Card]) -> None: incoming[identifier].pk = existing[identifier].pk # this must be explicitly set for bulk_update. updated.append(incoming[identifier]) deleted_ids = existing_ids - incoming_ids + deleted = [existing[identifier] for identifier in deleted_ids] with transaction.atomic(): - Card.objects.bulk_create(created) - Card.objects.bulk_update( - updated, - # update every field except for `identifier` - [ - "card_type", - "name", - "priority", - "source", - "source_verbose", - "folder_location", - "dpi", - "searchq", - "extension", - "date_created", - "date_modified", - "size", - "tags", - "language", - ], - batch_size=1000, - ) - Card.objects.filter(identifier__in=deleted_ids).delete() + if created: + Card.objects.bulk_create(created) + CardSearch().update(list(created), action="index") + if updated: + Card.objects.bulk_update( + updated, + # update every field except for `identifier` + [ + "card_type", + "name", + "priority", + "source", + "source_verbose", + "folder_location", + "dpi", + "searchq", + "extension", + "date_created", + "date_modified", + "size", + "tags", + "language", + ], + batch_size=1000, + ) + # as per this thread https://github.com/django-es/django-elasticsearch-dsl/issues/224#issuecomment-551955511 + # action type "index" is used for indexing new objects as well as updating existing objects + CardSearch().update(list(updated), action="index") + if deleted_ids: + Card.objects.filter(identifier__in=deleted_ids).delete() + CardSearch().update(list(deleted), action="delete") print( f" and done! That took {TEXT_BOLD}{(time.time() - t0):.2f}{TEXT_END} seconds.\n" - f"Created {len(created)}, updated {len(updated)}, and deleted {len(deleted_ids)} cards." + f"Created {TEXT_BOLD}{len(created)}{TEXT_END}, " + f"updated {TEXT_BOLD}{len(updated)}{TEXT_END}, " + f"and deleted {TEXT_BOLD}{len(deleted_ids)}{TEXT_END} cards." ) From 1db5f7b7c8a44080d3a752161ddf59b9e7b92d70 Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sun, 18 May 2025 10:03:47 +1000 Subject: [PATCH 09/11] backend: tests --- MPCAutofill/cardpicker/tests/factories.py | 1 + MPCAutofill/cardpicker/tests/test_sources.py | 95 +++++++++++++++++++- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/MPCAutofill/cardpicker/tests/factories.py b/MPCAutofill/cardpicker/tests/factories.py index c8be4ca1..acd83ccf 100644 --- a/MPCAutofill/cardpicker/tests/factories.py +++ b/MPCAutofill/cardpicker/tests/factories.py @@ -19,6 +19,7 @@ class SourceFactory(factory.django.DjangoModelFactory): class Meta: model = models.Source + identifier = factory.Sequence(lambda n: f"source_{n}") key = factory.Sequence(lambda n: f"source_{n}") name = factory.Sequence(lambda n: f"Source {n}") source_type = models.SourceTypeChoices.GOOGLE_DRIVE diff --git a/MPCAutofill/cardpicker/tests/test_sources.py b/MPCAutofill/cardpicker/tests/test_sources.py index da225629..eb228523 100644 --- a/MPCAutofill/cardpicker/tests/test_sources.py +++ b/MPCAutofill/cardpicker/tests/test_sources.py @@ -1,18 +1,24 @@ import datetime as dt +import freezegun import pytest +from django.core import management +from django.utils.timezone import make_aware, make_naive + +from cardpicker.documents import CardSearch from cardpicker.models import Card from cardpicker.sources.api import Folder, Image -from cardpicker.sources.update_database import update_database +from cardpicker.sources.update_database import bulk_sync_objects, update_database from cardpicker.tags import Tags +from cardpicker.tests import factories + +DEFAULT_DATE = dt.datetime(2023, 1, 1) class TestAPI: # region constants - DEFAULT_DATE = dt.datetime(2023, 1, 1) - FOLDER_A = Folder(id="a", name="Folder A", parent=None) FOLDER_B = Folder(id="b", name="Folder B", parent=FOLDER_A) FOLDER_C = Folder(id="c", name="Folder C [NSFW]", parent=FOLDER_B) @@ -354,4 +360,87 @@ def test_upsert(self, django_settings, elasticsearch, all_sources): pk_to_identifier_2 = {x.pk: x.identifier for x in Card.objects.all()} assert pk_to_identifier_1 == pk_to_identifier_2 + @pytest.mark.parametrize( + "existing_cards, incoming_cards", + [ + pytest.param( + [], + [], + id="no changes to empty database", + ), + pytest.param( + [("existing", "Existing Card", DEFAULT_DATE)], + [("existing", "Existing Card", DEFAULT_DATE)], + id="no changes to populated database", + ), + pytest.param( + [], + [("created", "Created Card", DEFAULT_DATE)], + id="create one card", + ), + pytest.param( + [("updated", "Card to Update", DEFAULT_DATE)], + [("updated", "Updated Card", DEFAULT_DATE + dt.timedelta(days=1))], + id="update one card", + ), + pytest.param( + [("deleted", "Card to Delete", DEFAULT_DATE)], + [], + id="delete one card", + ), + pytest.param( + [("updated", "Card to Update", DEFAULT_DATE), ("deleted", "Card to Delete", DEFAULT_DATE)], + [ + ("created", "Created Card", DEFAULT_DATE), + ("updated", "Updated Card", DEFAULT_DATE + dt.timedelta(days=1)), + ], + id="create + update + delete", + ), + pytest.param( + [("existing", "Existing Card", DEFAULT_DATE)], + [("existing", "Existing Card", DEFAULT_DATE), ("created", "Created Card", DEFAULT_DATE)], + id="create one card while another card exists and is not modified", + ), + ], + ) + @freezegun.freeze_time(DEFAULT_DATE) + def test_bulk_sync_objects(self, django_settings, elasticsearch, example_drive_1, existing_cards, incoming_cards): + # arrange - set up database and elasticsearch according to `existing_cards` + source = factories.SourceFactory() + for (identifier, searchq, date_modified) in existing_cards: + factories.CardFactory( + identifier=identifier, + searchq=searchq, + date_created=make_aware(DEFAULT_DATE), + date_modified=make_aware(date_modified), + source=source, + ) + management.call_command("search_index", "--rebuild", "-f") + + # act + bulk_sync_objects( + source=source, + cards=[ + Card( + identifier=identifier, + searchq=searchq, + date_created=make_aware(DEFAULT_DATE), + date_modified=make_aware(date_modified), + source=source, + # not strictly relevant for this test, but values for these non-nullable fields are required. + size=0, + ) + for (identifier, searchq, date_modified) in incoming_cards + ], + ) + + # assert - database and elasticsearch should now match `incoming_cards` + assert {(card.identifier, card.searchq, make_naive(card.date_modified)) for card in Card.objects.all()} == set( + incoming_cards + ) + assert { + (result.identifier, result.searchq_keyword, make_naive(result.date_modified)) + for result in CardSearch().search().scan() + } == set(incoming_cards) + # endregion From 34203a118805b565ed338fc2f338927505447b8f Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sun, 18 May 2025 12:29:30 +1000 Subject: [PATCH 10/11] backend: change ProjectMember model back to foreign keying to Card --- ...ectmember_projectmember_unique_and_more.py | 33 +++++++++++++++++++ MPCAutofill/cardpicker/models.py | 25 ++++++++++---- 2 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 MPCAutofill/cardpicker/migrations/0042_remove_projectmember_projectmember_unique_and_more.py diff --git a/MPCAutofill/cardpicker/migrations/0042_remove_projectmember_projectmember_unique_and_more.py b/MPCAutofill/cardpicker/migrations/0042_remove_projectmember_projectmember_unique_and_more.py new file mode 100644 index 00000000..7a496525 --- /dev/null +++ b/MPCAutofill/cardpicker/migrations/0042_remove_projectmember_projectmember_unique_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.14 on 2025-05-18 02:22 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("cardpicker", "0041_card_date_modified"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="projectmember", + name="projectmember_unique", + ), + migrations.RemoveField( + model_name="projectmember", + name="card_id", + ), + migrations.AddField( + model_name="projectmember", + name="card", + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to="cardpicker.card" + ), + ), + migrations.AddConstraint( + model_name="projectmember", + constraint=models.UniqueConstraint(fields=("card", "project", "slot", "face"), name="projectmember_unique"), + ), + ] diff --git a/MPCAutofill/cardpicker/models.py b/MPCAutofill/cardpicker/models.py index 28787c95..dfc216b6 100755 --- a/MPCAutofill/cardpicker/models.py +++ b/MPCAutofill/cardpicker/models.py @@ -363,8 +363,18 @@ def set_project_members(self, records: dict[str, dict[str, list[dict[str, Any]]] if (card_identifier := record.get("card_identifier"), None) is not None: card_identifiers.add(card_identifier) + card_identifiers_to_pk: dict[str, Card] = { + x.identifier: x for x in Card.objects.filter(identifier__in=card_identifiers) + } members: list[ProjectMember] = [ - ProjectMember(card_id=value.get("card_identifier", None), slot=value["slot"], query=query, face=face) + ProjectMember( + card=card_identifiers_to_pk[card_identifier] + if (card_identifier := value.get("card_identifier", None)) is not None + else None, + slot=value["slot"], + query=query, + face=face, + ) for face in Faces if (face_members := records.get(face, None)) is not None for query, values in face_members.items() @@ -390,19 +400,22 @@ def __str__(self) -> str: class ProjectMember(models.Model): - card_id = models.CharField(max_length=200, null=True, blank=True) + card = models.ForeignKey(to=Card, on_delete=models.SET_NULL, null=True, blank=True) project = models.ForeignKey(to=Project, on_delete=models.CASCADE) query = models.CharField(max_length=200) slot = models.IntegerField() face = models.CharField(max_length=5, choices=Faces.choices, default=Faces.FRONT) class Meta: - constraints = [ - models.UniqueConstraint(fields=["card_id", "project", "slot", "face"], name="projectmember_unique") - ] + constraints = [models.UniqueConstraint(fields=["card", "project", "slot", "face"], name="projectmember_unique")] def to_dict(self) -> dict[str, Any]: - return {"card_identifier": self.card_id, "query": self.query, "slot": self.slot, "face": self.face} + return { + "card_identifier": self.card.identifier if self.card else None, + "query": self.query, + "slot": self.slot, + "face": self.face, + } __all__ = [ From 8278b6510be04e007a0d23ed844bc684d8d4dfa9 Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sun, 18 May 2025 14:02:53 +1000 Subject: [PATCH 11/11] fixup --- frontend/src/features/ui/DynamicLogo.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/features/ui/DynamicLogo.tsx b/frontend/src/features/ui/DynamicLogo.tsx index c5ce0c47..7020f5f9 100644 --- a/frontend/src/features/ui/DynamicLogo.tsx +++ b/frontend/src/features/ui/DynamicLogo.tsx @@ -221,6 +221,7 @@ const SampleCardDocument: CardDocument = { searchq: "", extension: "png", dateCreated: "1st January, 2000", + dateModified: "1st January, 2000", downloadLink: "", size: 1, smallThumbnailUrl: "/logo-blank.png",