From d79ebd2053701dceda9cf1c2afb998f21c18bb68 Mon Sep 17 00:00:00 2001 From: Nicholas de Paola Date: Sun, 29 Dec 2024 13:53:34 +1000 Subject: [PATCH] Update Moxfield API integration according to new authentication method Resolves #265 --- .github/actions/test-backend/action.yml | 5 ++ .github/workflows/test-backend.yml | 1 + .github/workflows/test-desktop-tool.yml | 1 + .github/workflows/tests.yml | 2 + MPCAutofill/MPCAutofill/.env.dist | 2 + MPCAutofill/MPCAutofill/settings.py | 4 ++ .../cardpicker/integrations/game/mtg.py | 11 ++++- .../cardpicker/tests/test_integrations.py | 48 ++++++++++++++++++- MPCAutofill/requirements.txt | 1 + 9 files changed, 72 insertions(+), 3 deletions(-) diff --git a/.github/actions/test-backend/action.yml b/.github/actions/test-backend/action.yml index 8a8cf2d5e..6475d8bf0 100644 --- a/.github/actions/test-backend/action.yml +++ b/.github/actions/test-backend/action.yml @@ -4,6 +4,9 @@ inputs: google-drive-api-key: description: Your Google Drive API key, required for running the database crawler required: true + moxfield-secret: + description: Your Moxfield API secret, required for pulling data from Moxfield + required: true runs: using: composite steps: @@ -27,3 +30,5 @@ runs: run: | cd MPCAutofill && pytest . shell: bash + env: + MOXFIELD_SECRET: ${{ inputs.moxfield-secret }} diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index d58054dc2..fc37d1189 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -27,3 +27,4 @@ jobs: - uses: ./.github/actions/test-backend with: google-drive-api-key: ${{ secrets.GOOGLE_DRIVE_API_KEY }} + moxfield-secret: ${{ secrets.MOXFIELD_SECRET }} diff --git a/.github/workflows/test-desktop-tool.yml b/.github/workflows/test-desktop-tool.yml index 6b73ee64f..466274a59 100644 --- a/.github/workflows/test-desktop-tool.yml +++ b/.github/workflows/test-desktop-tool.yml @@ -19,3 +19,4 @@ jobs: - uses: ./.github/actions/test-desktop-tool with: google-drive-api-key: ${{ secrets.GOOGLE_DRIVE_API_KEY }} + moxfield-secret: ${{ secrets.MOXFIELD_SECRET }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c6892a1ed..0a2bde350 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,6 +26,7 @@ jobs: - uses: ./.github/actions/test-backend with: google-drive-api-key: ${{ secrets.GOOGLE_DRIVE_API_KEY }} + moxfield-secret: ${{ secrets.MOXFIELD_SECRET }} test-desktop-tool: name: Desktop tool tests runs-on: ${{ matrix.os }} @@ -40,6 +41,7 @@ jobs: - uses: ./.github/actions/test-desktop-tool with: google-drive-api-key: ${{ secrets.GOOGLE_DRIVE_API_KEY }} + moxfield-secret: ${{ secrets.MOXFIELD_SECRET }} test-frontend: name: Frontend tests runs-on: ubuntu-latest diff --git a/MPCAutofill/MPCAutofill/.env.dist b/MPCAutofill/MPCAutofill/.env.dist index 74024141e..aef4229d7 100644 --- a/MPCAutofill/MPCAutofill/.env.dist +++ b/MPCAutofill/MPCAutofill/.env.dist @@ -45,3 +45,5 @@ PATREON_REFRESH= PATREON_CLIENT= PATREON_SECRET= PATREON_URL= + +MOXFIELD_SECRET = diff --git a/MPCAutofill/MPCAutofill/settings.py b/MPCAutofill/MPCAutofill/settings.py index 37c49b9b9..a8f232c9b 100755 --- a/MPCAutofill/MPCAutofill/settings.py +++ b/MPCAutofill/MPCAutofill/settings.py @@ -12,6 +12,7 @@ import os import sys +from typing import Optional import django_stubs_ext import environ @@ -48,6 +49,9 @@ THEME = env("THEME", default="superhero") DESCRIPTION = env("DESCRIPTION", default="") +# Integration secrets +MOXFIELD_SECRET: Optional[str] = env("MOXFIELD_SECRET", default=None) + PREPEND_WWW = env("PREPEND_WWW", default=False) # Quick-start development settings - unsuitable for production diff --git a/MPCAutofill/cardpicker/integrations/game/mtg.py b/MPCAutofill/cardpicker/integrations/game/mtg.py index c97cd5dad..4d91db7e0 100644 --- a/MPCAutofill/cardpicker/integrations/game/mtg.py +++ b/MPCAutofill/cardpicker/integrations/game/mtg.py @@ -3,6 +3,10 @@ from typing import Any, Type from urllib.parse import parse_qs, urlparse +import ratelimit + +from django.conf import settings + from cardpicker.integrations.game.base import GameIntegration, ImportSite from cardpicker.models import DFCPair from cardpicker.utils import get_json_endpoint_rate_limited @@ -145,12 +149,15 @@ class Moxfield(ImportSite): def get_host_names() -> list[str]: return ["www.moxfield.com", "moxfield.com"] # moxfield prefers www. + # Note: requests to the Moxfield API must be rate limited to one request per second. @classmethod + @ratelimit.sleep_and_retry # type: ignore # `ratelimit` does not implement decorator typing correctly + @ratelimit.limits(calls=1, period=1) # type: ignore # `ratelimit` does not implement decorator typing correctly def retrieve_card_list(cls, url: str) -> str: path = urlparse(url).path deck_id = path.split("/")[-1] response = cls.request( - path=f"v2/decks/all/{deck_id}", netloc="api.moxfield.com", headers={"x-requested-by": "mpcautofill"} + path=f"v2/decks/all/{deck_id}", netloc="api.moxfield.com", headers={"User-Agent": settings.MOXFIELD_SECRET} ) response_json = response.json() card_list = "" @@ -319,7 +326,7 @@ def get_import_sites(cls) -> list[Type[ImportSite]]: Deckstats, MagicVille, ManaStack, - Moxfield, + *([Moxfield] if settings.MOXFIELD_SECRET else []), MTGGoldfish, Scryfall, TappedOut, diff --git a/MPCAutofill/cardpicker/tests/test_integrations.py b/MPCAutofill/cardpicker/tests/test_integrations.py index 849346779..5ffaa3de8 100644 --- a/MPCAutofill/cardpicker/tests/test_integrations.py +++ b/MPCAutofill/cardpicker/tests/test_integrations.py @@ -1,9 +1,15 @@ +import time from collections import Counter +from concurrent.futures import ThreadPoolExecutor from enum import Enum +from typing import Optional import pytest +import requests_mock -from cardpicker.integrations.game.mtg import MTG +from django.conf import settings as conf_settings + +from cardpicker.integrations.game.mtg import MTG, Moxfield from cardpicker.integrations.integrations import get_configured_game_integration @@ -44,6 +50,19 @@ class Decks(Enum): # endregion + # region fixtures + + @pytest.fixture() + def moxfield_secret_setter(self): + def _callable(moxfield_secret: Optional[str]): + conf_settings.MOXFIELD_SECRET = moxfield_secret + + old_secret = conf_settings.MOXFIELD_SECRET + yield _callable + conf_settings.MOXFIELD_SECRET = old_secret + + # endregion + # region tests def test_get_double_faced_card_pairs(self): @@ -58,4 +77,31 @@ def test_valid_url(self, client, django_settings, snapshot, url: str): assert decklist assert Counter(decklist.splitlines()) == snapshot + @pytest.mark.parametrize( + "moxfield_secret, is_moxfield_enabled", + [ + (None, False), + ("", False), + ("lorem ipsum", True), + ], + ) + def test_moxfield_enabled(self, moxfield_secret_setter, moxfield_secret, is_moxfield_enabled): + moxfield_secret_setter(moxfield_secret) + import_sites = MTG.get_import_sites() + if is_moxfield_enabled: + assert Moxfield in import_sites + else: + assert Moxfield not in import_sites + + def test_moxfield_rate_limit(self, monkeypatch): + with requests_mock.Mocker() as mock: + mock.get("https://api.moxfield.com/v2/decks/all/D42-or9pCk-uMi4XzRDziQ", json={}) + + with ThreadPoolExecutor(max_workers=3) as pool: + t0 = time.time() + pool.map(lambda _: [Moxfield.retrieve_card_list(self.Decks.MOXFIELD.value) for _ in range(2)], range(3)) + t1 = time.time() + t = t1 - t0 + assert t > 5 # one second between calls + # endregion diff --git a/MPCAutofill/requirements.txt b/MPCAutofill/requirements.txt index 10dc93737..344b2052b 100644 --- a/MPCAutofill/requirements.txt +++ b/MPCAutofill/requirements.txt @@ -26,6 +26,7 @@ pytest-elasticsearch~=3.0 python-dotenv~=1.0.0 ratelimit~=2.2.1 requests~=2.31.0 +requests-mock~=1.12.1 sentry-sdk~=1.30.0 syrupy~=3.0 testcontainers[postgres,elasticsearch]