diff --git a/pyproject.toml b/pyproject.toml index a4cc37b..676090f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,16 +6,18 @@ readme = "README.md" requires-python = ">=3.12" dependencies = [ "beautifulsoup4>=4.13.3", + "celery>=5.5.2", "django>=5.1.7", "django-tailwind[reload]>=3.8.0", "gunicorn>=23.0.0", "httpx>=0.28.1", "llm>=0.24.2", "llm-gemini>=0.18.1", - "logfire[django,psycopg,system-metrics]>=3.16.0", + "logfire[celery,django,psycopg,system-metrics]>=3.16.0", "lxml>=5.3.2", "markdown>=3.7", "psycopg[binary]>=3.2.5", + "pydantic-ai-slim[anthropic,openai]>=0.2.6", "pydantic-ai[logfire]>=0.2.6", "readtime>=3.0.0", "requests>=2.32.3", diff --git a/pythonkr_backend/pythonkr_backend/__init__.py b/pythonkr_backend/pythonkr_backend/__init__.py index e69de29..805a42b 100644 --- a/pythonkr_backend/pythonkr_backend/__init__.py +++ b/pythonkr_backend/pythonkr_backend/__init__.py @@ -0,0 +1,9 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from django.conf import settings + +if settings.CELERY_ALWAYS_EAGER is False: + from .celery import app as celery_app + __all__ = ('celery_app',) +else: + __all__ = () \ No newline at end of file diff --git a/pythonkr_backend/pythonkr_backend/celery.py b/pythonkr_backend/pythonkr_backend/celery.py new file mode 100644 index 0000000..e89cb52 --- /dev/null +++ b/pythonkr_backend/pythonkr_backend/celery.py @@ -0,0 +1,41 @@ +import logfire +from celery import Celery +from celery.signals import worker_init, beat_init + +app = Celery('proj') + + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@worker_init.connect() +def init_worker(*args, **kwargs): + logfire.configure(service_name="celery-worker", send_to_logfire='if-token-present') + logfire.instrument_celery() + +@beat_init.connect() +def init_beat(*args, **kwargs): + logfire.configure(service_name="celery-beat", send_to_logfire='if-token-present') + logfire.instrument_celery() + +@app.task +def add(x: int, y: int): + return x + y + +add.delay(42, 50) + + +app.conf.beat_schedule = { + "add-every-30-seconds": { + "task": "tasks.add", + "schedule": 30.0, + "args": (16, 16), + }, +} \ No newline at end of file diff --git a/pythonkr_backend/pythonkr_backend/settings/base.py b/pythonkr_backend/pythonkr_backend/settings/base.py index 0183eeb..0101fbd 100644 --- a/pythonkr_backend/pythonkr_backend/settings/base.py +++ b/pythonkr_backend/pythonkr_backend/settings/base.py @@ -187,23 +187,10 @@ "0.0.0.0", ] -# LOGGING - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "handlers": { - "console": { - "class": "logging.StreamHandler", - }, - }, - "root": { - "handlers": ["console"], - "level": "INFO", - }, -} - # import logfire # setup logfire #logfire.configure(environment='base', service_name="web") -#logfire.instrument_django() \ No newline at end of file +#logfire.instrument_django() + +# testing +CELERY_ALWAYS_EAGER = True \ No newline at end of file diff --git a/pythonkr_backend/pythonkr_backend/settings/localtesting.py b/pythonkr_backend/pythonkr_backend/settings/localtesting.py index f1d140e..24e3fe2 100644 --- a/pythonkr_backend/pythonkr_backend/settings/localtesting.py +++ b/pythonkr_backend/pythonkr_backend/settings/localtesting.py @@ -29,7 +29,15 @@ BAKERY_MULTISITE = True BUILD_DIR = os.path.join("/app/bakery_static", "build") + +# check WSGI environment +IS_PRODUCTION_SERVER = os.environ.get('IS_WSGI_ENVIRONMENT', 'False') == 'True' + + # logfire settings if IS_PRODUCTION_SERVER: logfire.configure(environment='localtest') - logfire.instrument_django() \ No newline at end of file + logfire.instrument_django() + +# testing +CELERY_ALWAYS_EAGER = True \ No newline at end of file diff --git a/pythonkr_backend/pythonkr_backend/settings/prod.py b/pythonkr_backend/pythonkr_backend/settings/prod.py index 960f37f..91e2c47 100644 --- a/pythonkr_backend/pythonkr_backend/settings/prod.py +++ b/pythonkr_backend/pythonkr_backend/settings/prod.py @@ -66,4 +66,15 @@ logfire.configure(environment='prod', service_name="web", service_version=sha_service_version) logfire.instrument_django() logfire.instrument_system_metrics() -#logfire.instrument_psycopg('psycopg') \ No newline at end of file +#logfire.instrument_psycopg('psycopg') + +# celery +CELERY_BROKER_PASSWORD = os.environ.get("CELERY_PASSWORD","FALSE") +CELERY_BROKER_USERNAME = os.environ.get("CELERY_USERNAME","FALSE") +CELERY_BROKER_VHOST = os.environ.get("CELERY_VHOST","FALSE") +# Celery Configuration Options +CELERY_ALWAYS_EAGER = False +CELERY_TIMEZONE = "Asia/Seoul" +CELERY_TASK_TRACK_STARTED = True +CELERY_BROKER_URL = f"amqp://{CELERY_BROKER_USERNAME}:{CELERY_BROKER_PASSWORD}@localhost:5672/{CELERY_BROKER_VHOST}" +CELERY_TASK_TIME_LIMIT = 30 * 60 \ No newline at end of file diff --git a/uv.lock b/uv.lock index 18ca061..ed0cfbb 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,18 @@ version = 1 revision = 2 requires-python = ">=3.12" +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -92,6 +104,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/cd/30110dc0ffcf3b131156077b90e9f60ed75711223f306da4db08eff8403b/beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b", size = 187285, upload-time = "2025-04-15T17:05:12.221Z" }, ] +[[package]] +name = "billiard" +version = "4.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/58/1546c970afcd2a2428b1bfafecf2371d8951cc34b46701bea73f4280989e/billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f", size = 155031, upload-time = "2024-09-21T13:40:22.491Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/da/43b15f28fe5f9e027b41c539abc5469052e9d48fd75f8ff094ba2a0ae767/billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb", size = 86766, upload-time = "2024-09-21T13:40:20.188Z" }, +] + [[package]] name = "boto3" version = "1.38.21" @@ -129,6 +150,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, ] +[[package]] +name = "celery" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/03/5d9c6c449248958f1a5870e633a29d7419ff3724c452a98ffd22688a1a6a/celery-5.5.2.tar.gz", hash = "sha256:4d6930f354f9d29295425d7a37261245c74a32807c45d764bedc286afd0e724e", size = 1666892, upload-time = "2025-04-25T20:10:04.695Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/94/8e825ac1cf59d45d20c4345d4461e6b5263ae475f708d047c3dad0ac6401/celery-5.5.2-py3-none-any.whl", hash = "sha256:54425a067afdc88b57cd8d94ed4af2ffaf13ab8c7680041ac2c4ac44357bdf4c", size = 438626, upload-time = "2025-04-25T20:10:01.383Z" }, +] + [[package]] name = "certifi" version = "2025.4.26" @@ -197,6 +237,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/1a/aff8bb287a4b1400f69e09a53bd65de96aa5cee5691925b38731c67fc695/click_default_group-1.2.4-py2.py3-none-any.whl", hash = "sha256:9b60486923720e7fc61731bdb32b617039aba820e22e1c88766b1125592eaa5f", size = 4123, upload-time = "2023-08-04T07:54:56.875Z" }, ] +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/1d/45434f64ed749540af821fd7e42b8e4d23ac04b1eda7c26613288d6cd8a8/click-plugins-1.1.1.tar.gz", hash = "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", size = 8164, upload-time = "2019-04-04T04:27:04.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/da/824b92d9942f4e472702488857914bdd50f73021efea15b4cad9aca8ecef/click_plugins-1.1.1-py2.py3-none-any.whl", hash = "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8", size = 7497, upload-time = "2019-04-04T04:27:03.36Z" }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, +] + [[package]] name = "cohere" version = "5.15.0" @@ -824,6 +901,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, ] +[[package]] +name = "kombu" +version = "5.5.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/0a/128b65651ed8120460fc5af754241ad595eac74993115ec0de4f2d7bc459/kombu-5.5.3.tar.gz", hash = "sha256:021a0e11fcfcd9b0260ef1fb64088c0e92beb976eb59c1dfca7ddd4ad4562ea2", size = 461784, upload-time = "2025-04-16T12:46:17.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/35/1407fb0b2f5b07b50cbaf97fce09ad87d3bfefbf64f7171a8651cd8d2f68/kombu-5.5.3-py3-none-any.whl", hash = "sha256:5b0dbceb4edee50aa464f59469d34b97864be09111338cfb224a10b6a163909b", size = 209921, upload-time = "2025-04-16T12:46:15.139Z" }, +] + [[package]] name = "laces" version = "0.1.2" @@ -894,6 +985,9 @@ wheels = [ ] [package.optional-dependencies] +celery = [ + { name = "opentelemetry-instrumentation-celery" }, +] django = [ { name = "opentelemetry-instrumentation-django" }, ] @@ -1120,6 +1214,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/89/0790abc5d9c4fc74bd3e03cb87afe2c820b1d1a112a723c1163ef32453ee/opentelemetry_instrumentation-0.54b1-py3-none-any.whl", hash = "sha256:a4ae45f4a90c78d7006c51524f57cd5aa1231aef031eae905ee34d5423f5b198", size = 31019, upload-time = "2025-05-16T19:02:15.611Z" }, ] +[[package]] +name = "opentelemetry-instrumentation-celery" +version = "0.54b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/71/4ac353874e0f7ca93591e1a74b7a290dec2027733bbb31bd76da3a74f97f/opentelemetry_instrumentation_celery-0.54b1.tar.gz", hash = "sha256:f2bd019afe9286214083ae2db95ed24adf9a0aa2e943177462d64ceb8380d78e", size = 14778, upload-time = "2025-05-16T19:03:37.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/be/90e2b7d26915639cfcdf6e200b309c9d64027ff752c56145bc149cd67d68/opentelemetry_instrumentation_celery-0.54b1-py3-none-any.whl", hash = "sha256:892ec6bf829a0d60cf3bffd1a8bb6fd8055f1194167b4e132e33321de8e05c24", size = 13809, upload-time = "2025-05-16T19:02:33.046Z" }, +] + [[package]] name = "opentelemetry-instrumentation-dbapi" version = "0.54b1" @@ -1726,17 +1834,19 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, + { name = "celery" }, { name = "django" }, { name = "django-tailwind", extra = ["reload"] }, { name = "gunicorn" }, { name = "httpx" }, { name = "llm" }, { name = "llm-gemini" }, - { name = "logfire", extra = ["django", "psycopg", "system-metrics"] }, + { name = "logfire", extra = ["celery", "django", "psycopg", "system-metrics"] }, { name = "lxml" }, { name = "markdown" }, { name = "psycopg", extra = ["binary"] }, { name = "pydantic-ai", extra = ["logfire"] }, + { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, { name = "readtime" }, { name = "requests" }, { name = "tiktoken" }, @@ -1753,17 +1863,19 @@ dev = [ [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.13.3" }, + { name = "celery", specifier = ">=5.5.2" }, { name = "django", specifier = ">=5.1.7" }, { name = "django-tailwind", extras = ["reload"], specifier = ">=3.8.0" }, { name = "gunicorn", specifier = ">=23.0.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "llm", specifier = ">=0.24.2" }, { name = "llm-gemini", specifier = ">=0.18.1" }, - { name = "logfire", extras = ["django", "psycopg", "system-metrics"], specifier = ">=3.16.0" }, + { name = "logfire", extras = ["celery", "django", "psycopg", "system-metrics"], specifier = ">=3.16.0" }, { name = "lxml", specifier = ">=5.3.2" }, { name = "markdown", specifier = ">=3.7" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2.5" }, { name = "pydantic-ai", extras = ["logfire"], specifier = ">=0.2.6" }, + { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], specifier = ">=0.2.6" }, { name = "readtime", specifier = ">=3.0.0" }, { name = "requests", specifier = ">=2.32.3" }, { name = "tiktoken", specifier = ">=0.9.0" }, @@ -2189,6 +2301,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/4b/4cef6ce21a2aaca9d852a6e84ef4f135d99fcd74fa75105e2fc0c8308acd/uvicorn-0.34.2-py3-none-any.whl", hash = "sha256:deb49af569084536d269fe0a6d67e3754f104cf03aba7c11c40f01aadf33c403", size = 62483, upload-time = "2025-04-19T06:02:48.42Z" }, ] +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, +] + [[package]] name = "wagtail" version = "7.0"