From 5e10a6a4b5883839856c0b6dd34c23267685017a Mon Sep 17 00:00:00 2001 From: "jose.padin" Date: Wed, 4 May 2022 19:05:01 +0200 Subject: [PATCH 01/29] [#305] Added abstract models. `models` module replaced by a `models` package containing an `abstract` module (for abstract models) and a `generic` module (for the default models, previously in the `models` module) --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/models/__init__.py | 3 + .../{models.py => models/abstract.py} | 59 ++------------- django_celery_results/models/generic.py | 73 +++++++++++++++++++ 3 files changed, 83 insertions(+), 52 deletions(-) create mode 100644 django_celery_results/models/__init__.py rename django_celery_results/{models.py => models/abstract.py} (83%) create mode 100644 django_celery_results/models/generic.py diff --git a/django_celery_results/models/__init__.py b/django_celery_results/models/__init__.py new file mode 100644 index 00000000..4165e265 --- /dev/null +++ b/django_celery_results/models/__init__.py @@ -0,0 +1,3 @@ +from .generic import ChordCounter, GroupResult, TaskResult + +__ALL__ = [ChordCounter, GroupResult, TaskResult] diff --git a/django_celery_results/models.py b/django_celery_results/models/abstract.py similarity index 83% rename from django_celery_results/models.py rename to django_celery_results/models/abstract.py index b472a5af..f3dc1662 100644 --- a/django_celery_results/models.py +++ b/django_celery_results/models/abstract.py @@ -1,10 +1,6 @@ -"""Database models.""" - -import json +"""Abstract models.""" from celery import states -from celery.result import GroupResult as CeleryGroupResult -from celery.result import result_from_tuple from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ @@ -15,8 +11,8 @@ TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) -class TaskResult(models.Model): - """Task result/status.""" +class AbstractTaskResult(models.Model): + """Abstract Task result/status.""" task_id = models.CharField( max_length=getattr( @@ -97,8 +93,8 @@ class TaskResult(models.Model): class Meta: """Table information.""" + abstract = True ordering = ['-date_done'] - verbose_name = _('task result') verbose_name_plural = _('task results') @@ -136,49 +132,8 @@ def __str__(self): return ''.format(self) -class ChordCounter(models.Model): - """Chord synchronisation.""" - - group_id = models.CharField( - max_length=getattr( - settings, - "DJANGO_CELERY_RESULTS_TASK_ID_MAX_LENGTH", - 255), - unique=True, - verbose_name=_("Group ID"), - help_text=_("Celery ID for the Chord header group"), - ) - sub_tasks = models.TextField( - help_text=_( - "JSON serialized list of task result tuples. " - "use .group_result() to decode" - ) - ) - count = models.PositiveIntegerField( - help_text=_( - "Starts at len(chord header) and decrements after each task is " - "finished" - ) - ) - - def group_result(self, app=None): - """Return the :class:`celery.result.GroupResult` of self. - - Arguments: - app (celery.app.base.Celery): app instance to create the - :class:`celery.result.GroupResult` with. - - """ - return CeleryGroupResult( - self.group_id, - [result_from_tuple(r, app=app) - for r in json.loads(self.sub_tasks)], - app=app - ) - - -class GroupResult(models.Model): - """Task Group result/status.""" +class AbstractGroupResult(models.Model): + """Abstract Task Group result/status.""" group_id = models.CharField( max_length=getattr( @@ -231,8 +186,8 @@ def __str__(self): class Meta: """Table information.""" + abstract = True ordering = ['-date_done'] - verbose_name = _('group result') verbose_name_plural = _('group results') diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py new file mode 100644 index 00000000..74faf614 --- /dev/null +++ b/django_celery_results/models/generic.py @@ -0,0 +1,73 @@ +"""Database models.""" + +import json + +from celery.result import GroupResult as CeleryGroupResult +from celery.result import result_from_tuple +from django.conf import settings +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_celery_results.models.abstract import ( + AbstractGroupResult, + AbstractTaskResult +) + + +class TaskResult(AbstractTaskResult): + """Task result/status.""" + + class Meta(AbstractTaskResult.Meta): + """Table information.""" + + abstract = False + + +class ChordCounter(models.Model): + """Chord synchronisation.""" + + group_id = models.CharField( + max_length=getattr( + settings, + "DJANGO_CELERY_RESULTS_TASK_ID_MAX_LENGTH", + 255), + unique=True, + verbose_name=_("Group ID"), + help_text=_("Celery ID for the Chord header group"), + ) + sub_tasks = models.TextField( + help_text=_( + "JSON serialized list of task result tuples. " + "use .group_result() to decode" + ) + ) + count = models.PositiveIntegerField( + help_text=_( + "Starts at len(chord header) and decrements after each task is " + "finished" + ) + ) + + def group_result(self, app=None): + """Return the GroupResult of self. + + Arguments: + --------- + app (Celery): app instance to create the GroupResult with. + + """ + return CeleryGroupResult( + self.group_id, + [result_from_tuple(r, app=app) + for r in json.loads(self.sub_tasks)], + app=app + ) + + +class GroupResult(AbstractGroupResult): + """Task Group result/status.""" + + class Meta(AbstractGroupResult.Meta): + """Table information.""" + + abstract = False From 1b1facb43c21fb671361316f95873a5f47b31ad2 Mon Sep 17 00:00:00 2001 From: "jose.padin" Date: Wed, 4 May 2022 18:20:21 +0200 Subject: [PATCH 02/29] [#305] `ChordCounter` moved from `abstract` module to `generic` module. Added some minor changes. --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/models/abstract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index f3dc1662..599fc60f 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -187,9 +187,9 @@ class Meta: """Table information.""" abstract = True - ordering = ['-date_done'] verbose_name = _('group result') verbose_name_plural = _('group results') + ordering = ['-date_done'] # Explicit names to solve https://code.djangoproject.com/ticket/33483 indexes = [ From b597c1c19848d70feb8cd861b55694ab2a77f040 Mon Sep 17 00:00:00 2001 From: "jose.padin" Date: Thu, 12 May 2022 18:38:25 +0200 Subject: [PATCH 03/29] Issue 305: abstract models * Fixed import bug --- django_celery_results/models/generic.py | 2 +- django_celery_results/{ => models}/managers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename django_celery_results/{ => models}/managers.py (99%) diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index 74faf614..09b9dfdb 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -10,7 +10,7 @@ from django_celery_results.models.abstract import ( AbstractGroupResult, - AbstractTaskResult + AbstractTaskResult, ) diff --git a/django_celery_results/managers.py b/django_celery_results/models/managers.py similarity index 99% rename from django_celery_results/managers.py rename to django_celery_results/models/managers.py index 1caa124b..812374f1 100644 --- a/django_celery_results/managers.py +++ b/django_celery_results/models/managers.py @@ -8,7 +8,7 @@ from django.conf import settings from django.db import connections, models, router, transaction -from .utils import now +from ..utils import now W_ISOLATION_REP = """ Polling results with transaction isolation level 'repeatable-read' From 1311f5ee8131e372101337d2072a2ac2f0ed50f5 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 7 Jun 2022 16:44:30 +0600 Subject: [PATCH 04/29] Update django_celery_results/models/__init__.py Co-authored-by: Allex --- django_celery_results/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_celery_results/models/__init__.py b/django_celery_results/models/__init__.py index 4165e265..5e478755 100644 --- a/django_celery_results/models/__init__.py +++ b/django_celery_results/models/__init__.py @@ -1,3 +1,3 @@ from .generic import ChordCounter, GroupResult, TaskResult -__ALL__ = [ChordCounter, GroupResult, TaskResult] +__all__ = ["ChordCounter", "GroupResult", "TaskResult"] From ab74d4daaad36c2df128bb5f4a5f1486ff0f8dfb Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Wed, 10 Aug 2022 08:44:30 +0200 Subject: [PATCH 05/29] [#305]: Improving abstract models implementation. Added a `helpers` module into `models` containing the functions `taskresult_model()` and `groupresult_model()`. * `taskresult_model()`: will try to find the custom model using a dotted path defined under the constant `CELERY_RESULTS_TASKRESULT_MODEL` in the settings of the user's project * `groupresult_model()` will try to do the same using under the constant `CELERY_RESULTS_GROUPRESULT_MODEL`. By default if these attributes are not found `django-celery-results` will use the default models (`models.TaskResult` and `models.GroupResult`). Updated database backend in order to use custom models for `TaskResult and `GroupResult` it they're present. Instead to import explicitely the `TaskResult` and the `GroupResult` (default models from `django-celery-results`) we make use of the model helpers to load the right classes, the custom ones if they're present otherwise we use the default ones. Getting data from `task_kwargs` to extend the `task_properties` and be able to store them into the database (using the custom models). First of all we need a way to get data from `task_kwargs` (or somewhere else) just before a `task_result` record is created, evaluate that data and find the right values that will be used to fill the new fields defined in the custom model. So for this purpose we added a settings module to `django-celery-results` which will hold default settings, the first setting that will contain is a function in charge to get a callback from the settings of the user project. This callback will be feeded by the task `task_kwargs`, which will be intercepted in `DatabaseBackend._get_extended_properties` just before the `task_kwargs` are encoded by `encode_content()` method and send it to the `store_result` method from the object manager of `TaskModel` (Custom/Default one). To end, we must to extend the arguments of the `store_result` method from the `TaskResult` Manager adding `extra_fields` argument that will make us able to send extra data to the custom model, when it's defined. --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- .gitignore | 3 +- django_celery_results/admin.py | 5 +- django_celery_results/backends/database.py | 15 ++++-- .../{models => }/managers.py | 8 ++-- django_celery_results/models/abstract.py | 2 +- django_celery_results/models/generic.py | 6 +++ django_celery_results/models/helpers.py | 47 +++++++++++++++++++ django_celery_results/settings.py | 16 +++++++ 8 files changed, 92 insertions(+), 10 deletions(-) rename django_celery_results/{models => }/managers.py (97%) create mode 100644 django_celery_results/models/helpers.py create mode 100644 django_celery_results/settings.py diff --git a/.gitignore b/.gitignore index 1eb8fa24..cb8ce423 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,5 @@ cover/ htmlcov/ coverage.xml .env -*.ignore \ No newline at end of file +*.ignore +.vscode diff --git a/django_celery_results/admin.py b/django_celery_results/admin.py index de5172af..e0654aec 100644 --- a/django_celery_results/admin.py +++ b/django_celery_results/admin.py @@ -10,7 +10,10 @@ ALLOW_EDITS = False pass -from .models import GroupResult, TaskResult +from .models.helpers import taskresult_model, groupresult_model + +GroupResult = groupresult_model() +TaskResult = taskresult_model() class TaskResultAdmin(admin.ModelAdmin): diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index a4e364a6..aa9acbb5 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -13,8 +13,8 @@ from kombu.exceptions import DecodeError from ..models import ChordCounter -from ..models import GroupResult as GroupResultModel -from ..models import TaskResult +from ..models.helpers import groupresult_model, taskresult_model +from ..settings import extend_task_props_callback EXCEPTIONS_TO_CATCH = (InterfaceError,) @@ -30,8 +30,8 @@ class DatabaseBackend(BaseDictBackend): """The Django database backend, using models to store task state.""" - TaskModel = TaskResult - GroupModel = GroupResultModel + TaskModel = taskresult_model() + GroupModel = groupresult_model() subpolling_interval = 0.5 def exception_safe_to_retry(self, exc): @@ -80,6 +80,13 @@ def _get_extended_properties(self, request, traceback): # task protocol 1 task_kwargs = getattr(request, 'kwargs', None) + # TODO: We assuming that task protocol 1 could be always in use. :/ + extra_fields = extend_task_props_callback( + getattr(request, 'kwargs', None) + ) + if extra_fields: + extended_props.update({"extra_fields": extra_fields}) + # Encode input arguments if task_args is not None: _, _, task_args = self.encode_content(task_args) diff --git a/django_celery_results/models/managers.py b/django_celery_results/managers.py similarity index 97% rename from django_celery_results/models/managers.py rename to django_celery_results/managers.py index 812374f1..b38a15e8 100644 --- a/django_celery_results/models/managers.py +++ b/django_celery_results/managers.py @@ -8,7 +8,7 @@ from django.conf import settings from django.db import connections, models, router, transaction -from ..utils import now +from .utils import now W_ISOLATION_REP = """ Polling results with transaction isolation level 'repeatable-read' @@ -119,7 +119,7 @@ def store_result(self, content_type, content_encoding, traceback=None, meta=None, periodic_task_name=None, task_name=None, task_args=None, task_kwargs=None, - worker=None, using=None, **kwargs): + worker=None, using=None, extra_fields=None, **kwargs): """Store the result and status of a task. Arguments: @@ -140,6 +140,7 @@ def store_result(self, content_type, content_encoding, exception (only passed if the task failed). meta (str): Serialized result meta data (this contains e.g. children). + extra_fields (dict, optional): Extra (model) fields to store. Keyword Arguments: exception_retry_count (int): How many times to retry by @@ -159,7 +160,8 @@ def store_result(self, content_type, content_encoding, 'task_name': task_name, 'task_args': task_args, 'task_kwargs': task_kwargs, - 'worker': worker + 'worker': worker, + **extra_fields } if 'date_started' in kwargs: fields['date_started'] = kwargs['date_started'] diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index 599fc60f..21973989 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -5,7 +5,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ -from . import managers +from .. import managers ALL_STATES = sorted(states.ALL_STATES) TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index 09b9dfdb..f1ec7e01 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -21,6 +21,7 @@ class Meta(AbstractTaskResult.Meta): """Table information.""" abstract = False + app_label = "django_celery_results" class ChordCounter(models.Model): @@ -48,6 +49,10 @@ class ChordCounter(models.Model): ) ) + class Meta: + app_label = "django_celery_results" + + def group_result(self, app=None): """Return the GroupResult of self. @@ -71,3 +76,4 @@ class Meta(AbstractGroupResult.Meta): """Table information.""" abstract = False + app_label = "django_celery_results" diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py new file mode 100644 index 00000000..668ea244 --- /dev/null +++ b/django_celery_results/models/helpers.py @@ -0,0 +1,47 @@ +from django.apps import apps +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured + +from .generic import TaskResult, GroupResult + +def taskresult_model(): + """Return the TaskResult model that is active in this project.""" + if not hasattr(settings, 'CELERY_RESULTS_TASKRESULT_MODEL'): + return TaskResult + + try: + return apps.get_model( + settings.CELERY_RESULTS_TASKRESULT_MODEL + ) + except ValueError: + raise ImproperlyConfigured( + "CELERY_RESULTS_TASKRESULT_MODEL must be of the form " + "'app_label.model_name'" + ) + except LookupError: + raise ImproperlyConfigured( + "CELERY_RESULTS_TASKRESULT_MODEL refers to model " + f"'{settings.CELERY_RESULTS_TASKRESULT_MODEL}' that has not " + "been installed" + ) + +def groupresult_model(): + """Return the GroupResult model that is active in this project.""" + if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): + return GroupResult + + try: + return apps.get_model( + settings.CELERY_RESULTS_GROUPRESULT_MODEL + ) + except ValueError: + raise ImproperlyConfigured( + "CELERY_RESULTS_GROUPRESULT_MODEL must be of the form " + "'app_label.model_name'" + ) + except LookupError: + raise ImproperlyConfigured( + "CELERY_RESULTS_GROUPRESULT_MODEL refers to model " + f"'{settings.CELERY_RESULTS_GROUPRESULT_MODEL}' that has not " + "been installed" + ) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py new file mode 100644 index 00000000..5f4eb61d --- /dev/null +++ b/django_celery_results/settings.py @@ -0,0 +1,16 @@ +from django.conf import settings + + +def get_callback_function(settings_name, default=None): + """Return the callback function for the given settings name.""" + + callback = getattr(settings, settings_name, None) + if callback is None: + return default + + if callable(callback): + return callback + +extend_task_props_callback = get_callback_function( + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK" +) \ No newline at end of file From 8c83433df4cb3f5d3a035ff7f9b3381f891ae98d Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Wed, 10 Aug 2022 21:27:41 +0200 Subject: [PATCH 06/29] [#305]: `extend_task_props_callback` relocated. `extend_task_props_callback` moved from `_get_extended_properties` to `_store_result`. Suggested by @AllesVeldman. `extend_task_props_callback` will get the `request` object as first parameter and a copy of `task_props` (avoiding potential manipulation of the original `task_props`) as second paramenter. --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/backends/database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index aa9acbb5..1f76c733 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -80,7 +80,7 @@ def _get_extended_properties(self, request, traceback): # task protocol 1 task_kwargs = getattr(request, 'kwargs', None) - # TODO: We assuming that task protocol 1 could be always in use. :/ + # TODO: We assume that task protocol 1 could be always in use. :/ extra_fields = extend_task_props_callback( getattr(request, 'kwargs', None) ) @@ -151,6 +151,8 @@ def _store_result( task_props.update( self._get_extended_properties(request, traceback) ) + task_props.update( + extend_task_props_callback(request, dict(task_props))) if status == states.STARTED: task_props['date_started'] = Now() From 0ea8ab8d09ab627cbd52c489855a25e5f5cf56ec Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 15 Aug 2022 20:42:13 +0200 Subject: [PATCH 07/29] [#305]: Added a default callable to `get_callback_function` `get_callback_function()` gets a default callback as an arg returning explicitely an empty dict. `get_callback_function()` raises an `ImproperlyConfigured` exception when the callback is not callable. --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/settings.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 5f4eb61d..a9889a13 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -1,16 +1,19 @@ from django.conf import settings +from django.core.exceptions import ImproperlyConfigured def get_callback_function(settings_name, default=None): """Return the callback function for the given settings name.""" callback = getattr(settings, settings_name, None) - if callback is None: + if not callback: return default - if callable(callback): - return callback + if not callable(callback): + raise ImproperlyConfigured(f"{settings_name} must be callable.") + + return callback extend_task_props_callback = get_callback_function( - "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK" + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK", dict ) \ No newline at end of file From 603637be7c21f8a5f36a58b8f6112d31a732050d Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 16 Aug 2022 09:12:57 +0200 Subject: [PATCH 08/29] Added newline to the end of --- django_celery_results/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index a9889a13..5f10baae 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -4,7 +4,7 @@ def get_callback_function(settings_name, default=None): """Return the callback function for the given settings name.""" - + callback = getattr(settings, settings_name, None) if not callback: return default @@ -14,6 +14,7 @@ def get_callback_function(settings_name, default=None): return callback + extend_task_props_callback = get_callback_function( "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK", dict -) \ No newline at end of file +) From 20a03e0d6c355fcb1f7730d0aa5edcfd6e4673a3 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 16 Aug 2022 16:23:18 +0200 Subject: [PATCH 09/29] [#305] Added a sanity check to `task_props_extension` Added `get_task_props_extension` to `settings` module which will raise an `ImproperlyConfigured` when the task_props_extension doesn't complies with the Mapping protocol. `DatabaseBackend` will make use of `get_task_props_extension` to update a potential custom model with the custom properties --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/backends/database.py | 10 ++++------ django_celery_results/settings.py | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 1f76c733..e46e6d72 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -1,5 +1,6 @@ import binascii import json +from typing import Mapping from celery import maybe_signature, states from celery.backends.base import BaseDictBackend, get_current_task @@ -14,7 +15,7 @@ from ..models import ChordCounter from ..models.helpers import groupresult_model, taskresult_model -from ..settings import extend_task_props_callback +from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) @@ -148,11 +149,8 @@ def _store_result( 'using': using, } - task_props.update( - self._get_extended_properties(request, traceback) - ) - task_props.update( - extend_task_props_callback(request, dict(task_props))) + task_props.update(self._get_extended_properties(request, traceback)) + task_props.update(get_task_props_extension(request, dict(task_props))) if status == states.STARTED: task_props['date_started'] = Now() diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 5f10baae..11cb524b 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -1,5 +1,6 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from collections.abc import Mapping def get_callback_function(settings_name, default=None): @@ -16,5 +17,21 @@ def get_callback_function(settings_name, default=None): extend_task_props_callback = get_callback_function( - "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK", dict + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK" ) + + +def get_task_props_extension(request, task_props): + """Extend the task properties with custom properties to fill custom models.""" + + task_props_extension = extend_task_props_callback(request, task_props) or {} + if task_props_extension is None: + return {} + + if not isinstance(task_props_extension, Mapping): + raise ImproperlyConfigured( + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping " + "instance." + ) + + return task_props_extension From 97880a2456e37eacd2b58136c782e6221b7bb5f5 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 16 Aug 2022 20:22:25 +0200 Subject: [PATCH 10/29] Fixed a NoneType error when the callback is not defined in project settings. --- django_celery_results/settings.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 11cb524b..c41a2469 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -5,7 +5,6 @@ def get_callback_function(settings_name, default=None): """Return the callback function for the given settings name.""" - callback = getattr(settings, settings_name, None) if not callback: return default @@ -22,12 +21,11 @@ def get_callback_function(settings_name, default=None): def get_task_props_extension(request, task_props): - """Extend the task properties with custom properties to fill custom models.""" - - task_props_extension = extend_task_props_callback(request, task_props) or {} - if task_props_extension is None: + """Extend the task properties with custom props to fill custom models.""" + if not extend_task_props_callback: return {} + task_props_extension = extend_task_props_callback(request, task_props) or {} if not isinstance(task_props_extension, Mapping): raise ImproperlyConfigured( "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping " From 49874bc9fefbbc7ccd962a450f0bd98f3232a526 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Thu, 13 Oct 2022 13:22:35 +0200 Subject: [PATCH 11/29] [#305] Added documentation about this feature. A new page added to the documentation where we explain how to use this feature. -- issue: celery/django-celery-results#305 pull-request: celery/django-celery-results#314 --- docs/extending_task_results.rst | 43 +++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 44 insertions(+) create mode 100644 docs/extending_task_results.rst diff --git a/docs/extending_task_results.rst b/docs/extending_task_results.rst new file mode 100644 index 00000000..145ff404 --- /dev/null +++ b/docs/extending_task_results.rst @@ -0,0 +1,43 @@ +Extending Task Results +====================== + +There are situations where you want to extend the Task Results with additional information that will make you able to retrieve information that was important at execution time of the task but not part of the task result itself. For example if you use :pypi:`django-celery-results` to track the task results from an tenant. + +To extend the Task Results model follow the next steps: + +#. Create a custom model that inherits from the abstract base class `django_celery_results.models.abstract.AbstractTaskResult`: + + .. code-block:: python + + from django_celery_results.models.abstract import AbstractTaskResult + + class TaskResult(AbstractTaskResult): + tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, null=True) + +#. Tell Django to use the custom `TaskResult` model by setting the `CELERY_RESULTS_TASKRESULT_MODEL` constant to the path of the custom model. + + .. code-block:: python + + CELERY_RESULTS_TASKRESULT_MODEL = 'myapp.TaskResult' + +#. Write a function in your Django project's :file:`settings.py` that will consume a `request` and `task_properties` as positional arguments and will return a dictionary with the additional information that you want to store in the your custom `TaskResult` model. The keys of this dictionary will be the fields of the custom model and the values the data you can retrieve from the `request` and/or `task_properties`. + + .. code-block:: python + + def extend_task_props_callback(request, task_properties): + """Extend task props with custom data from task_kwargs.""" + task_kwargs = getattr(request, "kwargs", None) + + return {"tenant_id": task_kwargs.get("tenant_id", None)} + +#. To let :pypi:`django-celery-results` call this function internally you've to set the `CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK` constant in your Django project's :file:`settings.py` with the function that you've just created. + + .. code-block:: python + + CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK = extend_task_props_callback + +#. Finally make sure that you're passing the additional information to the celery task when you're calling it. + + .. code-block:: python + + task.apply_async(kwargs={"tenant_id": tenant.id}) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index ad00baf2..65441516 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ Contents :maxdepth: 1 getting_started + extending_task_results injecting_metadata copyright From 8509a64bfc95ceca35a2042b53f0a668559b2937 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 Aug 2022 06:03:54 +0000 Subject: [PATCH 12/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- django_celery_results/admin.py | 2 +- django_celery_results/models/helpers.py | 3 ++- django_celery_results/settings.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/django_celery_results/admin.py b/django_celery_results/admin.py index e0654aec..aff82117 100644 --- a/django_celery_results/admin.py +++ b/django_celery_results/admin.py @@ -10,7 +10,7 @@ ALLOW_EDITS = False pass -from .models.helpers import taskresult_model, groupresult_model +from .models.helpers import groupresult_model, taskresult_model GroupResult = groupresult_model() TaskResult = taskresult_model() diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index 668ea244..a9818070 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -2,7 +2,8 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from .generic import TaskResult, GroupResult +from .generic import GroupResult, TaskResult + def taskresult_model(): """Return the TaskResult model that is active in this project.""" diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index c41a2469..6b246438 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -1,6 +1,7 @@ +from collections.abc import Mapping + from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from collections.abc import Mapping def get_callback_function(settings_name, default=None): From d72b7e83b094c5ac502612b7eaf8c885168b20d4 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Wed, 17 Aug 2022 17:02:38 +0200 Subject: [PATCH 13/29] Fixed a "wrong" description for the `ImproperlyConfigured` exception in `django_celery_results.settings` Fixed a "wrong" description for the `ImproperlyConfigured` exception raised when `task_props_extension` doesn't complies with the Mapping protocol. At this point `task_props_extension` is just a dict that inherits from Mapping, not an explicit instance. Thanks @AllexVeldman Co-authored-by: Allex --- django_celery_results/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 6b246438..05689fe5 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -29,8 +29,7 @@ def get_task_props_extension(request, task_props): task_props_extension = extend_task_props_callback(request, task_props) or {} if not isinstance(task_props_extension, Mapping): raise ImproperlyConfigured( - "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping " - "instance." + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping." ) return task_props_extension From 4b9dfefde43905a94279cb6b814c52b4e8d3321c Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Thu, 13 Oct 2022 19:51:00 +0200 Subject: [PATCH 14/29] [#305] Fixed some pre-commit failures --- django_celery_results/backends/database.py | 1 - django_celery_results/models/generic.py | 1 - django_celery_results/models/helpers.py | 5 +++-- django_celery_results/settings.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index e46e6d72..fbe8fcbd 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -1,6 +1,5 @@ import binascii import json -from typing import Mapping from celery import maybe_signature, states from celery.backends.base import BaseDictBackend, get_current_task diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index f1ec7e01..582983eb 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -52,7 +52,6 @@ class ChordCounter(models.Model): class Meta: app_label = "django_celery_results" - def group_result(self, app=None): """Return the GroupResult of self. diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index a9818070..fb64fd84 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -9,7 +9,7 @@ def taskresult_model(): """Return the TaskResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_TASKRESULT_MODEL'): return TaskResult - + try: return apps.get_model( settings.CELERY_RESULTS_TASKRESULT_MODEL @@ -26,11 +26,12 @@ def taskresult_model(): "been installed" ) + def groupresult_model(): """Return the GroupResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): return GroupResult - + try: return apps.get_model( settings.CELERY_RESULTS_GROUPRESULT_MODEL diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 05689fe5..022b52a0 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -26,7 +26,7 @@ def get_task_props_extension(request, task_props): if not extend_task_props_callback: return {} - task_props_extension = extend_task_props_callback(request, task_props) or {} + task_props_extension = extend_task_props_callback(request, task_props) or {} # noqa E501 if not isinstance(task_props_extension, Mapping): raise ImproperlyConfigured( "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping." From 60fd681d927e9542e5c61ce574751c4360104eaa Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 17 Oct 2022 19:03:01 +0200 Subject: [PATCH 15/29] Update docs/extending_task_results.rst Co-authored-by: Allex --- docs/extending_task_results.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extending_task_results.rst b/docs/extending_task_results.rst index 145ff404..bc8df9be 100644 --- a/docs/extending_task_results.rst +++ b/docs/extending_task_results.rst @@ -1,7 +1,7 @@ Extending Task Results ====================== -There are situations where you want to extend the Task Results with additional information that will make you able to retrieve information that was important at execution time of the task but not part of the task result itself. For example if you use :pypi:`django-celery-results` to track the task results from an tenant. +There are situations where you want to extend the Task Results with additional information that will make you able to retrieve information that was important at execution time of the task but not part of the task result itself. For example if you use :pypi:`django-celery-results` to track the task results from a tenant. To extend the Task Results model follow the next steps: From c2042fa45dd263f18b73de16fcd6c3894c2807b4 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 17 Oct 2022 19:03:35 +0200 Subject: [PATCH 16/29] Update docs/extending_task_results.rst Co-authored-by: Allex --- docs/extending_task_results.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extending_task_results.rst b/docs/extending_task_results.rst index bc8df9be..85f19ed4 100644 --- a/docs/extending_task_results.rst +++ b/docs/extending_task_results.rst @@ -30,7 +30,7 @@ To extend the Task Results model follow the next steps: return {"tenant_id": task_kwargs.get("tenant_id", None)} -#. To let :pypi:`django-celery-results` call this function internally you've to set the `CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK` constant in your Django project's :file:`settings.py` with the function that you've just created. +#. To let :pypi:`django-celery-results` call this function, you'll have to set the `CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK` constant in your Django project's :file:`settings.py`. .. code-block:: python From 1cf3b5241c96f43be12c478d739860087a219609 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 17 Oct 2022 19:03:59 +0200 Subject: [PATCH 17/29] Update docs/extending_task_results.rst Co-authored-by: Allex --- docs/extending_task_results.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extending_task_results.rst b/docs/extending_task_results.rst index 85f19ed4..167422b9 100644 --- a/docs/extending_task_results.rst +++ b/docs/extending_task_results.rst @@ -20,7 +20,7 @@ To extend the Task Results model follow the next steps: CELERY_RESULTS_TASKRESULT_MODEL = 'myapp.TaskResult' -#. Write a function in your Django project's :file:`settings.py` that will consume a `request` and `task_properties` as positional arguments and will return a dictionary with the additional information that you want to store in the your custom `TaskResult` model. The keys of this dictionary will be the fields of the custom model and the values the data you can retrieve from the `request` and/or `task_properties`. +#. Write a function that will consume a `request` and `task_properties` as positional arguments and will return a dictionary with the additional information that you want to store in your custom `TaskResult` model. The keys of this dictionary will be the fields of the custom model and the values the data you can retrieve from the `request` and/or `task_properties`. .. code-block:: python From c1bd4c25a157e7293fac4f9d77f1f094f2f01aee Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 21 Oct 2024 21:10:09 +0200 Subject: [PATCH 18/29] feat(models): add `AbstractChordCounter` and update `ChordCounter` - Added new abstract model `AbstractChordCounter` in `abstract.py` for Chord synchronization, including fields for `group_id`, `sub_tasks`, and `count`. - Updated `ChordCounter` model in `generic.py` to inherit from `AbstractChordCounter`. - Moved `group_result` method from `ChordCounter` to `AbstractChordCounter`. - Added helper function `chordcounter_model` in `helpers.py` to return the active `ChordCounter` model. - Updated import statements in `database.py` to use `chordcounter_model` helper function. - Added `ChordCounterModel` attribute to `DatabaseBackend` class, and updated `create` and `get` methods to use `ChordCounterModel` instead of `ChordCounter`. Relates to: #305 --- django_celery_results/backends/database.py | 10 ++--- django_celery_results/models/abstract.py | 51 ++++++++++++++++++++++ django_celery_results/models/generic.py | 47 ++------------------ django_celery_results/models/helpers.py | 25 ++++++++++- 4 files changed, 84 insertions(+), 49 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index fbe8fcbd..02f5db65 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,8 +12,7 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models import ChordCounter -from ..models.helpers import groupresult_model, taskresult_model +from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) @@ -32,6 +31,7 @@ class DatabaseBackend(BaseDictBackend): TaskModel = taskresult_model() GroupModel = groupresult_model() + ChordCounterModel = chordcounter_model() subpolling_interval = 0.5 def exception_safe_to_retry(self, exc): @@ -248,7 +248,7 @@ def apply_chord(self, header_result_args, body, **kwargs): results = [r.as_tuple() for r in header_result] chord_size = body.get("chord_size", None) or len(results) data = json.dumps(results) - ChordCounter.objects.create( + self.ChordCounterModel.objects.create( group_id=header_result.id, sub_tasks=data, count=chord_size ) @@ -265,10 +265,10 @@ def on_chord_part_return(self, request, state, result, **kwargs): # SELECT FOR UPDATE is not supported on all databases try: chord_counter = ( - ChordCounter.objects.select_for_update() + self.ChordCounterModel.objects.select_for_update() .get(group_id=gid) ) - except ChordCounter.DoesNotExist: + except self.ChordCounterModel.DoesNotExist: logger.warning("Can't find ChordCounter for Group %s", gid) return chord_counter.count -= 1 diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index 21973989..c7a0925c 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -1,11 +1,15 @@ """Abstract models.""" +import json + from celery import states +from celery.result import result_from_tuple from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ from .. import managers +from ..models.helpers import groupresult_model ALL_STATES = sorted(states.ALL_STATES) TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) @@ -132,6 +136,53 @@ def __str__(self): return ''.format(self) +class AbstractChordCounter(models.Model): + """Abstract Chord synchronisation.""" + + group_id = models.CharField( + max_length=getattr( + settings, + "DJANGO_CELERY_RESULTS_TASK_ID_MAX_LENGTH", + 255 + ), + unique=True, + verbose_name=_("Group ID"), + help_text=_("Celery ID for the Chord header group"), + ) + sub_tasks = models.TextField( + help_text=_( + "JSON serialized list of task result tuples. " + "use .group_result() to decode" + ) + ) + count = models.PositiveIntegerField( + help_text=_( + "Starts at len(chord header) and decrements after each task is " + "finished" + ) + ) + + class Meta: + """Table information.""" + + abstract = True + + def group_result(self, app=None): + """Return the GroupResult of self. + + Arguments: + --------- + app (Celery): app instance to create the GroupResult with. + + """ + return groupresult_model()( + self.group_id, + [result_from_tuple(r, app=app) + for r in json.loads(self.sub_tasks)], + app=app + ) + + class AbstractGroupResult(models.Model): """Abstract Task Group result/status.""" diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index 582983eb..e5b5da7e 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -2,13 +2,10 @@ import json -from celery.result import GroupResult as CeleryGroupResult -from celery.result import result_from_tuple -from django.conf import settings -from django.db import models from django.utils.translation import gettext_lazy as _ from django_celery_results.models.abstract import ( + AbstractChordCounter, AbstractGroupResult, AbstractTaskResult, ) @@ -24,49 +21,13 @@ class Meta(AbstractTaskResult.Meta): app_label = "django_celery_results" -class ChordCounter(models.Model): +class ChordCounter(AbstractChordCounter): """Chord synchronisation.""" - group_id = models.CharField( - max_length=getattr( - settings, - "DJANGO_CELERY_RESULTS_TASK_ID_MAX_LENGTH", - 255), - unique=True, - verbose_name=_("Group ID"), - help_text=_("Celery ID for the Chord header group"), - ) - sub_tasks = models.TextField( - help_text=_( - "JSON serialized list of task result tuples. " - "use .group_result() to decode" - ) - ) - count = models.PositiveIntegerField( - help_text=_( - "Starts at len(chord header) and decrements after each task is " - "finished" - ) - ) - - class Meta: + class Meta(AbstractChordCounter.Meta): + abstract = False app_label = "django_celery_results" - def group_result(self, app=None): - """Return the GroupResult of self. - - Arguments: - --------- - app (Celery): app instance to create the GroupResult with. - - """ - return CeleryGroupResult( - self.group_id, - [result_from_tuple(r, app=app) - for r in json.loads(self.sub_tasks)], - app=app - ) - class GroupResult(AbstractGroupResult): """Task Group result/status.""" diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index fb64fd84..0d9da60a 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from .generic import GroupResult, TaskResult +from .generic import ChordCounter, GroupResult, TaskResult def taskresult_model(): @@ -27,6 +27,29 @@ def taskresult_model(): ) +def chordcounter_model(): + """Return the ChordCounter model that is active in this project.""" + + if not hasattr(settings, 'CELERY_RESULTS_CHORDCOUNTER_MODEL'): + return ChordCounter + + try: + return apps.get_model( + settings.CELERY_RESULTS_CHORDCOUNTER_MODEL + ) + except ValueError: + raise ImproperlyConfigured( + "CELERY_RESULTS_CHORDCOUNTER_MODEL must be of the form " + "'app_label.model_name'" + ) + except LookupError: + raise ImproperlyConfigured( + "CELERY_RESULTS_CHORDCOUNTER_MODEL refers to model " + f"'{settings.CELERY_RESULTS_CHORDCOUNTER_MODEL}' that has not " + "been installed" + ) + + def groupresult_model(): """Return the GroupResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): From 35911403890dbcb560605d0df2ddbaab0187b802 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 29 Oct 2024 09:40:47 +0100 Subject: [PATCH 19/29] fix: refactor helper functions to avoid circular dependencies --- django_celery_results/models/generic.py | 4 ++-- django_celery_results/models/helpers.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index e5b5da7e..25c0744c 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -1,7 +1,5 @@ """Database models.""" -import json - from django.utils.translation import gettext_lazy as _ from django_celery_results.models.abstract import ( @@ -25,6 +23,8 @@ class ChordCounter(AbstractChordCounter): """Chord synchronisation.""" class Meta(AbstractChordCounter.Meta): + """Table information.""" + abstract = False app_label = "django_celery_results" diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index 0d9da60a..13387643 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -2,12 +2,12 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from .generic import ChordCounter, GroupResult, TaskResult - def taskresult_model(): """Return the TaskResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_TASKRESULT_MODEL'): + from .generic import TaskResult + return TaskResult try: @@ -31,6 +31,8 @@ def chordcounter_model(): """Return the ChordCounter model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_CHORDCOUNTER_MODEL'): + from .generic import ChordCounter + return ChordCounter try: @@ -53,6 +55,8 @@ def chordcounter_model(): def groupresult_model(): """Return the GroupResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): + from .generic import GroupResult + return GroupResult try: From 885f08cb1cc15142eac56572ea03e2d116e34598 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 29 Oct 2024 10:05:27 +0100 Subject: [PATCH 20/29] fix: undefined name 'ChordCounter' and minor fixes --- django_celery_results/backends/database.py | 12 +++++++++--- django_celery_results/models/generic.py | 2 -- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 02f5db65..7c08b46d 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,7 +12,11 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model +from ..models.helpers import ( + chordcounter_model, + groupresult_model, + taskresult_model, +) from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) @@ -81,7 +85,7 @@ def _get_extended_properties(self, request, traceback): task_kwargs = getattr(request, 'kwargs', None) # TODO: We assume that task protocol 1 could be always in use. :/ - extra_fields = extend_task_props_callback( + extra_fields = get_task_props_extension( getattr(request, 'kwargs', None) ) if extra_fields: @@ -258,7 +262,9 @@ def on_chord_part_return(self, request, state, result, **kwargs): if not gid or not tid: return call_callback = False - with transaction.atomic(using=router.db_for_write(ChordCounter)): + with transaction.atomic( + using=router.db_for_write(self.ChordCounterModel) + ): # We need to know if `count` hits 0. # wrap the update in a transaction # with a `select_for_update` lock to prevent race conditions. diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index 25c0744c..7b626dae 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -1,7 +1,5 @@ """Database models.""" -from django.utils.translation import gettext_lazy as _ - from django_celery_results.models.abstract import ( AbstractChordCounter, AbstractGroupResult, From 3f1bfacbac3e69e935f37231a0ed91e8ce030de0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:05:49 +0000 Subject: [PATCH 21/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- django_celery_results/backends/database.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 7c08b46d..95000d19 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,11 +12,7 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import ( - chordcounter_model, - groupresult_model, - taskresult_model, -) +from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) From 271b0806702331a53d281e326ae786add9f4f375 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Thu, 31 Oct 2024 16:04:23 +0100 Subject: [PATCH 22/29] fix: 'TypeError' introduced in previous commit --- django_celery_results/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_celery_results/managers.py b/django_celery_results/managers.py index b38a15e8..58c00c9c 100644 --- a/django_celery_results/managers.py +++ b/django_celery_results/managers.py @@ -161,7 +161,7 @@ def store_result(self, content_type, content_encoding, 'task_args': task_args, 'task_kwargs': task_kwargs, 'worker': worker, - **extra_fields + 'extra_fields': extra_fields } if 'date_started' in kwargs: fields['date_started'] = kwargs['date_started'] From 73952697bdfe1be4e4cc57a3f07de687725bcd86 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Thu, 31 Oct 2024 16:29:21 +0100 Subject: [PATCH 23/29] fix: include 'extra_fields' conditionally --- django_celery_results/managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_celery_results/managers.py b/django_celery_results/managers.py index 58c00c9c..274b85cf 100644 --- a/django_celery_results/managers.py +++ b/django_celery_results/managers.py @@ -161,8 +161,11 @@ def store_result(self, content_type, content_encoding, 'task_args': task_args, 'task_kwargs': task_kwargs, 'worker': worker, - 'extra_fields': extra_fields } + + if extra_fields is not None: + fields.update(extra_fields) + if 'date_started' in kwargs: fields['date_started'] = kwargs['date_started'] From ed17294b5edc7005f5f08d00e12d16c84a4daa2f Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Fri, 1 Nov 2024 20:28:11 +0100 Subject: [PATCH 24/29] fix: 'get_task_props_extensions()' missing 1 required argument --- django_celery_results/backends/database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 95000d19..1e3dcc5e 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -82,6 +82,7 @@ def _get_extended_properties(self, request, traceback): # TODO: We assume that task protocol 1 could be always in use. :/ extra_fields = get_task_props_extension( + request, getattr(request, 'kwargs', None) ) if extra_fields: From 7be45b58f372470f5c66c87c876cfb740ab1068f Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 5 Nov 2024 21:13:38 +0100 Subject: [PATCH 25/29] fix: TypeError introducedn on prev commit on 'AbstractChordCounter' --- django_celery_results/backends/database.py | 6 +++++- django_celery_results/models/abstract.py | 5 ++--- django_celery_results/models/helpers.py | 8 ++------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 1e3dcc5e..dc20f2c1 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,7 +12,11 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model +from ..models.helpers import ( + chordcounter_model, + groupresult_model, + taskresult_model, +) from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index c7a0925c..40e0f376 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -3,13 +3,12 @@ import json from celery import states -from celery.result import result_from_tuple +from celery.result import CeleryGroupResult, result_from_tuple from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ from .. import managers -from ..models.helpers import groupresult_model ALL_STATES = sorted(states.ALL_STATES) TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) @@ -175,7 +174,7 @@ def group_result(self, app=None): app (Celery): app instance to create the GroupResult with. """ - return groupresult_model()( + return CeleryGroupResult( self.group_id, [result_from_tuple(r, app=app) for r in json.loads(self.sub_tasks)], diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index 13387643..0d9da60a 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -2,12 +2,12 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from .generic import ChordCounter, GroupResult, TaskResult + def taskresult_model(): """Return the TaskResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_TASKRESULT_MODEL'): - from .generic import TaskResult - return TaskResult try: @@ -31,8 +31,6 @@ def chordcounter_model(): """Return the ChordCounter model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_CHORDCOUNTER_MODEL'): - from .generic import ChordCounter - return ChordCounter try: @@ -55,8 +53,6 @@ def chordcounter_model(): def groupresult_model(): """Return the GroupResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): - from .generic import GroupResult - return GroupResult try: From e7fb95e0d88e03f8f2667eb525d2f858628013a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:15:19 +0000 Subject: [PATCH 26/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- django_celery_results/backends/database.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index dc20f2c1..1e3dcc5e 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,11 +12,7 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import ( - chordcounter_model, - groupresult_model, - taskresult_model, -) +from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) From 6245d831e51ddfeac6c0d1f2cbbeb655de6aefbd Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Wed, 6 Nov 2024 21:20:11 +0100 Subject: [PATCH 27/29] fix: ImportError introduced in previous commit in 'abstract.py' --- django_celery_results/models/abstract.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index 40e0f376..f193c479 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -3,7 +3,8 @@ import json from celery import states -from celery.result import CeleryGroupResult, result_from_tuple +from celery.result import GroupResult as CeleryGroupResult +from celery.result import result_from_tuple from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ From 34e36cd0d92f928c44159af78d0cb05cbd931e95 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Fri, 8 Nov 2024 07:46:03 +0100 Subject: [PATCH 28/29] style: Reformat import statements for better readability in 'database.py' --- django_celery_results/backends/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 1e3dcc5e..dc20f2c1 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,7 +12,11 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model +from ..models.helpers import ( + chordcounter_model, + groupresult_model, + taskresult_model, +) from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) From 4c24fde1d9f6b36e109e7a6e3db69ac9de369d3f Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Fri, 8 Nov 2024 08:00:00 +0100 Subject: [PATCH 29/29] fix: Update configuration for isort and black to enforce line length and multi-line output This commit addresses an issue where the pre-commit hook merges import formatted with isort's multiline output mode 3 back into a single line, resulting in a flake8 E508 violation in CI due to lines exceeding 79 characters --- setup.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.cfg b/setup.cfg index 26f1ac65..8086949d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,3 +18,8 @@ match-dir = [^migrations] [isort] profile=black +line_length=79 +multi_line_output=3 + +[black] +line-length = 79