From 198353f814987d4529856ea73d125b3f59e2befe Mon Sep 17 00:00:00 2001 From: Ben Harling Date: Wed, 13 Feb 2019 09:38:38 +0000 Subject: [PATCH 1/5] Add utiity to determine model classes from orm lookups spanning relations --- cacheops/utils.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cacheops/utils.py b/cacheops/utils.py index 988990a4..d0d44255 100644 --- a/cacheops/utils.py +++ b/cacheops/utils.py @@ -141,6 +141,31 @@ def wrapper(request, *args, **kwargs): return cached_view +def get_model_from_lookup(base_model, orm_lookup): + """ + Given a base model and an ORM lookup, follow any relations and return + the final model class of the lookup. + """ + + result = base_model + for field_name in orm_lookup.split('__'): + + if field_name.endswith('_set'): + field_name = field_name.split('_set')[0] + + try: + field = result._meta.get_field(field_name) + except models.FieldDoesNotExist: + break + + if hasattr(field, 'related_model'): + result = field.related_model + else: + break + + return result + + ### Whitespace handling for template tags from django.utils.safestring import mark_safe From 22b3a4de883c7b1f9576863ec30c654a35173f2f Mon Sep 17 00:00:00 2001 From: Ben Harling Date: Wed, 13 Feb 2019 10:45:33 +0000 Subject: [PATCH 2/5] Added cache_prefetch_related() operator --- cacheops/query.py | 35 +++++++++++++++++++++++++++++++++-- tests/test_extras.py | 28 +++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/cacheops/query.py b/cacheops/query.py index dcc69e49..698f648e 100644 --- a/cacheops/query.py +++ b/cacheops/query.py @@ -12,13 +12,13 @@ from django.utils.encoding import smart_str, force_text from django.core.exceptions import ImproperlyConfigured from django.db import DEFAULT_DB_ALIAS -from django.db.models import Manager, Model +from django.db.models import Manager, Model, Prefetch from django.db.models.query import QuerySet from django.db.models.sql.datastructures import EmptyResultSet from django.db.models.signals import pre_save, post_save, post_delete, m2m_changed from .conf import model_profile, settings, ALL_OPS -from .utils import monkey_mix, stamp_fields, func_cache_key, cached_view_fab, family_has_profile +from .utils import monkey_mix, stamp_fields, func_cache_key, cached_view_fab, family_has_profile, get_model_from_lookup from .sharding import get_prefix from .redis import redis_client, handle_connection_failure, load_script from .tree import dnfs @@ -249,6 +249,37 @@ def nocache(self): else: return self.cache(ops=[]) + def cache_prefetch_related(self, *lookups, ops=None, timeout=None, lock=None): + """ + Same as prefetch_related but attempts to pull relations from the cache instead + + lookups - same as for django's vanilla prefetch_related() + ops - a subset of {'get', 'fetch', 'count', 'exists', 'aggregate'}, + ops caching to be turned on, all enabled by default + timeout - override default cache timeout + lock - use lock to prevent dog-pile effect + """ + + # If relations are already fetched there is no point to continuing + if self._prefetch_done: + return self + + prefetches = [] + + for pf in lookups: + if isinstance(pf, Prefetch): + pf.queryset = pf.queryset.cache(ops=ops, timeout=timeout, lock=lock) + prefetches.append(pf) + + if isinstance(pf, str): + model_class = get_model_from_lookup(self.model, pf) + prefetches.append( + Prefetch(pf, model_class._default_manager.all().cache(ops=ops, timeout=timeout, lock=lock)) + ) + + return self.prefetch_related(*prefetches) + + def cloning(self, cloning=1000): self._cloning = cloning return self diff --git a/tests/test_extras.py b/tests/test_extras.py index 8b9def62..849f0b5f 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -1,13 +1,15 @@ from django.db import connections from django.test import TestCase from django.test import override_settings +from django.db.models import Prefetch from cacheops import cached_as, no_invalidation, invalidate_obj, invalidate_model, invalidate_all from cacheops.conf import settings from cacheops.signals import cache_read, cache_invalidated +from cacheops.utils import get_model_from_lookup from .utils import BaseTestCase, make_inc -from .models import Post, Category, Local, DbAgnostic, DbBinded +from .models import Post, Category, Local, DbAgnostic, DbBinded, Brand, Label class SettingsTests(TestCase): @@ -177,3 +179,27 @@ def test_db_agnostic_disabled(self): with self.assertNumQueries(1, using='slave'): list(DbBinded.objects.cache().using('slave')) + + +class CachedPrefetchTest(BaseTestCase): + + def test_get_model_from_lookup(self): + assert get_model_from_lookup(Brand, 'labels') is Label + + def test_cache_prefetch_related(self): + qs = Brand.objects.all().cache_prefetch_related('labels') + + pf = qs._prefetch_related_lookups[0] + + assert isinstance(pf, Prefetch) + assert pf.queryset.model is Label + assert pf.queryset._cacheprofile + + def test_cache_prefetch_related_with_ops(self): + qs = Brand.objects.all().cache_prefetch_related('labels', ops=['get']) + + pf = qs._prefetch_related_lookups[0] + + self.assertEqual(pf.queryset._cacheprofile['ops'], {'get'}) + + From c410cd0af0ab254fbb12513ff2a9788c3d6e66c3 Mon Sep 17 00:00:00 2001 From: Ben Harling Date: Thu, 21 Feb 2019 16:33:04 +0000 Subject: [PATCH 3/5] Removed extra arguments from cache_prefetch_related(), shallow copy Prefetch objects instead of modifying existing --- cacheops/query.py | 10 ++++++---- tests/test_extras.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cacheops/query.py b/cacheops/query.py index 698f648e..202c62fe 100644 --- a/cacheops/query.py +++ b/cacheops/query.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import copy import sys import json import threading @@ -249,7 +250,7 @@ def nocache(self): else: return self.cache(ops=[]) - def cache_prefetch_related(self, *lookups, ops=None, timeout=None, lock=None): + def cache_prefetch_related(self, *lookups): """ Same as prefetch_related but attempts to pull relations from the cache instead @@ -268,13 +269,14 @@ def cache_prefetch_related(self, *lookups, ops=None, timeout=None, lock=None): for pf in lookups: if isinstance(pf, Prefetch): - pf.queryset = pf.queryset.cache(ops=ops, timeout=timeout, lock=lock) - prefetches.append(pf) + item = copy.copy(pf) + item.queryset = item.queryset.cache(ops=['fetch']) + prefetches.append(item) if isinstance(pf, str): model_class = get_model_from_lookup(self.model, pf) prefetches.append( - Prefetch(pf, model_class._default_manager.all().cache(ops=ops, timeout=timeout, lock=lock)) + Prefetch(pf, model_class._default_manager.all().cache(ops=['fetch'])) ) return self.prefetch_related(*prefetches) diff --git a/tests/test_extras.py b/tests/test_extras.py index 849f0b5f..45890e99 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -196,10 +196,10 @@ def test_cache_prefetch_related(self): assert pf.queryset._cacheprofile def test_cache_prefetch_related_with_ops(self): - qs = Brand.objects.all().cache_prefetch_related('labels', ops=['get']) + qs = Brand.objects.all().cache_prefetch_related('labels') pf = qs._prefetch_related_lookups[0] - self.assertEqual(pf.queryset._cacheprofile['ops'], {'get'}) + self.assertEqual(pf.queryset._cacheprofile['ops'], {'fetch'}) From ff96819cde0ee7670a7b287c1b6025391d86be3e Mon Sep 17 00:00:00 2001 From: Ben Harling Date: Thu, 21 Feb 2019 22:32:08 +0000 Subject: [PATCH 4/5] Update query.py Remove docstring for unimplemented args --- cacheops/query.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cacheops/query.py b/cacheops/query.py index 202c62fe..12ec165a 100644 --- a/cacheops/query.py +++ b/cacheops/query.py @@ -255,10 +255,6 @@ def cache_prefetch_related(self, *lookups): Same as prefetch_related but attempts to pull relations from the cache instead lookups - same as for django's vanilla prefetch_related() - ops - a subset of {'get', 'fetch', 'count', 'exists', 'aggregate'}, - ops caching to be turned on, all enabled by default - timeout - override default cache timeout - lock - use lock to prevent dog-pile effect """ # If relations are already fetched there is no point to continuing From 290da1804a90c6c9d61a9ab2ebc334fe342374f3 Mon Sep 17 00:00:00 2001 From: Ben Harling Date: Sun, 5 Apr 2020 09:31:13 +0100 Subject: [PATCH 5/5] Fix linting --- cacheops/query.py | 4 ++-- tests/test_extras.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cacheops/query.py b/cacheops/query.py index 84a2bc00..9ae69151 100644 --- a/cacheops/query.py +++ b/cacheops/query.py @@ -25,7 +25,8 @@ MAX_GET_RESULTS = None from .conf import model_profile, settings, ALL_OPS -from .utils import monkey_mix, stamp_fields, func_cache_key, cached_view_fab, family_has_profile, get_model_from_lookup +from .utils import monkey_mix, stamp_fields, func_cache_key, cached_view_fab, \ + family_has_profile, get_model_from_lookup from .utils import md5 from .sharding import get_prefix from .redis import redis_client, handle_connection_failure, load_script @@ -280,7 +281,6 @@ def cache_prefetch_related(self, *lookups): return self.prefetch_related(*prefetches) - def cloning(self, cloning=1000): self._cloning = cloning return self diff --git a/tests/test_extras.py b/tests/test_extras.py index 7d5c0ab8..caf3d1f4 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -1,4 +1,4 @@ -from django.db import connections, transaction +from django.db import transaction from django.db.models import Prefetch from django.test import TestCase, override_settings @@ -207,5 +207,3 @@ def test_cache_prefetch_related_with_ops(self): pf = qs._prefetch_related_lookups[0] self.assertEqual(pf.queryset._cacheprofile['ops'], {'fetch'}) - -