diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index f49a0e9a9..85f3bd33e 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -3,13 +3,13 @@ set -eux # Install django-mongodb-backend -/opt/python/3.10/bin/python3 -m venv venv +/opt/python/3.12/bin/python3 -m venv venv . venv/bin/activate python -m pip install -U pip pip install -e . # Install django and test dependencies -git clone --branch mongodb-5.2.x https://github.com/mongodb-forks/django django_repo +git clone --branch mongodb-6.0.x https://github.com/mongodb-forks/django django_repo pushd django_repo/tests/ pip install -e .. pip install -r requirements/py3.txt diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index f16a13d41..d6d5c9245 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -17,7 +17,7 @@ jobs: persist-credentials: false - uses: actions/setup-python@v6 with: - python-version: '3.10' + python-version: '3.12' cache: 'pip' cache-dependency-path: 'pyproject.toml' - name: Install Python dependencies @@ -39,7 +39,7 @@ jobs: with: cache: 'pip' cache-dependency-path: 'pyproject.toml' - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: | pip install -U pip diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index 6f9553422..62cf9819c 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -33,7 +33,7 @@ jobs: uses: actions/checkout@v5 with: repository: 'mongodb-forks/django' - ref: 'mongodb-5.2.x' + ref: 'mongodb-6.0.x' path: 'django_repo' persist-credentials: false - name: Install system packages for Django's Python test dependencies diff --git a/.github/workflows/test-python-geo.yml b/.github/workflows/test-python-geo.yml index 0976bb064..cd7725bd0 100644 --- a/.github/workflows/test-python-geo.yml +++ b/.github/workflows/test-python-geo.yml @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@v5 with: repository: 'mongodb-forks/django' - ref: 'mongodb-5.2.x' + ref: 'mongodb-6.0.x' path: 'django_repo' persist-credentials: false - name: Install system packages for Django's Python test dependencies diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index ff36c8949..5067433c2 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -33,7 +33,7 @@ jobs: uses: actions/checkout@v5 with: repository: 'mongodb-forks/django' - ref: 'mongodb-5.2.x' + ref: 'mongodb-6.0.x' path: 'django_repo' persist-credentials: false - name: Install system packages for Django's Python test dependencies diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1ab8dc416..055443596 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -21,6 +21,6 @@ python: - docs build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.11" + python: "3.12" diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index d8b1ac19e..9f723d4ea 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -1,8 +1,8 @@ -__version__ = "5.2.2.dev0" +__version__ = "6.0.0b0.dev0" # Check Django compatibility before other imports which may fail if the # wrong version of Django is installed. -from .utils import check_django_compatability, parse_uri +from .utils import check_django_compatability check_django_compatability() @@ -15,8 +15,6 @@ from .lookups import register_lookups # noqa: E402 from .query import register_nodes # noqa: E402 -__all__ = ["parse_uri"] - register_aggregates() register_checks() register_expressions() diff --git a/django_mongodb_backend/aggregates.py b/django_mongodb_backend/aggregates.py index 798c75d6d..f05e4c919 100644 --- a/django_mongodb_backend/aggregates.py +++ b/django_mongodb_backend/aggregates.py @@ -1,5 +1,13 @@ -from django.db.models.aggregates import Aggregate, Count, StdDev, Variance -from django.db.models.expressions import Case, Value, When +from django.db import NotSupportedError +from django.db.models.aggregates import ( + Aggregate, + AggregateFilter, + Count, + StdDev, + StringAgg, + Variance, +) +from django.db.models.expressions import Case, Col, Value, When from django.db.models.lookups import IsNull from .query_utils import process_lhs @@ -9,7 +17,11 @@ def aggregate(self, compiler, connection, operator=None, resolve_inner_expression=False): - if self.filter: + # TODO: isinstance(self.filter, Col) works around failure of + # aggregation.tests.AggregateTestCase.test_distinct_on_aggregate. Is this + # correct? + if self.filter is not None and not isinstance(self.filter, Col): + # Generate a CASE statement for this aggregate. node = self.copy() node.filter = None source_expressions = node.get_source_expressions() @@ -24,6 +36,10 @@ def aggregate(self, compiler, connection, operator=None, resolve_inner_expressio return {f"${operator}": lhs_mql} +def aggregate_filter(self, compiler, connection): + return self.condition.as_mql(compiler, connection) + + def count(self, compiler, connection, resolve_inner_expression=False): """ When resolve_inner_expression=True, return the MQL that resolves as a @@ -65,8 +81,14 @@ def stddev_variance(self, compiler, connection): return aggregate(self, compiler, connection, operator=operator) +def string_agg(self, compiler, connection): + raise NotSupportedError("StringAgg is not supported.") + + def register_aggregates(): Aggregate.as_mql = aggregate + AggregateFilter.as_mql = aggregate_filter Count.as_mql = count StdDev.as_mql = stddev_variance + StringAgg.as_mql = string_agg Variance.as_mql = stddev_variance diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 7e29c5003..326644683 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -102,6 +102,18 @@ class DatabaseFeatures(GISFeatures, BaseDatabaseFeatures): "model_fields.test_jsonfield.TestSaveLoad.test_bulk_update_custom_get_prep_value", # To debug: https://github.com/mongodb/django-mongodb-backend/issues/362 "constraints.tests.UniqueConstraintTests.test_validate_case_when", + # StringAgg is not supported. + "aggregation.tests.AggregateTestCase.test_distinct_on_stringagg", + "aggregation.tests.AggregateTestCase.test_string_agg_escapes_delimiter", + "aggregation.tests.AggregateTestCase.test_string_agg_filter", + "aggregation.tests.AggregateTestCase.test_string_agg_filter_in_subquery", + "aggregation.tests.AggregateTestCase.test_stringagg_default_value", + # bulk_create() population of _order not implemented. + # https://github.com/django/django/commit/953095d1e603fe0f8f01175b1409ca23818dcff9 + "contenttypes_tests.test_order_with_respect_to.OrderWithRespectToGFKTests.test_bulk_create_allows_duplicate_order_values", + "contenttypes_tests.test_order_with_respect_to.OrderWithRespectToGFKTests.test_bulk_create_mixed_scenario", + "contenttypes_tests.test_order_with_respect_to.OrderWithRespectToGFKTests.test_bulk_create_respects_mixed_manual_order", + "contenttypes_tests.test_order_with_respect_to.OrderWithRespectToGFKTests.test_bulk_create_with_existing_children", } # $bitAnd, #bitOr, and $bitXor are new in MongoDB 6.3. _django_test_expected_failures_bitwise = { @@ -142,6 +154,7 @@ def django_test_expected_failures(self): "validation.test_unique.PerformUniqueChecksTest.test_unique_db_default", }, "Insert expressions aren't supported.": { + "basic.tests.ModelTest.test_save_expressions", "bulk_create.tests.BulkCreateTests.test_bulk_insert_now", "bulk_create.tests.BulkCreateTests.test_bulk_insert_expressions", "expressions.tests.BasicExpressionsTests.test_new_object_create", @@ -204,6 +217,7 @@ def django_test_expected_failures(self): "prefetch_related.tests.LookupOrderingTest.test_order", "prefetch_related.tests.MultiDbTests.test_using_is_honored_m2m", "prefetch_related.tests.MultiTableInheritanceTest", + "prefetch_related.tests.PrefetchRelatedMTICacheTests", "prefetch_related.tests.PrefetchRelatedTests", "prefetch_related.tests.ReadPrefetchedObjectsCacheTests", "prefetch_related.tests.Ticket21410Tests", @@ -566,6 +580,7 @@ def django_test_expected_failures(self): "Custom lookups are not supported.": { "custom_lookups.tests.BilateralTransformTests", "custom_lookups.tests.LookupTests.test_basic_lookup", + "custom_lookups.tests.LookupTests.test_custom_lookup_with_subquery", "custom_lookups.tests.LookupTests.test_custom_name_lookup", "custom_lookups.tests.LookupTests.test_div3_extract", "custom_lookups.tests.SubqueryTransformTests.test_subquery_usage", @@ -583,6 +598,9 @@ def django_test_expected_failures(self): "test_utils.tests.DisallowedDatabaseQueriesTests.test_disallowed_database_queries", "test_utils.tests.DisallowedDatabaseQueriesTests.test_disallowed_thread_database_connection", }, + "search lookup not supported on non-Atlas.": { + "expressions.tests.BasicExpressionsTests.test_lookups_subquery", + }, } @cached_property diff --git a/django_mongodb_backend/fields/json.py b/django_mongodb_backend/fields/json.py index aeb792d75..0e3fd3674 100644 --- a/django_mongodb_backend/fields/json.py +++ b/django_mongodb_backend/fields/json.py @@ -24,7 +24,12 @@ def build_json_mql_path(lhs, key_transforms): get_field = {"$getField": {"input": result, "field": key}} # Handle array indexing if the key is a digit. If key is something # like '001', it's not an array index despite isdigit() returning True. - if key.isdigit() and str(int(key)) == key: + try: + int(key) + is_digit = str(int(key)) == key + except ValueError: + is_digit = False + if is_digit: result = { "$cond": { "if": {"$isArray": result}, diff --git a/django_mongodb_backend/gis/operations.py b/django_mongodb_backend/gis/operations.py index b5d5df1d5..729ea197e 100644 --- a/django_mongodb_backend/gis/operations.py +++ b/django_mongodb_backend/gis/operations.py @@ -40,6 +40,7 @@ def gis_operators(self): "FromWKT", "GeoHash", "GeometryDistance", + "GeometryType", "Intersection", "IsEmpty", "IsValid", @@ -52,6 +53,7 @@ def gis_operators(self): "Perimeter", "PointOnSurface", "Reverse", + "Rotate", "Scale", "SnapToGrid", "SymDifference", diff --git a/django_mongodb_backend/utils.py b/django_mongodb_backend/utils.py index 0240250cf..8c68bf442 100644 --- a/django_mongodb_backend/utils.py +++ b/django_mongodb_backend/utils.py @@ -1,16 +1,13 @@ import copy import time -import warnings import django from django.conf import settings from django.core.exceptions import ImproperlyConfigured, ValidationError from django.db.backends.utils import logger -from django.utils.deprecation import RemovedInDjango60Warning from django.utils.functional import SimpleLazyObject from django.utils.text import format_lazy from django.utils.version import get_version_tuple -from pymongo.uri_parser import parse_uri as pymongo_parse_uri def check_django_compatability(): @@ -30,50 +27,6 @@ def check_django_compatability(): ) -def parse_uri(uri, *, db_name=None, options=None, test=None): - """ - Convert the given uri into a dictionary suitable for Django's DATABASES - setting. - """ - warnings.warn( - 'parse_uri() is deprecated. Put the connection string in DATABASES["HOST"] instead.', - RemovedInDjango60Warning, - stacklevel=2, - ) - uri = pymongo_parse_uri(uri) - host = None - port = None - if uri["fqdn"]: - # This is a SRV URI and the host is the fqdn. - host = f"mongodb+srv://{uri['fqdn']}" - else: - nodelist = uri.get("nodelist") - if len(nodelist) == 1: - host, port = nodelist[0] - elif len(nodelist) > 1: - host = ",".join([f"{host}:{port}" for host, port in nodelist]) - db_name = db_name or uri["database"] - if not db_name: - raise ImproperlyConfigured("You must provide the db_name parameter.") - opts = uri.get("options") - if options: - opts.update(options) - settings_dict = { - "ENGINE": "django_mongodb_backend", - "NAME": db_name, - "HOST": host, - "PORT": port, - "USER": uri.get("username"), - "PASSWORD": uri.get("password"), - "OPTIONS": opts, - } - if "authSource" not in settings_dict["OPTIONS"] and uri["database"]: - settings_dict["OPTIONS"]["authSource"] = uri["database"] - if test: - settings_dict["TEST"] = test - return settings_dict - - def prefix_validation_error(error, prefix, code, params): """ Prefix a validation error message while maintaining the existing diff --git a/pyproject.toml b/pyproject.toml index 725ab9d61..de13891a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version", "dependencies"] description = "Django MongoDB Backend" readme = "README.md" license = {file="LICENSE"} -requires-python = ">=3.10" +requires-python = ">=3.12" authors = [ { name = "The MongoDB Python Team" }, ] @@ -27,8 +27,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] diff --git a/tests/backend_/utils/__init__.py b/tests/backend_/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/backend_/utils/test_parse_uri.py b/tests/backend_/utils/test_parse_uri.py deleted file mode 100644 index a87154162..000000000 --- a/tests/backend_/utils/test_parse_uri.py +++ /dev/null @@ -1,117 +0,0 @@ -from unittest.mock import patch - -import pymongo -from django.core.exceptions import ImproperlyConfigured -from django.test import SimpleTestCase -from django.test.utils import ignore_warnings -from django.utils.deprecation import RemovedInDjango60Warning - -from django_mongodb_backend import parse_uri - - -@ignore_warnings(category=RemovedInDjango60Warning) -class ParseURITests(SimpleTestCase): - def test_simple_uri(self): - settings_dict = parse_uri("mongodb://cluster0.example.mongodb.net/myDatabase") - self.assertEqual(settings_dict["ENGINE"], "django_mongodb_backend") - self.assertEqual(settings_dict["NAME"], "myDatabase") - self.assertEqual(settings_dict["HOST"], "cluster0.example.mongodb.net") - self.assertEqual(settings_dict["OPTIONS"], {"authSource": "myDatabase"}) - - def test_db_name(self): - settings_dict = parse_uri("mongodb://cluster0.example.mongodb.net/", db_name="myDatabase") - self.assertEqual(settings_dict["ENGINE"], "django_mongodb_backend") - self.assertEqual(settings_dict["NAME"], "myDatabase") - self.assertEqual(settings_dict["HOST"], "cluster0.example.mongodb.net") - self.assertEqual(settings_dict["OPTIONS"], {}) - - def test_db_name_overrides_default_auth_db(self): - settings_dict = parse_uri( - "mongodb://cluster0.example.mongodb.net/default_auth_db", db_name="myDatabase" - ) - self.assertEqual(settings_dict["ENGINE"], "django_mongodb_backend") - self.assertEqual(settings_dict["NAME"], "myDatabase") - self.assertEqual(settings_dict["HOST"], "cluster0.example.mongodb.net") - self.assertEqual(settings_dict["OPTIONS"], {"authSource": "default_auth_db"}) - - def test_no_database(self): - msg = "You must provide the db_name parameter." - with self.assertRaisesMessage(ImproperlyConfigured, msg): - parse_uri("mongodb://cluster0.example.mongodb.net") - - def test_srv_uri_with_options(self): - uri = "mongodb+srv://my_user:my_password@cluster0.example.mongodb.net/my_database?retryWrites=true&w=majority" - # patch() prevents a crash when PyMongo attempts to resolve the - # nonexistent SRV record. - with patch("dns.resolver.resolve"): - settings_dict = parse_uri(uri) - self.assertEqual(settings_dict["NAME"], "my_database") - self.assertEqual(settings_dict["HOST"], "mongodb+srv://cluster0.example.mongodb.net") - self.assertEqual(settings_dict["USER"], "my_user") - self.assertEqual(settings_dict["PASSWORD"], "my_password") - self.assertIsNone(settings_dict["PORT"]) - self.assertEqual( - settings_dict["OPTIONS"], - {"authSource": "my_database", "retryWrites": True, "w": "majority", "tls": True}, - ) - - def test_localhost(self): - settings_dict = parse_uri("mongodb://localhost/db") - self.assertEqual(settings_dict["HOST"], "localhost") - self.assertEqual(settings_dict["PORT"], 27017) - - def test_localhost_with_port(self): - settings_dict = parse_uri("mongodb://localhost:27018/db") - self.assertEqual(settings_dict["HOST"], "localhost") - self.assertEqual(settings_dict["PORT"], 27018) - - def test_hosts_with_ports(self): - settings_dict = parse_uri("mongodb://localhost:27017,localhost:27018/db") - self.assertEqual(settings_dict["HOST"], "localhost:27017,localhost:27018") - self.assertEqual(settings_dict["PORT"], None) - - def test_hosts_without_ports(self): - settings_dict = parse_uri("mongodb://host1.net,host2.net/db") - self.assertEqual(settings_dict["HOST"], "host1.net:27017,host2.net:27017") - self.assertEqual(settings_dict["PORT"], None) - - def test_auth_source_in_query_string(self): - settings_dict = parse_uri("mongodb://localhost/?authSource=auth", db_name="db") - self.assertEqual(settings_dict["NAME"], "db") - self.assertEqual(settings_dict["OPTIONS"], {"authSource": "auth"}) - - def test_auth_source_in_query_string_overrides_defaultauthdb(self): - settings_dict = parse_uri("mongodb://localhost/db?authSource=auth") - self.assertEqual(settings_dict["NAME"], "db") - self.assertEqual(settings_dict["OPTIONS"], {"authSource": "auth"}) - - def test_options_kwarg(self): - options = {"authSource": "auth", "retryWrites": True} - settings_dict = parse_uri( - "mongodb://cluster0.example.mongodb.net/myDatabase?retryWrites=false&retryReads=true", - options=options, - ) - self.assertEqual( - settings_dict["OPTIONS"], - {"authSource": "auth", "retryWrites": True, "retryReads": True}, - ) - - def test_test_kwarg(self): - settings_dict = parse_uri("mongodb://localhost/db", test={"NAME": "test_db"}) - self.assertEqual(settings_dict["TEST"], {"NAME": "test_db"}) - - def test_invalid_credentials(self): - msg = "The empty string is not valid username" - with self.assertRaisesMessage(pymongo.errors.InvalidURI, msg): - parse_uri("mongodb://:@localhost") - - def test_no_scheme(self): - with self.assertRaisesMessage(pymongo.errors.InvalidURI, "Invalid URI scheme"): - parse_uri("cluster0.example.mongodb.net") - - -class ParseURIDeprecationTests(SimpleTestCase): - def test_message(self): - msg = 'parse_uri() is deprecated. Put the connection string in DATABASES["HOST"] instead.' - with self.assertRaisesMessage(RemovedInDjango60Warning, msg): - parse_uri("mongodb://cluster0.example.mongodb.net/") diff --git a/tests/expression_converter_/test_op_expressions.py b/tests/expression_converter_/test_op_expressions.py index ce4caf2d4..7b903bcaf 100644 --- a/tests/expression_converter_/test_op_expressions.py +++ b/tests/expression_converter_/test_op_expressions.py @@ -15,7 +15,7 @@ class ConversionTestCase(SimpleTestCase): "boolean": True, "NoneType": None, "string": "string", - "datetime": datetime.datetime.now(datetime.timezone.utc), + "datetime": datetime.datetime.now(datetime.UTC), "duration": datetime.timedelta(days=5, hours=3), "uuid": UUID("12345678123456781234567812345678"), }