Support SQLAlchemy 2.0

.github/workflows/tests.yml

@@ -31,41 +31,20 @@ jobs:
 
 
     strategy:
-      fail-fast: false
       matrix:
-        include:
-          # sqlalchemylatest (i.e. > 2.0.0) is not yet supported
-          # for any version of python
-
-          - {python: '3.7', tox: "py37-sqlalchemy1.0"}
-          - {python: '3.7', tox: "py37-sqlalchemy1.1"}
-          - {python: '3.7', tox: "py37-sqlalchemy1.2"}
-          - {python: '3.7', tox: "py37-sqlalchemy1.3"}
-          - {python: '3.7', tox: "py37-sqlalchemy1.4"}
-
-          - {python: '3.8', tox: "py38-sqlalchemy1.0"}
-          - {python: '3.8', tox: "py38-sqlalchemy1.1"}
-          - {python: '3.8', tox: "py38-sqlalchemy1.2"}
-          - {python: '3.8', tox: "py38-sqlalchemy1.3"}
-          - {python: '3.8', tox: "py38-sqlalchemy1.4"}
-
-          - {python: '3.9', tox: "py39-sqlalchemy1.0"}
-          - {python: '3.9', tox: "py39-sqlalchemy1.1"}
-          - {python: '3.9', tox: "py39-sqlalchemy1.2"}
-          - {python: '3.9', tox: "py39-sqlalchemy1.3"}
-          - {python: '3.9', tox: "py39-sqlalchemy1.4"}
-
-          # python3.10 with sqlalchemy <= 1.1 errors with:
-          # AttributeError: module 'collections' has no attribute 'MutableMapping'
-          - {python: '3.10', tox: "py310-sqlalchemy1.2"}
-          - {python: '3.10', tox: "py310-sqlalchemy1.3"}
-          - {python: '3.10', tox: "py310-sqlalchemy1.4"}
+        python: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install '.[dev]'
+          python -m pip install tox-gh-actions~=2.12.0
+      - name: Test with tox
+        run: tox
 
-      - run: pip install tox~=3.28
-      - run: tox -e ${{ matrix.tox }}

.gitignore

@@ -10,3 +10,8 @@ __pycache__/
 .coverage.*
 .cache
 .tox
+
+venv
+.venv
+.idea
+build

.python-version

@@ -0,0 +1,6 @@
+3.7.17
+3.8.18
+3.9.18
+3.10.13
+3.11.7
+3.12.1

setup.py

@@ -26,14 +26,15 @@
         'dev': [
             'pytest>=4.6.9',
             'coverage~=5.0.4',
-            'sqlalchemy-utils>=0.37',
+            'sqlalchemy-utils>=0.38.3',
             'flake8',
             'restructuredtext-lint',
             'Pygments',
             'coverage-conditional-plugin',
+            'tox~=3.28'
         ],
-        'mysql': ['mysql-connector-python-rf==2.2.2'],
-        'postgresql': ['psycopg2==2.8.4'],
+        'mysql': ['mysql-connector-python-rf>=2.2.2'],
+        'postgresql': ['psycopg2>=2.8.4'],
     },
     zip_safe=True,
     license='Apache License, Version 2.0',

sqlalchemy_filters/models.py

@@ -1,17 +1,22 @@
+import operator
+
 from sqlalchemy import __version__ as sqlalchemy_version
 from sqlalchemy.exc import InvalidRequestError
 from sqlalchemy.orm import mapperlib
 from sqlalchemy.inspection import inspect
-from sqlalchemy.util import symbol
 import types
 
 from .exceptions import BadQuery, FieldNotFound, BadSpec
 
 
-def sqlalchemy_version_lt(version):
+def sqlalchemy_version_cmp(op, version):
     """compares sqla version < version"""
 
-    return tuple(sqlalchemy_version.split('.')) < tuple(version.split('.'))
+    ops = {'<': operator.lt, '>=': operator.ge}
+    return ops[op](
+        tuple(sqlalchemy_version.split('.')),
+        tuple(version.split('.'))
+    )
 
 
 class Field(object):
@@ -51,11 +56,19 @@ def _get_valid_field_names(self):
 
 
 def _is_hybrid_property(orm_descriptor):
-    return orm_descriptor.extension_type == symbol('HYBRID_PROPERTY')
+    # SQLAlchemy 2 treats extension_type as an enum, not a symbol()
+    # Enum is at sqlalchemy.ext.hybrid.HybridExtensionType
+
+    return (str(orm_descriptor.extension_type)
+            in ("symbol('HYBRID_PROPERTY')", 'HybridExtensionType.HYBRID_PROPERTY'))
 
 
 def _is_hybrid_method(orm_descriptor):
-    return orm_descriptor.extension_type == symbol('HYBRID_METHOD')
+    # SQLAlchemy 2 treats extension_type as an enum, not a symbol()
+    # Enum is at sqlalchemy.ext.hybrid.HybridExtensionType
+
+    return (str(orm_descriptor.extension_type)
+            in ("symbol('HYBRID_METHOD')", 'HybridExtensionType.HYBRID_METHOD'))
 
 
 def get_model_from_table(table):  # pragma: no_cover_sqlalchemy_lt_1_4
@@ -68,7 +81,7 @@ def get_model_from_table(table):  # pragma: no_cover_sqlalchemy_lt_1_4
     return None
 
 
-def get_query_models(query):
+def get_query_models(query):  # pragma: nocover
     """Get models from query.
 
     :param query:
@@ -80,39 +93,39 @@ def get_query_models(query):
     models = [col_desc['entity'] for col_desc in query.column_descriptions]
 
     # account joined entities
-    if sqlalchemy_version_lt('1.4'):  # pragma: no_cover_sqlalchemy_gte_1_4
+    if sqlalchemy_version_cmp('<', '1.4'):
         models.extend(mapper.class_ for mapper in query._join_entities)
-    else:  # pragma: no_cover_sqlalchemy_lt_1_4
+    else:
         try:
             models.extend(
                 mapper.class_
                 for mapper
                 in query._compile_state()._join_entities
             )
-        except InvalidRequestError:
+        except (InvalidRequestError, AttributeError):
             # query might not contain columns yet, hence cannot be compiled
-            # try to infer the models from various internals
-            for table_tuple in query._setup_joins + query._legacy_setup_joins:
-                model_class = get_model_from_table(table_tuple[0])
-                if model_class:
-                    models.append(model_class)
+            # or query might be a sqla2.0 select statement
+            pass
+        # also try to infer the models from various internals
+        all_joins = query._setup_joins
+        if hasattr(query, "_legacy_setup_joins"):
+            all_joins += query._legacy_setup_joins
+
+        for table_tuple in all_joins:
+            models.append(get_model_from_table(table_tuple[0]))
 
     # account also query.select_from entities
-    model_class = None
-    if sqlalchemy_version_lt('1.4'):  # pragma: no_cover_sqlalchemy_gte_1_4
+    if sqlalchemy_version_cmp('<', '1.1'):  # sqla 1.0
         if query._select_from_entity:
-            model_class = (
-                query._select_from_entity
-                if sqlalchemy_version_lt('1.1')
-                else query._select_from_entity.class_
-            )
-    else:  # pragma: no_cover_sqlalchemy_lt_1_4
+            models.append(query._select_from_entity)
+    elif sqlalchemy_version_cmp('<', '1.4'):  # sqla 1.1-1.3
+        if query._select_from_entity:
+            models.append(query._select_from_entity.class_)
+    else:  # sqla 1.4
         if query._from_obj:
-            model_class = get_model_from_table(query._from_obj[0])
-    if model_class and (model_class not in models):
-        models.append(model_class)
+            models.append(get_model_from_table(query._from_obj[0]))
 
-    return {model.__name__: model for model in models}
+    return {model.__name__: model for model in models if model is not None}
 
 
 def get_model_from_spec(spec, query, default_model=None):
@@ -191,23 +204,28 @@ def auto_join(query, *model_names):
     last_model = list(query_models)[-1]
     model_registry = (
         last_model._decl_class_registry
-        if sqlalchemy_version_lt('1.4')
+        if sqlalchemy_version_cmp('<', '1.4')
         else last_model.registry._class_registry
     )
 
+    existing_models = set(query_models)
+
     for name in model_names:
         model = get_model_class_by_name(model_registry, name)
-        if model and (model not in get_query_models(query).values()):
+        if model and (model not in query_models):
             try:
-                if sqlalchemy_version_lt('1.4'):  # pragma: no_cover_sqlalchemy_gte_1_4
-                    query = query.join(model)
-                else:  # pragma: no_cover_sqlalchemy_lt_1_4
+                tmp = query.join(model)
+                tmp = tmp.add_entity(model)  # Explicitly add to SELECT clause
+                existing_models.add(model)  # Keep track of added models
+                if (
+                    sqlalchemy_version_cmp('>=', '1.4')
+                    and hasattr(tmp, '_compile_state')
+                ):  # pragma: nocover
                     # https://docs.sqlalchemy.org/en/14/changelog/migration_14.html
                     # Many Core and ORM statement objects now perform much of
                     # their construction and validation in the compile phase
-                    tmp = query.join(model)
                     tmp._compile_state()
-                    query = tmp
+                query = tmp
             except InvalidRequestError:
                 pass  # can't be autojoined
     return query

test/conftest.py

@@ -131,7 +131,7 @@ def connection(db_uri, db_engine_options, is_postgresql):
 
     yield connection
 
-    Base.metadata.drop_all()
+    Base.metadata.drop_all(engine)
     destroy_database(db_uri)
 
 

test/interface/test_filters.py

@@ -6,11 +6,13 @@
 from six import string_types
 from sqlalchemy import func
 from sqlalchemy.orm import joinedload
+from sqlalchemy.sql import select
 
 from sqlalchemy_filters import apply_filters
 from sqlalchemy_filters.exceptions import (
     BadFilterFormat, BadSpec, FieldNotFound
 )
+from sqlalchemy_filters.models import sqlalchemy_version_cmp
 
 from test.models import Foo, Bar, Qux, Corge
 
@@ -1316,3 +1318,27 @@ def test_filter_by_hybrid_methods(self, session):
         assert set(map(type, quxs)) == {Qux}
         assert {qux.id for qux in quxs} == {4}
         assert {qux.three_times_count() for qux in quxs} == {45}
+
+
+class TestSelectObject:
+
+    @pytest.mark.usefixtures('multiple_foos_inserted')
+    def test_filter_on_select(self, session):
+        if sqlalchemy_version_cmp('<', '1.4'):
+            pytest.skip("Sqlalchemy select style 2.0 not supported")
+
+        query = select(Foo)
+        filters = [
+            {
+                'model': 'Bar',
+                'field': 'name',
+                'op': '==',
+                'value': 'name_2'
+            }
+        ]
+
+        query = apply_filters(query, filters)
+        result = session.execute(query).fetchall()
+
+        assert len(result) == 1
+        assert result[0][0].name == 'name_2'