Add support for relative cursors. (#8048)

Author: michaelstaib

src/Directory.Packages.props

@@ -26,9 +26,9 @@
     <PackageVersion Include="Microsoft.Azure.Functions.Worker.Core" Version="1.4.0" />
     <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Abstractions" Version="1.1.0" />
     <PackageVersion Include="Microsoft.Bcl.HashCode" Version="1.1.1" />
-    <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" />
-    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" />
-    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.9.2" />
+    <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0" />
+    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.13.0" />
+    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.13.0" />
     <PackageVersion Include="Microsoft.Extensions.Identity.Core" Version="7.0.3" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.0" />
     <PackageVersion Include="Microsoft.OpenApi" Version="1.6.14" />
@@ -93,6 +93,7 @@
     <PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="9.0.2" />
     <PackageVersion Include="System.IO.Packaging" Version="9.0.2" />
     <PackageVersion Include="System.IO.Pipelines" Version="9.0.2" />
+    <PackageVersion Include="System.IO.Hashing" Version="9.0.2" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
     <PackageVersion Include="Microsoft.AspNetCore.Components.Web" Version="8.0.0" />
@@ -117,6 +118,7 @@
     <PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="8.0.0" />
     <PackageVersion Include="System.IO.Packaging" Version="8.0.1" />
     <PackageVersion Include="System.IO.Pipelines" Version="8.0.0" />
+    <PackageVersion Include="System.IO.Hashing" Version="8.0.0" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
     <PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="6.0.0" />

src/GreenDonut/src/GreenDonut.Data.EntityFramework/Expressions/ExpressionHelpers.cs

@@ -40,25 +40,25 @@ internal static class ExpressionHelpers
     /// <exception cref="ArgumentException">
     /// If the number of keys does not match the number of values.
     /// </exception>
-    public static Expression<Func<T, bool>> BuildWhereExpression<T>(
+    public static (Expression<Func<T, bool>> WhereExpression, int Offset, bool ReverseOrder) BuildWhereExpression<T>(
         ReadOnlySpan<CursorKey> keys,
-        ReadOnlySpan<object?> cursor,
+        Cursor cursor,
         bool forward)
     {
         if (keys.Length == 0)
         {
             throw new ArgumentException("At least one key must be specified.", nameof(keys));
         }
 
-        if (keys.Length != cursor.Length)
+        if (keys.Length != cursor.Values.Length)
         {
-            throw new ArgumentException("The number of keys must match the number of values.", nameof(cursor));
+            throw new ArgumentException("The number of keys must match the number of values.", nameof(cursor.Values));
         }
 
-        var cursorExpr = new Expression[cursor.Length];
-        for (var i = 0; i < cursor.Length; i++)
+        var cursorExpr = new Expression[cursor.Values.Length];
+        for (var i = 0; i < cursor.Values.Length; i++)
         {
-            cursorExpr[i] = CreateParameter(cursor[i], keys[i].Expression.ReturnType);
+            cursorExpr[i] = CreateParameter(cursor.Values[i], keys[i].Expression.ReturnType);
         }
 
         var handled = new List<CursorKey>();
@@ -118,7 +118,10 @@ public static Expression<Func<T, bool>> BuildWhereExpression<T>(
             handled.Add(key);
         }
 
-        return Expression.Lambda<Func<T, bool>>(expression!, parameter);
+        var offset = cursor.Offset ?? 0;
+        var reverseOrder = offset < 0; // Reverse order if offset is negative
+
+        return (Expression.Lambda<Func<T, bool>>(expression!, parameter), Math.Abs(offset), reverseOrder);
     }
 
     /// <summary>
@@ -153,7 +156,7 @@ public static Expression<Func<T, bool>> BuildWhereExpression<T>(
     /// If the number of keys is less than one or
     /// the number of order expressions does not match the number of order methods.
     /// </exception>
-    public static Expression<Func<IGrouping<TK, TV>, Group<TK, TV>>> BuildBatchSelectExpression<TK, TV>(
+    public static (Expression<Func<IGrouping<TK, TV>, Group<TK, TV>>> SelectExpression, bool ReverseOrder) BuildBatchSelectExpression<TK, TV>(
         PagingArguments arguments,
         ReadOnlySpan<CursorKey> keys,
         ReadOnlySpan<LambdaExpression> orderExpressions,
@@ -181,14 +184,8 @@ public static Expression<Func<IGrouping<TK, TV>, Group<TK, TV>>> BuildBatchSelec
 
         for (var i = 0; i < orderExpressions.Length; i++)
         {
-            var methodName = orderMethods[i];
+            var methodName = forward ? orderMethods[i] : ReverseOrder(orderMethods[i]);
             var orderExpression = orderExpressions[i];
-
-            if (!forward)
-            {
-                methodName = ReverseOrder(methodName);
-            }
-
             var delegateType = typeof(Func<,>).MakeGenericType(typeof(TV), orderExpression.Body.Type);
             var typedOrderExpression = Expression.Lambda(delegateType, orderExpression.Body, orderExpression.Parameters);
 
@@ -200,16 +197,44 @@ public static Expression<Func<IGrouping<TK, TV>, Group<TK, TV>>> BuildBatchSelec
                 typedOrderExpression);
         }
 
+        var reverseOrder = false;
+        var offset = 0;
+
         if (arguments.After is not null)
         {
             var cursor = CursorParser.Parse(arguments.After, keys);
-            source = BuildBatchWhereExpression<TV>(source, keys, cursor, forward);
+            var (whereExpr, cursorOffset, reverse) = BuildWhereExpression<TV>(keys, cursor, forward: true);
+            source = Expression.Call(typeof(Enumerable), "Where", [typeof(TV)], source, whereExpr);
+            offset = cursorOffset;
+            reverseOrder = reverse;
         }
 
         if (arguments.Before is not null)
         {
             var cursor = CursorParser.Parse(arguments.Before, keys);
-            source = BuildBatchWhereExpression<TV>(source, keys, cursor, forward);
+            var (whereExpr, cursorOffset, reverse) = BuildWhereExpression<TV>(keys, cursor, forward: false);
+            source = Expression.Call(typeof(Enumerable), "Where", [typeof(TV)], source, whereExpr);
+            offset = cursorOffset;
+            reverseOrder = reverse;
+        }
+
+        if (reverseOrder)
+        {
+            source = Expression.Call(
+                typeof(Enumerable),
+                "Reverse",
+                [typeof(TV)],
+                source);
+        }
+
+        if (offset > 0)
+        {
+            source = Expression.Call(
+                typeof(Enumerable),
+                "Skip",
+                [typeof(TV)],
+                source,
+                Expression.Constant(offset));
         }
 
         if (arguments.First is not null)
@@ -248,7 +273,7 @@ public static Expression<Func<IGrouping<TK, TV>, Group<TK, TV>>> BuildBatchSelec
         };
 
         var createGroup = Expression.MemberInit(Expression.New(groupType), bindings);
-        return Expression.Lambda<Func<IGrouping<TK, TV>, Group<TK, TV>>>(createGroup, group);
+        return (Expression.Lambda<Func<IGrouping<TK, TV>, Group<TK, TV>>>(createGroup, group), reverseOrder);
 
         static string ReverseOrder(string method)
             => method switch
@@ -261,12 +286,10 @@ static string ReverseOrder(string method)
             };
 
         static MethodInfo GetEnumerableMethod(string methodName, Type elementType, LambdaExpression keySelector)
-        {
-            return typeof(Enumerable)
+            => typeof(Enumerable)
                 .GetMethods(BindingFlags.Static | BindingFlags.Public)
                 .First(m => m.Name == methodName && m.GetParameters().Length == 2)
                 .MakeGenericMethod(elementType, keySelector.Body.Type);
-        }
     }
 
     private static MethodCallExpression BuildBatchWhereExpression<T>(
@@ -355,7 +378,7 @@ private static MethodCallExpression BuildBatchWhereExpression<T>(
     /// <exception cref="ArgumentNullException"></exception>
     public static OrderRewriterResult ExtractAndRemoveOrder(Expression expression)
     {
-        if(expression is null)
+        if (expression is null)
         {
             throw new ArgumentNullException(nameof(expression));
         }

src/GreenDonut/src/GreenDonut.Data.EntityFramework/Extensions/PagingQueryableExtensions.cs

@@ -75,10 +75,7 @@ public static async ValueTask<Page<T>> ToPageAsync<T>(
         bool includeTotalCount,
         CancellationToken cancellationToken = default)
     {
-        if (source == null)
-        {
-            throw new ArgumentNullException(nameof(source));
-        }
+        ArgumentNullException.ThrowIfNull(source);
 
         source = QueryHelpers.EnsureOrderPropsAreSelected(source);
 
@@ -98,36 +95,79 @@ public static async ValueTask<Page<T>> ToPageAsync<T>(
                 nameof(arguments));
         }
 
+        if (arguments.EnableRelativeCursors
+            && string.IsNullOrEmpty(arguments.After)
+            && string.IsNullOrEmpty(arguments.Before))
+        {
+            includeTotalCount = true;
+        }
+
         var originalQuery = source;
         var forward = arguments.Last is null;
         var requestedCount = int.MaxValue;
+        var reverseOrder = false;
+        var offset = 0;
+        int? totalCount = null;
 
         if (arguments.After is not null)
         {
             var cursor = CursorParser.Parse(arguments.After, keys);
-            source = source.Where(BuildWhereExpression<T>(keys, cursor, true));
+            var (whereExpr, cursorOffset, reverse) = BuildWhereExpression<T>(keys, cursor, true);
+
+            source = source.Where(whereExpr);
+            offset = cursorOffset;
+            reverseOrder = reverse;
+
+            if (!includeTotalCount)
+            {
+                totalCount ??= cursor.TotalCount;
+            }
         }
 
         if (arguments.Before is not null)
         {
             var cursor = CursorParser.Parse(arguments.Before, keys);
-            source = source.Where(BuildWhereExpression<T>(keys, cursor, false));
+            var (whereExpr, cursorOffset, reverse) = BuildWhereExpression<T>(keys, cursor, false);
+
+            source = source.Where(whereExpr);
+            offset = cursorOffset;
+            reverseOrder = reverse;
+
+            if (!includeTotalCount)
+            {
+                totalCount ??= cursor.TotalCount;
+            }
+        }
+
+        // Reverse order if offset is negative
+        if (reverseOrder)
+        {
+            source = source.Reverse();
         }
 
         if (arguments.First is not null)
         {
+            if (offset > 0)
+            {
+                source = source.Skip(offset * arguments.First.Value);
+            }
+
             source = source.Take(arguments.First.Value + 1);
             requestedCount = arguments.First.Value;
         }
 
         if (arguments.Last is not null)
         {
+            if (offset > 0)
+            {
+                source = source.Skip(offset * arguments.Last.Value);
+            }
+
             source = source.Reverse().Take(arguments.Last.Value + 1);
             requestedCount = arguments.Last.Value;
         }
 
         var builder = ImmutableArray.CreateBuilder<T>();
-        int? totalCount = null;
         var fetchCount = 0;
 
         if (includeTotalCount)
@@ -142,12 +182,12 @@ public static async ValueTask<Page<T>> ToPageAsync<T>(
                 totalCount ??= item.TotalCount;
                 fetchCount++;
 
+                builder.Add(item.Item);
+
                 if (fetchCount > requestedCount)
                 {
                     break;
                 }
-
-                builder.Add(item.Item);
             }
         }
         else
@@ -159,12 +199,12 @@ public static async ValueTask<Page<T>> ToPageAsync<T>(
             {
                 fetchCount++;
 
+                builder.Add(item);
+
                 if (fetchCount > requestedCount)
                 {
                     break;
                 }
-
-                builder.Add(item);
             }
         }
 
@@ -173,11 +213,23 @@ public static async ValueTask<Page<T>> ToPageAsync<T>(
             return Page<T>.Empty;
         }
 
-        if (!forward)
+        if (!forward ^ reverseOrder)
         {
             builder.Reverse();
         }
 
+        if (builder.Count > requestedCount)
+        {
+            if (!forward ^ reverseOrder)
+            {
+                builder.RemoveAt(0);
+            }
+            else
+            {
+                builder.RemoveAt(requestedCount);
+            }
+        }
+
         return CreatePage(builder.ToImmutable(), arguments, keys, fetchCount, totalCount);
     }
 
@@ -367,6 +419,13 @@ public static async ValueTask<Dictionary<TKey, Page<TValue>>> ToBatchPageAsync<T
                 nameof(arguments));
         }
 
+        if (arguments.EnableRelativeCursors
+            && string.IsNullOrEmpty(arguments.After)
+            && string.IsNullOrEmpty(arguments.Before))
+        {
+            includeTotalCount = true;
+        }
+
         Dictionary<TKey, int>? counts = null;
         if (includeTotalCount)
         {
@@ -383,7 +442,7 @@ public static async ValueTask<Dictionary<TKey, Page<TValue>>> ToBatchPageAsync<T
 
         var forward = arguments.Last is null;
         var requestedCount = int.MaxValue;
-        var selectExpression =
+        var (selectExpression, reverseOrder) =
             BuildBatchSelectExpression<TKey, TElement>(
                 arguments,
                 keys,
@@ -405,6 +464,11 @@ public static async ValueTask<Dictionary<TKey, Page<TValue>>> ToBatchPageAsync<T
             .WithCancellation(cancellationToken)
             .ConfigureAwait(false))
         {
+            if (reverseOrder)
+            {
+                item.Items.Reverse();
+            }
+
             if (item.Items.Count == 0)
             {
                 map.Add(item.Key, Page<TValue>.Empty);
@@ -520,6 +584,17 @@ private static Page<T> CreatePage<T>(
             hasNext = true;
         }
 
+        if (arguments.EnableRelativeCursors && totalCount is not null)
+        {
+            return new Page<T>(
+                items,
+                hasNext,
+                hasPrevious,
+                (item, o, p, c) => CursorFormatter.Format(item, keys, new CursorPageInfo(o, p, c)),
+                0,
+                totalCount.Value);
+        }
+
         return new Page<T>(
             items,
             hasNext,
@@ -532,7 +607,7 @@ private static CursorKey[] ParseDataSetKeys<T>(IQueryable<T> source)
     {
         var parser = new CursorKeyParser();
         parser.Visit(source.Expression);
-        return parser.Keys.ToArray();
+        return [.. parser.Keys];
     }
 
     private sealed class InterceptorHolder

src/GreenDonut/src/GreenDonut.Data.Primitives/GreenDonut.Data.Primitives.csproj

@@ -7,4 +7,8 @@
     <Description>This package contains the basic building blocks of the DataLoader linq query integration.</Description>
   </PropertyGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="GreenDonut.Data.EntityFramework" />
+  </ItemGroup>
+
 </Project>