fix: Strict Projection Object Typing (#13993)

* Projection Type Should handle all exclusion and inclusions

* fix tests

* remove comment

* remove test

* fields which are not in schema should be allowed

* support $slice and $elemMatch

* $slice and $elemMatch should be allowed with inclusion and exclusion queries

* feat: ✨ support dot notation and a combination of object and dot notation projection

* should not project methods inside Date type

* use replacer type

Author: pshaddel

test/types/queries.test.ts

@@ -15,7 +15,8 @@ import {
   QuerySelector,
   InferSchemaType,
   ProjectionFields,
-  QueryOptions
+  QueryOptions,
+  ProjectionType
 } from 'mongoose';
 import { ObjectId } from 'mongodb';
 import { expectAssignable, expectError, expectNotAssignable, expectType } from 'tsd';
@@ -54,6 +55,9 @@ interface ISubdoc {
   myId?: Types.ObjectId;
   id?: number;
   tags?: string[];
+  profiles: {
+    name?: string
+  }
 }
 
 interface ITest {
@@ -154,6 +158,37 @@ const p1: Record<string, number> = Test.find().projection('age docs.id');
 const p2: Record<string, number> | null = Test.find().projection();
 const p3: null = Test.find().projection(null);
 
+expectError(Test.find({ }, { name: 'ss' })); // Only 0 and 1 are allowed
+expectError(Test.find({ }, { name: 3 })); // Only 0 and 1 are allowed
+expectError(Test.find({ }, { name: true, age: false, endDate: true, tags: 1 })); // Exclusion in a inclusion projection is not allowed
+expectError(Test.find({ }, { name: true, age: false, endDate: true })); // Inclusion in a exclusion projection is not allowed
+expectError(Test.find({ }, { name: false, age: false, tags: false, child: { name: false }, docs: { myId: false, id: true } })); // Inclusion in a exclusion projection is not allowed in nested objects and arrays
+expectError(Test.find({ }, { tags: { something: 1 } })); // array of strings or numbers should only be allowed to be a boolean or 1 and 0
+Test.find({}, { name: true, age: true, endDate: true, tags: 1, child: { name: true }, docs: { myId: true, id: true } }); // This should be allowed
+Test.find({}, { name: 1, age: 1, endDate: 1, tags: 1, child: { name: 1 }, docs: { myId: 1, id: 1 } }); // This should be allowed
+Test.find({}, { _id: 0, name: 1, age: 1, endDate: 1, tags: 1, child: 1, docs: 1 }); // _id is an exception and should be allowed to be excluded
+Test.find({}, { name: 0, age: 0, endDate: 0, tags: 0, child: 0, docs: 0 }); // This should be allowed
+Test.find({}, { name: 0, age: 0, endDate: 0, tags: 0, child: { name: 0 }, docs: { myId: 0, id: 0 } }); // This should be allowed
+Test.find({}, { name: 1, age: 1, _id: 0 }); // This should be allowed since _id is an exception
+Test.find({}, { someOtherField: 1 }); // This should be allowed since it's not a field in the schema
+expectError(Test.find({}, { name: { $slice: 1 } })); // $slice should only be allowed on arrays
+Test.find({}, { tags: { $slice: 1 } }); // $slice should be allowed on arrays
+Test.find({}, { tags: { $slice: [1, 2] } }); // $slice with the format of [ <number to skip>, <number to return> ] should also be allowed on arrays
+expectError(Test.find({}, { age: { $elemMatch: {} } })); // $elemMatch should not be allowed on non arrays
+Test.find({}, { tags: { $elemMatch: {} } }); // $elemMatch should be allowed on arrays
+expectError(Test.find({}, { tags: { $slice: 1, $elemMatch: {} } })); // $elemMatch and $slice should not be allowed together
+Test.find({}, { age: 1, tags: { $slice: 5 } }); // $slice should be allowed in inclusion projection
+Test.find({}, { age: 0, tags: { $slice: 5 } }); // $slice should be allowed in exclusion projection
+Test.find({}, { age: 1, tags: { $elemMatch: {} } }); // $elemMatch should be allowed in inclusion projection
+Test.find({}, { age: 0, tags: { $elemMatch: {} } }); // $elemMatch should be allowed in exclusion projection
+expectError(Test.find({}, { 'docs.id': 11 })); // Dot notation should be allowed and does not accept any
+expectError(Test.find({}, { docs: { id: '1' } })); // Dot notation should be able to use a combination with objects
+Test.find({}, { docs: { id: false } }); // Dot notation should be allowed with valid values - should correctly handle arrays
+Test.find({}, { docs: { id: true } }); // Dot notation should be allowed with valid values - should correctly handle arrays
+Test.find({}, { child: 1 }); // Dot notation should be able to use a combination with objects
+Test.find({}, { 'docs.profiles': { name: 1 } }); // should support a combination of dot notation and objects
+expectError(Test.find({}, { 'docs.profiles': { name: 'aa' } })); // should support a combination of dot notation and objects
+expectError(Test.find({}, { endDate: { toString: 1 } }));
 // Sorting
 Test.find().sort();
 Test.find().sort('-name');

test/types/schema.test.ts

@@ -760,7 +760,7 @@ function pluginOptions() {
   }
 
   const schema = new Schema({});
-  expectType<Schema<any>>(schema.plugin(pluginFunction)); // test that chaining would be possible
+  expectType<typeof schema>(schema.plugin(pluginFunction)); // test that chaining would be possible
 
   // could not add strict tests that the parameters are inferred correctly, because i dont know how this would be done in tsd
 

types/index.d.ts

@@ -589,9 +589,96 @@ declare module 'mongoose' {
 
   export type ReturnsNewDoc = { new: true } | { returnOriginal: false } | { returnDocument: 'after' };
 
-  export type ProjectionElementType = number | string;
-  export type ProjectionType<T> = { [P in keyof T]?: ProjectionElementType } | AnyObject | string;
+  type ArrayOperators = { $slice: number | [number, number]; $elemMatch?: undefined } | { $elemMatch: object; $slice?: undefined };
+  type Projector<T, Element> = T extends Array<infer U>
+    ? Projector<U, Element> | ArrayOperators
+    : T extends object
+      ? {
+        [K in keyof T]?: T[K] extends object ? Projector<T[K], Element> | Element : Element;
+      }
+      : Element;
+  type _IDType = { _id?: boolean | 1 | 0 };
+  type InclusionProjection<T> = NestedPartial<Projector<NestedRequired<DotKeys<T>>, true | 1> & _IDType>;
+  type ExclusionProjection<T> = NestedPartial<Projector<NestedRequired<DotKeys<T>>, false | 0> & _IDType>;
+  type ProjectionUnion<T> = InclusionProjection<T> | ExclusionProjection<T>;
+
+  type NestedRequired<T> = T extends Array<infer U>
+    ? Array<NestedRequired<U>>
+    : T extends object
+      ? {
+        [K in keyof T]-?: NestedRequired<T[K]>;
+      }
+      : T;
+  type NestedPartial<T> = T extends Array<infer U>
+    ? Array<NestedPartial<U>>
+    : T extends object
+      ? {
+        [K in keyof T]?: NestedPartial<T[K]>;
+      }
+      : T;
+
+  /** https://stackoverflow.com/questions/58434389/typescript-deep-keyof-of-a-nested-object/58436959#58436959 for dot nested implementation it is used and then modified */
+  type DotPrefix<T extends string> = T extends '' ? '' : `.${T}`;
+
+  /** https://stackoverflow.com/questions/75419012/how-do-i-limit-the-level-of-nesting-on-a-recursive-type-in-typescript */
+  type Length<T extends unknown[]> = T extends { length: infer L } ? L : never;
+  type BuildTuple<L extends number, T extends unknown[] = []> = T extends { length: L } ? T : BuildTuple<L, [...T, unknown]>;
+  type MinusOne<N extends number> = BuildTuple<N> extends [...infer U, unknown] ? Length<U> : never;
 
+  /**
+   * Generates a union from dot path in object
+   * We have to give it the Depth to prevent infinite recursion and also prevent slow compilation when the object is too much nested
+   */
+  type DotNestedKeys<T, Depth extends number> = (
+    Depth extends 0
+      ? never
+      : T extends object
+        ? { [K in Exclude<keyof T, symbol>]: `${K}` | `${K}${DotPrefix<DotNestedKeys<T[K], MinusOne<Depth>>>}` }[Exclude<keyof T, symbol>]
+        : ''
+  ) extends infer D
+    ? Extract<D, string>
+    : never;
+  type FindDottedPathType<T, Path extends string> = Path extends `${infer K}.${infer R}`
+    ? K extends keyof T
+      ? FindDottedPathType<T[K] extends Array<infer U> ? U : T[K], R>
+      : never
+    : Path extends keyof T
+      ? T[Path]
+      : never;
+  type ExtractNestedArrayElement<T> = T extends (infer U)[]
+    ? ExtractNestedArrayElement<U>
+    : T extends object
+      ? { [K in keyof T]: ExtractNestedArrayElement<T[K]> }
+      : T;
+  type DotnotationMaximumDepth = 4;
+  /**
+   * Create dot path for nested objects
+   * It creates dot notation for arrays similar to mongodb. For example { a: { c: { b: number}[] }[] } => 'a.c.b': number, 'a.c': { b: number }[]
+   */
+  type DotKeys<DocType> = {
+    [key in DotNestedKeys<ExtractNestedArrayElement<DocType>, DotnotationMaximumDepth>]?: FindDottedPathType<NestedRequired<DocType>, key>;
+  };
+
+  /**
+   * This types are equivalent to primary types
+   */
+  type SpecialTypes = DateSchemaDefinition | Date | globalThis.Date | DateConstructor | Types.Buffer | Types.Decimal128 | Types.Buffer | BooleanSchemaDefinition | NumberSchemaDefinition;
+  type Replacer<T> = T extends SpecialTypes ? string : T;
+  /**
+   * Date type is like a Primitiv type for us and we do not want to project something inside it.
+   * ObjectId is also similar.
+   */
+  type ReplaceSpecialTypes<T> = T extends SpecialTypes
+    ? Replacer<T>
+    : T extends Array<infer U>
+      ? Array<ReplaceSpecialTypes<U>>
+      : T extends object
+        ? {
+          [K in keyof T]?: ReplaceSpecialTypes<T[K]>;
+        }
+        : Replacer<T>;
+
+  export type ProjectionType<T> = (ProjectionUnion<ReplaceSpecialTypes<T>> & AnyObject) | string | ((...agrs: any) => any);
   export type SortValues = SortOrder;
 
   export type SortOrder = -1 | 1 | 'asc' | 'ascending' | 'desc' | 'descending';

types/query.d.ts

@@ -95,7 +95,7 @@ declare module 'mongoose' {
     updatedAt?: boolean;
   }
 
-  interface QueryOptions<DocType = unknown> extends
+  interface QueryOptions<DocType = any> extends
     PopulateOption,
     SessionOption {
     arrayFilters?: { [key: string]: any }[];
@@ -122,7 +122,7 @@ declare module 'mongoose' {
     new?: boolean;
 
     overwriteDiscriminatorKey?: boolean;
-    projection?: ProjectionType<DocType>;
+    projection?: ProjectionType<any>;
     /**
      * if true, returns the full ModifyResult rather than just the document
      */