Merge pull request #15390 from Automattic/csfle

CSFLE support

Author: vkarpov15

.eslintrc.js

@@ -14,7 +14,8 @@ module.exports = {
     '!.*',
     'node_modules',
     '.git',
-    'data'
+    'data',
+    '.config'
   ],
   overrides: [
     {

.github/workflows/encryption-tests.yml

@@ -4,7 +4,7 @@ on:
   push: 
     branches: ['master']
   pull_request:
-    branches: [ 'master' ]
+    branches: [ 'master', 'csfle' ]
   workflow_dispatch: {}
 
 permissions:

.gitignore

@@ -72,4 +72,4 @@ notes.md
 list.out
 
 data
-*.pid
+fle-cluster-config.json

docs/field-level-encryption.md

@@ -16,11 +16,141 @@ The resulting document will look similar to the following to a client that doesn
 
 You can read more about CSFLE on the [MongoDB CSFLE documentation](https://www.mongodb.com/docs/manual/core/csfle/) and [this blog post about CSFLE in Node.js](https://www.mongodb.com/developer/languages/javascript/client-side-field-level-encryption-csfle-mongodb-node/).
 
-Note that Mongoose does **not** currently have any Mongoose-specific APIs for CSFLE.
-Mongoose defers all CSFLE-related work to the MongoDB Node.js driver, so the [`autoEncryption` option](https://mongodb.github.io/node-mongodb-native/5.6/interfaces/AutoEncryptionOptions.html) for `mongoose.connect()` and `mongoose.createConnection()` is where you put all CSFLE-related configuration.
-Mongoose schemas currently don't support CSFLE configuration.
+## Automatic FLE in Mongoose
 
-## Setting Up Field Level Encryption with Mongoose
+Mongoose supports the declaration of encrypted schemas - schemas that, when connected to a model, utilize MongoDB's Client Side
+Field Level Encryption or Queryable Encryption under the hood.  Mongoose automatically generates either an `encryptedFieldsMap` or a
+`schemaMap` when instantiating a MongoClient and encrypts fields on write and decrypts fields on reads.
+
+### Encryption types
+
+MongoDB has two different automatic encryption implementations: client side field level encryption (CSFLE) and queryable encryption (QE).  
+See [choosing an in-use encryption approach](https://www.mongodb.com/docs/v7.3/core/queryable-encryption/about-qe-csfle/#choosing-an-in-use-encryption-approach).
+
+###  Declaring Encrypted Schemas
+
+The following schema declares two properties, `name` and `ssn`.  `ssn` is encrypted using queryable encryption, and
+is configured for equality queries:
+
+```javascript
+const encryptedUserSchema = new Schema({ 
+  name: String,
+  ssn: { 
+    type: String, 
+    // 1
+    encrypt: { 
+      keyId: '<uuid string of key id>',
+      queries: 'equality'
+    }
+  }
+  // 2
+}, { encryptionType: 'queryableEncryption' });
+```
+
+To declare a field as encrypted, you must:
+
+1. Annotate the field with encryption metadata in the schema definition
+2. Choose an encryption type for the schema and configure the schema for the encryption type
+
+Not all schematypes are supported for CSFLE and QE.  For an overview of supported BSON types, refer to MongoDB's documentation.
+
+### Registering Models
+
+Encrypted schemas can be registered on the global mongoose object or on a specific connection, so long as models are registered before the connection
+is established:
+
+```javascript
+// specific connection
+const GlobalUserModel = mongoose.model('User', encryptedUserSchema);
+
+// specific connection
+const connection = mongoose.createConnection();
+const UserModel = connection.model('User', encryptedUserSchema);
+```
+
+### Connecting and configuring encryption options
+
+Field level encryption in Mongoose works by generating the encryption schema that the MongoDB driver expects for each encrypted model on the connection.  This happens automatically when the model's connection is established.
+
+Queryable encryption and CSFLE require all the same configuration as outlined in the [MongoDB encryption in-use documentation](https://www.mongodb.com/docs/manual/core/security-in-use-encryption/), except for the schemaMap or encryptedFieldsMap options.
+
+```javascript
+const keyVaultNamespace = 'client.encryption';
+const kmsProviders = { local: { key } };
+await connection.openUri(`mongodb://localhost:27017`, {
+  // Configure auto encryption
+  autoEncryption: {
+    keyVaultNamespace: 'datakeys.datakeys',
+    kmsProviders
+  }
+});
+```
+
+Once the connection is established, Mongoose's operations will work as usual.  Writes are encrypted automatically by the MongoDB driver prior to sending them to the server and reads are decrypted by the driver after fetching documents from the server.
+
+### Discriminators
+
+Discriminators are supported for encrypted models as well:
+
+```javascript
+const connection = createConnection();
+
+const schema = new Schema({
+  name: {
+    type: String, encrypt: { keyId }
+  }
+}, {
+  encryptionType: 'queryableEncryption'
+});
+
+const Model = connection.model('BaseUserModel', schema);
+const ModelWithAge = model.discriminator('ModelWithAge', new Schema({
+  age: {
+    type: Int32, encrypt: { keyId: keyId2 }
+  }
+}, {
+  encryptionType: 'queryableEncryption'
+}));
+
+const ModelWithBirthday = model.discriminator('ModelWithBirthday', new Schema({
+  dob: {
+    type: Int32, encrypt: { keyId: keyId3 }
+  }
+}, {
+  encryptionType: 'queryableEncryption'
+}));
+```
+
+When generating encryption schemas, Mongoose merges all discriminators together for all of the discriminators declared on the same namespace.  As a result, discriminators that declare the same key with different types are not supported.  Furthermore, all discriminators for the same namespace must share the same encryption type - it is not possible to configure discriminators on the same model for both CSFLE and Queryable Encryption.
+
+## Managing Data Keys
+
+Mongoose provides a convenient API to obtain a [ClientEncryption](https://mongodb.github.io/node-mongodb-native/Next/classes/ClientEncryption.html)
+object configured to manage data keys in the key vault.  A client encryption can be obtained with the `Model.clientEncryption()` helper:
+
+```javascript
+const connection = createConnection();
+
+const schema = new Schema({
+  name: {
+    type: String, encrypt: { keyId }
+  }
+}, {
+  encryptionType: 'queryableEncryption'
+});
+
+const Model = connection.model('BaseUserModel', schema);
+await connection.openUri(`mongodb://localhost:27017`, {
+  autoEncryption: {
+    keyVaultNamespace: 'datakeys.datakeys',
+    kmsProviders: { local: '....' }
+  }
+});
+
+const clientEncryption = Model.clientEncryption();
+```
+
+## Manual FLE in Mongoose
 
 First, you need to install the [mongodb-client-encryption npm package](https://www.npmjs.com/package/mongodb-client-encryption).
 This is MongoDB's official package for setting up encryption keys.

lib/drivers/node-mongodb-native/connection.js

@@ -12,6 +12,7 @@ const pkg = require('../../../package.json');
 const processConnectionOptions = require('../../helpers/processConnectionOptions');
 const setTimeout = require('../../helpers/timers').setTimeout;
 const utils = require('../../utils');
+const Schema = require('../../schema');
 
 /**
  * A [node-mongodb-native](https://github.com/mongodb/node-mongodb-native) connection implementation.
@@ -320,6 +321,20 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio
     };
   }
 
+  const { schemaMap, encryptedFieldsMap } = this._buildEncryptionSchemas();
+
+  if ((Object.keys(schemaMap).length > 0 || Object.keys(encryptedFieldsMap).length) && !options.autoEncryption) {
+    throw new Error('Must provide `autoEncryption` when connecting with encrypted schemas.');
+  }
+
+  if (Object.keys(schemaMap).length > 0) {
+    options.autoEncryption.schemaMap = schemaMap;
+  }
+
+  if (Object.keys(encryptedFieldsMap).length > 0) {
+    options.autoEncryption.encryptedFieldsMap = encryptedFieldsMap;
+  }
+
   this.readyState = STATES.connecting;
   this._connectionString = uri;
 
@@ -343,6 +358,56 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio
   return this;
 };
 
+/**
+ * Given a connection, which may or may not have encrypted models, build
+ * a schemaMap and/or an encryptedFieldsMap for the connection, combining all models
+ * into a single schemaMap and encryptedFields map.
+ *
+ * @returns the generated schemaMap and encryptedFieldsMap
+  */
+NativeConnection.prototype._buildEncryptionSchemas = function() {
+  const qeMappings = {};
+  const csfleMappings = {};
+
+  const encryptedModels = Object.values(this.models).filter(model => model.schema._hasEncryptedFields());
+
+  // If discriminators are configured for the collection, there might be multiple models
+  // pointing to the same namespace.  For this scenario, we merge all the schemas for each namespace
+  // into a single schema and then generate a schemaMap/encryptedFieldsMap for the combined schema.
+  for (const model of encryptedModels) {
+    const { schema, collection: { collectionName } } = model;
+    const namespace = `${this.$dbName}.${collectionName}`;
+    const mappings = schema.encryptionType() === 'csfle' ? csfleMappings : qeMappings;
+
+    mappings[namespace] ??= new Schema({}, { encryptionType: schema.encryptionType() });
+
+    const isNonRootDiscriminator = schema.discriminatorMapping && !schema.discriminatorMapping.isRoot;
+    if (isNonRootDiscriminator) {
+      const rootSchema = schema._baseSchema;
+      schema.eachPath((pathname) => {
+        if (rootSchema.path(pathname)) return;
+        if (!mappings[namespace]._hasEncryptedField(pathname)) return;
+
+        throw new Error(`Cannot have duplicate keys in discriminators with encryption. key=${pathname}`);
+      });
+    }
+
+    mappings[namespace].add(schema);
+  }
+
+  const schemaMap = Object.fromEntries(Object.entries(csfleMappings).map(
+    ([namespace, schema]) => ([namespace, schema._buildSchemaMap()])
+  ));
+
+  const encryptedFieldsMap = Object.fromEntries(Object.entries(qeMappings).map(
+    ([namespace, schema]) => ([namespace, schema._buildEncryptedFields()])
+  ));
+
+  return {
+    schemaMap, encryptedFieldsMap
+  };
+};
+
 /*!
  * ignore
  */

lib/drivers/node-mongodb-native/index.js

@@ -7,3 +7,4 @@
 exports.BulkWriteResult = require('./bulkWriteResult');
 exports.Collection = require('./collection');
 exports.Connection = require('./connection');
+exports.ClientEncryption = require('mongodb').ClientEncryption;

lib/helpers/model/discriminator.js

@@ -17,6 +17,51 @@ const CUSTOMIZABLE_DISCRIMINATOR_OPTIONS = {
   methods: true
 };
 
+/**
+ * Validate fields declared on the child schema when either schema is configured for encryption.  Specifically, this function ensures that:
+ *
+ * - any encrypted fields are declared on exactly one of the schemas (not both)
+ * - encrypted fields cannot be declared on either the parent or child schema, where the other schema declares the same field without encryption.
+ *
+ * @param {Schema} parentSchema
+ * @param {Schema} childSchema
+ */
+function validateDiscriminatorSchemasForEncryption(parentSchema, childSchema) {
+  if (parentSchema.encryptionType() == null && childSchema.encryptionType() == null) return;
+
+  const allSharedNestedPaths = setIntersection(
+    allNestedPaths(parentSchema),
+    allNestedPaths(childSchema)
+  );
+
+  for (const path of allSharedNestedPaths) {
+    if (parentSchema._hasEncryptedField(path) && childSchema._hasEncryptedField(path)) {
+      throw new Error(`encrypted fields cannot be declared on both the base schema and the child schema in a discriminator. path=${path}`);
+    }
+
+    if (parentSchema._hasEncryptedField(path) || childSchema._hasEncryptedField(path)) {
+      throw new Error(`encrypted fields cannot have the same path as a non-encrypted field for discriminators. path=${path}`);
+    }
+  }
+
+  function allNestedPaths(schema) {
+    return [...Object.keys(schema.paths), ...Object.keys(schema.singleNestedPaths)];
+  }
+
+  /**
+   * @param {Iterable<string>} i1
+   * @param {Iterable<string>} i2
+   */
+  function* setIntersection(i1, i2) {
+    const s1 = new Set(i1);
+    for (const item of i2) {
+      if (s1.has(item)) {
+        yield item;
+      }
+    }
+  }
+}
+
 /*!
  * ignore
  */
@@ -80,6 +125,8 @@ module.exports = function discriminator(model, name, schema, tiedValue, applyPlu
     value = tiedValue;
   }
 
+  validateDiscriminatorSchemasForEncryption(model.schema, schema);
+
   function merge(schema, baseSchema) {
     // Retain original schema before merging base schema
     schema._baseSchema = baseSchema;

lib/model.js

@@ -69,7 +69,6 @@ const minimize = require('./helpers/minimize');
 const MongooseBulkSaveIncompleteError = require('./error/bulkSaveIncompleteError');
 const ObjectExpectedError = require('./error/objectExpected');
 const decorateBulkWriteResult = require('./helpers/model/decorateBulkWriteResult');
-
 const modelCollectionSymbol = Symbol('mongoose#Model#collection');
 const modelDbSymbol = Symbol('mongoose#Model#db');
 const modelSymbol = require('./helpers/symbols').modelSymbol;
@@ -4893,6 +4892,39 @@ Model.compile = function compile(name, schema, collectionName, connection, base)
   return model;
 };
 
+/**
+ * If auto encryption is enabled, returns a ClientEncryption instance that is configured with the same settings that
+ * Mongoose's underlying MongoClient is using.  If the client has not yet been configured, returns null.
+ *
+ * @returns {ClientEncryption | null}
+ */
+Model.clientEncryption = function clientEncryption() {
+  const ClientEncryption = this.base.driver.get().ClientEncryption;
+  if (!ClientEncryption) {
+    throw new Error('The mongodb driver must be used to obtain a ClientEncryption object.');
+  }
+
+  const client = this.collection?.conn?.client;
+
+  if (!client) return null;
+
+  const autoEncryptionOptions = client.options.autoEncryption;
+
+  if (!autoEncryptionOptions) return null;
+
+  const {
+    keyVaultNamespace,
+    keyVaultClient,
+    kmsProviders,
+    credentialProviders,
+    proxyOptions,
+    tlsOptions
+  } = autoEncryptionOptions;
+  return new ClientEncryption(keyVaultClient ?? client,
+    { keyVaultNamespace, kmsProviders, credentialProviders, proxyOptions, tlsOptions }
+  );
+};
+
 /**
  * Update this model to use the new connection, including updating all internal
  * references and creating a new `Collection` instance using the new connection.