Merge pull request #15214 from Automattic/8.10

8.10

.eslintrc.js

@@ -14,7 +14,8 @@ module.exports = {
     '**/docs/js/native.js',
     '!.*',
     'node_modules',
-    '.git'
+    '.git',
+    'data'
   ],
   overrides: [
     {
@@ -104,7 +105,7 @@ module.exports = {
     // 'markdown'
   ],
   parserOptions: {
-    ecmaVersion: 2020
+    ecmaVersion: 2022
   },
   env: {
     node: true,

.github/workflows/encryption-tests.yml

@@ -0,0 +1,39 @@
+name: Encryption Tests 
+
+on:
+  push: 
+    branches: ['master']
+  pull_request:
+    branches: [ 'master' ]
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+  id-token: write
+
+jobs:
+  run-tests:
+    permissions:
+      # required for all workflows
+      security-events: write
+      id-token: write
+      contents: write
+    runs-on: ubuntu-latest
+    name: Encryption tests
+    env:
+      FORCE_COLOR: true
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Setup node
+        uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+        with:
+          node-version: 22
+      - name: Install Dependencies
+        run: npm install
+      - name: Install mongodb-client-encryption
+        run: npm install mongodb-client-encryption
+      - name: Setup Tests
+        run: npm run setup-test-encryption
+      - name: Run Tests
+        run: npm run test-encryption

.github/workflows/test.yml

@@ -108,7 +108,7 @@ jobs:
       - name: Setup Deno
         uses: denoland/setup-deno@v2
         with:
-          deno-version: v1.37.x
+          deno-version: v2.1.x
       - run: deno --version
       - run: npm install
       - name: Run Deno tests

.gitignore

@@ -67,3 +67,6 @@ examples/ecommerce-netlify-functions/.netlify/state.json
 
 notes.md
 list.out
+
+data
+*.pid

CONTRIBUTING.md

@@ -46,6 +46,13 @@ If you have a question about Mongoose (not a bug report) please post it to eithe
   * execute `npm run test-tsd` to run the typescript tests
   * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time.
   * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command.
+* in order to run tests that require an cluster with encryption locally, run `npm run setup-test-encryption` followed by `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file.
+  * These scripts can take a few minutes to run.
+  * To change an encryption configuration, it is recommended to follow these steps:
+    * Edit the variables in `scripts/configure-cluster-with-encryption.sh` with your desired configuration.
+    * Restart your shell.
+    * Delete the `data/` directory if it exists.
+    * Finally, run the configuration script.
 
 ## Documentation
 

lib/aggregate.js

@@ -13,6 +13,7 @@ const getConstructorName = require('./helpers/getConstructorName');
 const prepareDiscriminatorPipeline = require('./helpers/aggregate/prepareDiscriminatorPipeline');
 const stringifyFunctionOperators = require('./helpers/aggregate/stringifyFunctionOperators');
 const utils = require('./utils');
+const { modelSymbol } = require('./helpers/symbols');
 const read = Query.prototype.read;
 const readConcern = Query.prototype.readConcern;
 
@@ -46,13 +47,17 @@ const validRedactStringValues = new Set(['$$DESCEND', '$$PRUNE', '$$KEEP']);
  * @see MongoDB https://www.mongodb.com/docs/manual/applications/aggregation/
  * @see driver https://mongodb.github.io/node-mongodb-native/4.9/classes/Collection.html#aggregate
  * @param {Array} [pipeline] aggregation pipeline as an array of objects
- * @param {Model} [model] the model to use with this aggregate.
+ * @param {Model|Connection} [modelOrConn] the model or connection to use with this aggregate.
  * @api public
  */
 
-function Aggregate(pipeline, model) {
+function Aggregate(pipeline, modelOrConn) {
   this._pipeline = [];
-  this._model = model;
+  if (modelOrConn == null || modelOrConn[modelSymbol]) {
+    this._model = modelOrConn;
+  } else {
+    this._connection = modelOrConn;
+  }
   this.options = {};
 
   if (arguments.length === 1 && Array.isArray(pipeline)) {
@@ -1041,12 +1046,24 @@ Aggregate.prototype.pipeline = function() {
  */
 
 Aggregate.prototype.exec = async function exec() {
-  if (!this._model) {
+  if (!this._model && !this._connection) {
     throw new Error('Aggregate not bound to any Model');
   }
   if (typeof arguments[0] === 'function') {
     throw new MongooseError('Aggregate.prototype.exec() no longer accepts a callback');
   }
+
+  if (this._connection) {
+    if (!this._pipeline.length) {
+      throw new MongooseError('Aggregate has empty pipeline');
+    }
+
+    this._optionsForExec();
+
+    const cursor = await this._connection.client.db().aggregate(this._pipeline, this.options);
+    return await cursor.toArray();
+  }
+
   const model = this._model;
   const collection = this._model.collection;
 

lib/collection.js

@@ -29,7 +29,7 @@ function Collection(name, conn, opts) {
   this.collectionName = name;
   this.conn = conn;
   this.queue = [];
-  this.buffer = true;
+  this.buffer = !conn?._hasOpened;
   this.emitter = new EventEmitter();
 
   if (STATES.connected === this.conn.readyState) {
@@ -311,13 +311,7 @@ Collection.prototype._getBufferTimeoutMS = function _getBufferTimeoutMS() {
   if (opts && opts.schemaUserProvidedOptions != null && opts.schemaUserProvidedOptions.bufferTimeoutMS != null) {
     return opts.schemaUserProvidedOptions.bufferTimeoutMS;
   }
-  if (conn.config.bufferTimeoutMS != null) {
-    return conn.config.bufferTimeoutMS;
-  }
-  if (conn.base != null && conn.base.get('bufferTimeoutMS') != null) {
-    return conn.base.get('bufferTimeoutMS');
-  }
-  return 10000;
+  return conn._getBufferTimeoutMS();
 };
 
 /*!

lib/connection.js

@@ -824,12 +824,56 @@ Connection.prototype.dropCollection = async function dropCollection(collection)
 
 Connection.prototype._waitForConnect = async function _waitForConnect() {
   if ((this.readyState === STATES.connecting || this.readyState === STATES.disconnected) && this._shouldBufferCommands()) {
-    await new Promise(resolve => {
-      this._queue.push({ fn: resolve });
-    });
+    const bufferTimeoutMS = this._getBufferTimeoutMS();
+    let timeout = null;
+    let timedOut = false;
+    // The element that this function pushes onto `_queue`, stored to make it easy to remove later
+    const queueElement = {};
+    await Promise.race([
+      new Promise(resolve => {
+        queueElement.fn = resolve;
+        this._queue.push(queueElement);
+      }),
+      new Promise(resolve => {
+        timeout = setTimeout(
+          () => {
+            timedOut = true;
+            resolve();
+          },
+          bufferTimeoutMS
+        );
+      })
+    ]);
+
+    if (timedOut) {
+      const index = this._queue.indexOf(queueElement);
+      if (index !== -1) {
+        this._queue.splice(index, 1);
+      }
+      const message = 'Connection operation buffering timed out after ' + bufferTimeoutMS + 'ms';
+      throw new MongooseError(message);
+    } else if (timeout != null) {
+      // Not strictly necessary, but avoid the extra overhead of creating a new MongooseError
+      // in case of success
+      clearTimeout(timeout);
+    }
   }
 };
 
+/*!
+ * Get the default buffer timeout for this connection
+ */
+
+Connection.prototype._getBufferTimeoutMS = function _getBufferTimeoutMS() {
+  if (this.config.bufferTimeoutMS != null) {
+    return this.config.bufferTimeoutMS;
+  }
+  if (this.base != null && this.base.get('bufferTimeoutMS') != null) {
+    return this.base.get('bufferTimeoutMS');
+  }
+  return 10000;
+};
+
 /**
  * Helper for MongoDB Node driver's `listCollections()`.
  * Returns an array of collection objects.
@@ -1156,6 +1200,10 @@ Connection.prototype.close = async function close(force) {
     this.$wasForceClosed = !!force;
   }
 
+  if (this._lastHeartbeatAt != null) {
+    this._lastHeartbeatAt = null;
+  }
+
   for (const model of Object.values(this.models)) {
     // If manually disconnecting, make sure to clear each model's `$init`
     // promise, so Mongoose knows to re-run `init()` in case the
@@ -1742,6 +1790,18 @@ Connection.prototype.syncIndexes = async function syncIndexes(options = {}) {
  * @api public
  */
 
+/**
+ * Runs a [db-level aggregate()](https://www.mongodb.com/docs/manual/reference/method/db.aggregate/) on this connection's underlying `db`
+ *
+ * @method aggregate
+ * @memberOf Connection
+ * @param {Array} pipeline
+ * @param {Object} [options]
+ * @param {Boolean} [options.cursor=false] If true, make the Aggregate resolve to a Mongoose AggregationCursor rather than an array
+ * @return {Aggregate} Aggregation wrapper
+ * @api public
+ */
+
 /**
  * Removes the database connection with the given name created with with `useDb()`.
  *

lib/cursor/aggregationCursor.js

@@ -41,11 +41,17 @@ function AggregationCursor(agg) {
   this.cursor = null;
   this.agg = agg;
   this._transforms = [];
+  const connection = agg._connection;
   const model = agg._model;
   delete agg.options.cursor.useMongooseAggCursor;
   this._mongooseOptions = {};
 
-  _init(model, this, agg);
+  if (connection) {
+    this.cursor = connection.db.aggregate(agg._pipeline, agg.options || {});
+    setImmediate(() => this.emit('cursor', this.cursor));
+  } else {
+    _init(model, this, agg);
+  }
 }
 
 util.inherits(AggregationCursor, Readable);

lib/document.js

@@ -807,8 +807,8 @@ function init(self, obj, doc, opts, prefix) {
               reason: e
             }));
           }
-        } else if (opts.hydratedPopulatedDocs) {
-          doc[i] = schemaType.cast(value, self, true);
+        } else if (schemaType && opts.hydratedPopulatedDocs) {
+          doc[i] = schemaType.cast(value, self, true, undefined, { hydratedPopulatedDocs: true });
 
           if (doc[i] && doc[i].$__ && doc[i].$__.wasPopulated) {
             self.$populated(path, doc[i].$__.wasPopulated.value, doc[i].$__.wasPopulated.options);
@@ -4256,24 +4256,25 @@ function applySchemaTypeTransforms(self, json) {
 
   for (const path of paths) {
     const schematype = schema.paths[path];
-    if (typeof schematype.options.transform === 'function') {
+    const topLevelTransformFunction = schematype.options.transform ?? schematype.constructor?.defaultOptions?.transform;
+    const embeddedSchemaTypeTransformFunction = schematype.$embeddedSchemaType?.options?.transform
+      ?? schematype.$embeddedSchemaType?.constructor?.defaultOptions?.transform;
+    if (typeof topLevelTransformFunction === 'function') {
       const val = self.$get(path);
       if (val === undefined) {
         continue;
       }
-      const transformedValue = schematype.options.transform.call(self, val);
+      const transformedValue = topLevelTransformFunction.call(self, val);
       throwErrorIfPromise(path, transformedValue);
       utils.setValue(path, transformedValue, json);
-    } else if (schematype.$embeddedSchemaType != null &&
-        typeof schematype.$embeddedSchemaType.options.transform === 'function') {
+    } else if (typeof embeddedSchemaTypeTransformFunction === 'function') {
       const val = self.$get(path);
       if (val === undefined) {
         continue;
       }
       const vals = [].concat(val);
-      const transform = schematype.$embeddedSchemaType.options.transform;
       for (let i = 0; i < vals.length; ++i) {
-        const transformedValue = transform.call(self, vals[i]);
+        const transformedValue = embeddedSchemaTypeTransformFunction.call(self, vals[i]);
         vals[i] = transformedValue;
         throwErrorIfPromise(path, transformedValue);
       }

lib/drivers/node-mongodb-native/connection.js

@@ -132,6 +132,17 @@ NativeConnection.prototype.useDb = function(name, options) {
   return newConn;
 };
 
+/**
+ * Runs a [db-level aggregate()](https://www.mongodb.com/docs/manual/reference/method/db.aggregate/) on this connection's underlying `db`
+ *
+ * @param {Array} pipeline
+ * @param {Object} [options]
+ */
+
+NativeConnection.prototype.aggregate = function aggregate(pipeline, options) {
+  return new this.base.Aggregate(null, this).append(pipeline).option(options ?? {});
+};
+
 /**
  * Removes the database connection with the given name created with `useDb()`.
  *
@@ -270,6 +281,11 @@ NativeConnection.prototype.createClient = async function createClient(uri, optio
       delete options.autoSearchIndex;
     }
 
+    if ('bufferTimeoutMS' in options) {
+      this.config.bufferTimeoutMS = options.bufferTimeoutMS;
+      delete options.bufferTimeoutMS;
+    }
+
     // Backwards compat
     if (options.user || options.pass) {
       options.auth = options.auth || {};
@@ -415,6 +431,9 @@ function _setClient(conn, client, options, dbName) {
       }
     });
   }
+
+  conn._lastHeartbeatAt = null;
+
   client.on('serverHeartbeatSucceeded', () => {
     conn._lastHeartbeatAt = Date.now();
   });

lib/helpers/createJSONSchemaTypeDefinition.js

@@ -0,0 +1,24 @@
+'use strict';
+
+/**
+ * Handles creating `{ type: 'object' }` vs `{ bsonType: 'object' }` vs `{ bsonType: ['object', 'null'] }`
+ *
+ * @param {String} type
+ * @param {String} bsonType
+ * @param {Boolean} useBsonType
+ * @param {Boolean} isRequired
+ */
+
+module.exports = function createJSONSchemaTypeArray(type, bsonType, useBsonType, isRequired) {
+  if (useBsonType) {
+    if (isRequired) {
+      return { bsonType };
+    }
+    return { bsonType: [bsonType, 'null'] };
+  } else {
+    if (isRequired) {
+      return { type };
+    }
+    return { type: [type, 'null'] };
+  }
+};