From beeddbb7e651ac38a1129cd30f0b0f227c4c4560 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Wed, 11 May 2022 17:06:38 +0200 Subject: [PATCH 01/36] improve issue templates --- .github/ISSUE_TEMPLATE.md | 20 ------ .github/ISSUE_TEMPLATE/bug.yml | 86 ++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 5 ++ .github/ISSUE_TEMPLATE/feature.yml | 37 +++++++++++ .github/ISSUE_TEMPLATE/help.yml | 62 +++++++++++++++++++ .github/ISSUE_TEMPLATE/other.yml | 22 +++++++ .github/ISSUE_TEMPLATE/performance.yml | 81 ++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/regression.yml | 80 ++++++++++++++++++++++++ 8 files changed, 373 insertions(+), 20 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/bug.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature.yml create mode 100644 .github/ISSUE_TEMPLATE/help.yml create mode 100644 .github/ISSUE_TEMPLATE/other.yml create mode 100644 .github/ISSUE_TEMPLATE/performance.yml create mode 100644 .github/ISSUE_TEMPLATE/regression.yml diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 29d4c024659..00000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - -**Do you want to request a *feature* or report a *bug*?** - -**What is the current behavior?** - -**If the current behavior is a bug, please provide the steps to reproduce.** - - - - -**What is the expected behavior?** - -**What are the versions of Node.js, Mongoose and MongoDB you are using? Note that "latest" is not a version.** - - diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 00000000000..6e0a71371fe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,86 @@ +name: 🪲 Bug report +description: Create a report to help us improve + +body: + + - type: markdown + attributes: + value: | + Before you submit an issue we recommend to read the [documentation](https://mongoosejs.com/docs/guide.html). + + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I have written a descriptive issue title + required: true + - label: | + I have searched existing issues to ensure the bug has not already been reported + required: true + + - type: input + id: mongoose-version + attributes: + label: Mongoose version + placeholder: 6.x.x + validations: + required: true + + - type: input + id: node-version + attributes: + label: Node.js version + placeholder: 14.x + validations: + required: true + + - type: input + id: mongo-version + attributes: + label: MongoDB version + placeholder: 5.x + validations: + required: true + + - type: dropdown + id: os + attributes: + label: Operating system + options: + - Linux + - macOS + - Windows + validations: + required: false + + - type: input + id: os-version + attributes: + label: Operating system version (i.e. 20.04, 11.3, 10) + validations: + required: false + + - type: textarea + id: description + attributes: + label: Description + description: A clear and concise description of what the bug is. + validations: + required: true + + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to Reproduce + description: | + List of steps, sample code, or a link to code or a project that reproduces the behavior. + Make sure you place a stack trace inside a [code (```) block](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) to avoid linking unrelated issues. + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..11a4b80ddb9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +documentation: + - name: 📘 Documentation + url: https://mongoosejs.com/docs/guide.html + about: The manual of mongoose diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 00000000000..b2ace34d028 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,37 @@ +name: 🚀 Feature Proposal +description: Submit a proposal for a new feature +labels: ['enhancement', 'new feature'] + +body: + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I have written a descriptive issue title + required: true + - label: | + I have searched existing issues to ensure the feature has not already been requested + required: true + + - type: textarea + id: proposal + attributes: + label: 🚀 Feature Proposal + description: A clear and concise description of what the feature is. + validations: + required: true + + - type: textarea + id: motivation + attributes: + label: Motivation + description: The motivation for the proposal. + + - type: textarea + id: example + attributes: + label: Example + description: | + An example for how this feature would be used. + Make sure you place example code inside a [code (```) block](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) to avoid linking unrelated issues. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/help.yml b/.github/ISSUE_TEMPLATE/help.yml new file mode 100644 index 00000000000..889adbf0908 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/help.yml @@ -0,0 +1,62 @@ +name: 🙋 Help +description: Open an issue and request for individual help +labels: ['help', 'help wanted'] + +body: + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I have written a descriptive issue title + required: true + + - type: input + id: mongoose-version + attributes: + label: Mongoose version + placeholder: 6.x.x + validations: + required: true + + - type: input + id: node-version + attributes: + label: Node.js version + placeholder: 14.x + validations: + required: true + + - type: input + id: mongo-version + attributes: + label: MongoDB version + placeholder: 5.x + validations: + required: true + + - type: dropdown + id: os + attributes: + label: Operating system + options: + - Linux + - macOS + - Windows + validations: + required: false + + - type: input + id: os-version + attributes: + label: Operating system version (i.e. 20.04, 11.3, 10) + validations: + required: false + + - type: textarea + id: text + attributes: + label: Issue + description: | + Give as much detail as you can to help us understand. + Make sure you place example code inside a [code (```) block](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) to avoid linking unrelated issues. diff --git a/.github/ISSUE_TEMPLATE/other.yml b/.github/ISSUE_TEMPLATE/other.yml new file mode 100644 index 00000000000..5579df501c9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/other.yml @@ -0,0 +1,22 @@ +name: ❔ Other +description: Open an issue that is not feature or bug related + +body: + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I have written a descriptive issue title + required: true + - label: | + I have searched existing issues to ensure the issue has not already been raised + required: true + + - type: textarea + id: text + attributes: + label: Issue + description: | + Give as much detail as you can to help us understand. + Make sure you place example code inside a [code (```) block](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) to avoid linking unrelated issues. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/performance.yml b/.github/ISSUE_TEMPLATE/performance.yml new file mode 100644 index 00000000000..26cddfd06fc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/performance.yml @@ -0,0 +1,81 @@ +name: 🦥 Performance issue +description: Report a performance issue +labels: ['performance'] + +body: + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I have written a descriptive issue title + required: true + - label: | + I have searched existing issues to ensure the regression has not already been reported + required: true + + - type: input + id: working-version + attributes: + label: Last working version + placeholder: 1.x.x + validations: + required: true + + - type: input + id: stopped-working-version + attributes: + label: Stopped working in version + placeholder: 2.x.x + validations: + required: true + + - type: input + id: node-version + attributes: + label: Node.js version + placeholder: 14.x + validations: + required: true + + - type: dropdown + id: os + attributes: + label: Operating system + options: + - Linux + - macOS + - Windows + validations: + required: false + + - type: input + id: os-version + attributes: + label: Operating system version (i.e. 20.04, 11.3, 10) + validations: + required: false + + - type: textarea + id: description + attributes: + label: 🦥 Performance issue + description: A clear and concise description of what the performance issue is. + validations: + required: true + + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to Reproduce + description: | + List of steps, sample code, or a link to code or a project that reproduces the behavior. + Make sure you place a stack trace inside a [code (```) block](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) to avoid linking unrelated issues. + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. diff --git a/.github/ISSUE_TEMPLATE/regression.yml b/.github/ISSUE_TEMPLATE/regression.yml new file mode 100644 index 00000000000..4d51e43b9ed --- /dev/null +++ b/.github/ISSUE_TEMPLATE/regression.yml @@ -0,0 +1,80 @@ +name: 💥 Regression Report +description: Report unexpected behavior that worked in previous versions + +body: + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I have written a descriptive issue title + required: true + - label: | + I have searched existing issues to ensure the regression has not already been reported + required: true + + - type: input + id: working-version + attributes: + label: Last working version + placeholder: 1.x.x + validations: + required: true + + - type: input + id: stopped-working-version + attributes: + label: Stopped working in version + placeholder: 2.x.x + validations: + required: true + + - type: input + id: node-version + attributes: + label: Node.js version + placeholder: 14.x + validations: + required: true + + - type: dropdown + id: os + attributes: + label: Operating system + options: + - Linux + - macOS + - Windows + validations: + required: false + + - type: input + id: os-version + attributes: + label: Operating system version (i.e. 20.04, 11.3, 10) + validations: + required: false + + - type: textarea + id: description + attributes: + label: 💥 Regression Report + description: A clear and concise description of what the regression is. + validations: + required: true + + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to Reproduce + description: | + List of steps, sample code, or a link to code or a project that reproduces the behavior. + Make sure you place a stack trace inside a [code (```) block](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) to avoid linking unrelated issues. + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. From d1ffce8dafd253289287aafa9dfb36caa60f1f5a Mon Sep 17 00:00:00 2001 From: uzlopak Date: Wed, 11 May 2022 17:18:50 +0200 Subject: [PATCH 02/36] when requesting help, point to the docs --- .github/ISSUE_TEMPLATE/bug.yml | 6 ------ .github/ISSUE_TEMPLATE/help.yml | 5 +++++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 6e0a71371fe..52a366c804f 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -2,12 +2,6 @@ name: 🪲 Bug report description: Create a report to help us improve body: - - - type: markdown - attributes: - value: | - Before you submit an issue we recommend to read the [documentation](https://mongoosejs.com/docs/guide.html). - - type: checkboxes id: prerequisites attributes: diff --git a/.github/ISSUE_TEMPLATE/help.yml b/.github/ISSUE_TEMPLATE/help.yml index 889adbf0908..c9ffe193647 100644 --- a/.github/ISSUE_TEMPLATE/help.yml +++ b/.github/ISSUE_TEMPLATE/help.yml @@ -3,6 +3,11 @@ description: Open an issue and request for individual help labels: ['help', 'help wanted'] body: + - type: markdown + attributes: + value: | + Before you submit an issue we recommend to read the [documentation](https://mongoosejs.com/docs/guide.html). + - type: checkboxes id: prerequisites attributes: From fda32c3b218b0c3ed2a4cc9305d5a5fca8c9914c Mon Sep 17 00:00:00 2001 From: uzlopak Date: Wed, 11 May 2022 17:22:46 +0200 Subject: [PATCH 03/36] refine performance issue --- .github/ISSUE_TEMPLATE/performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/performance.yml b/.github/ISSUE_TEMPLATE/performance.yml index 26cddfd06fc..5d35d07af3b 100644 --- a/.github/ISSUE_TEMPLATE/performance.yml +++ b/.github/ISSUE_TEMPLATE/performance.yml @@ -11,7 +11,7 @@ body: - label: I have written a descriptive issue title required: true - label: | - I have searched existing issues to ensure the regression has not already been reported + I have searched existing issues to ensure the performance issue has not already been reported required: true - type: input From 0044913813610767da07d9f9874731eed137d5a4 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Fri, 13 May 2022 16:50:25 -0400 Subject: [PATCH 04/36] added test case --- test/query.test.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/query.test.js b/test/query.test.js index ab65be08332..0dd0f4e7775 100644 --- a/test/query.test.js +++ b/test/query.test.js @@ -3962,4 +3962,16 @@ describe('Query', function() { assert.equal(result.length, 1); assert.equal(result[0].name, '@foo.com'); }); + it('allows a transform option for lean on a query gh-10423', async function() { + const testSchema = new mongoose.Schema({ + name: String + }); + const Test = db.model('gh10423', testSchema); + await Test.create({name: 'foo'}); + const result = await Test.findOne().lean({ transform: (doc, ret) => { + delete ret._id; + return ret; + }}); + assert.equal(result._id, undefined); + }); }); From 2c9b41718cf649642bd8f81e4446c9e01bc490be Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Mon, 16 May 2022 17:28:12 -0400 Subject: [PATCH 05/36] `completeOneLean()` done --- lib/query.js | 48 ++++++++++++++++++++++++++++++++++++++++++++-- test/query.test.js | 30 ++++++++++++++++++++++++----- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/lib/query.js b/lib/query.js index d444c1e0725..cef5182cd68 100644 --- a/lib/query.js +++ b/lib/query.js @@ -2217,7 +2217,8 @@ Query.prototype._find = wrapThunk(function(callback) { // Separate options to pass down to `completeMany()` in case we need to // set a session on the document const completeManyOptions = Object.assign({}, { - session: this && this.options && this.options.session || null + session: this && this.options && this.options.session || null, + transform: mongooseOptions.lean.transform || null }); const cb = (err, docs) => { @@ -2241,7 +2242,9 @@ Query.prototype._find = wrapThunk(function(callback) { }); } return mongooseOptions.lean ? - callback(null, docs) : + // call _completeManyLean here? + _completeManyLean(_this.model, docs, fields, userProvidedFields, completeManyOptions, callback) : + // callback(null, docs) : completeMany(_this.model, docs, fields, userProvidedFields, completeManyOptions, callback); } @@ -2415,6 +2418,9 @@ Query.prototype._completeOne = function(doc, res, callback) { const mongooseOptions = this._mongooseOptions; // `rawResult` const options = this.options; + if (!options.lean && mongooseOptions.lean) { + options.lean = mongooseOptions.lean; + } if (options.explain) { return callback(null, doc); @@ -4007,12 +4013,50 @@ Query.prototype._findAndModify = function(type, callback) { */ function _completeOneLean(doc, res, opts, callback) { + // need to recurse on subdocs like arrays and what not. + if (opts.lean && opts.lean.transform) { + for (const key of Object.keys(doc)) { + if (Array.isArray(doc[key])) { + // call completeOneLean or completeManyLean? + // if staying in completeOneLean + doc[key].forEach((item) => { + console.log(item); + for (const k in item) { + console.log(k); + if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { + _completeOneLean(item[k], res, opts); + } + opts.lean.transform({}, item); + } + }); + } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { + _completeOneLean(doc[key], res, opts); + } + } + opts.lean.transform({}, doc); + if (callback) { + return callback(null, doc); + } else { + return; + } + } if (opts.rawResult) { return callback(null, res); } return callback(null, doc); } +/*! + * ignore + */ + +function _completeManyLean(model, docs, fields, userProvidedFields, opts, callback) { + docs.forEach(doc => { + opts.transform({}, doc); + }); + return callback(null, docs); +} + /*! * Override mquery.prototype._mergeUpdate to handle mongoose objects in * updates. diff --git a/test/query.test.js b/test/query.test.js index 0dd0f4e7775..765de6e8032 100644 --- a/test/query.test.js +++ b/test/query.test.js @@ -3963,15 +3963,35 @@ describe('Query', function() { assert.equal(result[0].name, '@foo.com'); }); it('allows a transform option for lean on a query gh-10423', async function() { + const arraySchema = new mongoose.Schema({ + sub: String + }); + const subDoc = new mongoose.Schema({ + nickName: String + }); const testSchema = new mongoose.Schema({ - name: String + name: String, + foo: [arraySchema], + otherName: subDoc }); const Test = db.model('gh10423', testSchema); - await Test.create({name: 'foo'}); - const result = await Test.findOne().lean({ transform: (doc, ret) => { + await Test.create({ name: 'foo', foo: [{ sub: 'Test' }, { sub: 'Testerson' }], otherName: { nickName: 'Bar' } }); + /* + const result = await Test.find().lean({ transform: (doc, ret) => { + delete ret._id; + return ret; + } }); + console.log('The result', result); + assert.equal(result[0]._id, undefined); + */ + const single = await Test.findOne().lean({ transform: (doc, ret) => { delete ret._id; return ret; - }}); - assert.equal(result._id, undefined); + } }); + console.log('The single document', single); + assert.equal(single._id, undefined); + assert.equal(single.otherName._id, undefined); + assert.equal(single.foo[0]._id, undefined); + assert.equal(single.foo[0]._id, undefined); }); }); From b8dbcd8381851b4ca69c3dc56bea139f8ab293e8 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Mon, 16 May 2022 18:02:00 -0400 Subject: [PATCH 06/36] removed logs --- lib/query.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/query.js b/lib/query.js index cef5182cd68..be7d296d66e 100644 --- a/lib/query.js +++ b/lib/query.js @@ -4020,9 +4020,7 @@ function _completeOneLean(doc, res, opts, callback) { // call completeOneLean or completeManyLean? // if staying in completeOneLean doc[key].forEach((item) => { - console.log(item); for (const k in item) { - console.log(k); if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { _completeOneLean(item[k], res, opts); } From 92cae73a76924658824c98f1efaabcb1de52cefa Mon Sep 17 00:00:00 2001 From: uzlopak Date: Tue, 17 May 2022 07:27:38 +0200 Subject: [PATCH 07/36] adapt changes requested by PR comments --- .github/ISSUE_TEMPLATE/bug.yml | 20 +------ .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/ISSUE_TEMPLATE/performance.yml | 22 +------ .github/ISSUE_TEMPLATE/regression.yml | 80 -------------------------- 4 files changed, 4 insertions(+), 120 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/regression.yml diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 52a366c804f..3089b69e90e 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -32,29 +32,11 @@ body: - type: input id: mongo-version attributes: - label: MongoDB version + label: MongoDB server version placeholder: 5.x validations: required: true - - type: dropdown - id: os - attributes: - label: Operating system - options: - - Linux - - macOS - - Windows - validations: - required: false - - - type: input - id: os-version - attributes: - label: Operating system version (i.e. 20.04, 11.3, 10) - validations: - required: false - - type: textarea id: description attributes: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 11a4b80ddb9..d552e213c6f 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false documentation: - name: 📘 Documentation url: https://mongoosejs.com/docs/guide.html - about: The manual of mongoose + about: Mongoose Docs diff --git a/.github/ISSUE_TEMPLATE/performance.yml b/.github/ISSUE_TEMPLATE/performance.yml index 5d35d07af3b..8eef06255b5 100644 --- a/.github/ISSUE_TEMPLATE/performance.yml +++ b/.github/ISSUE_TEMPLATE/performance.yml @@ -17,7 +17,7 @@ body: - type: input id: working-version attributes: - label: Last working version + label: Last performant version placeholder: 1.x.x validations: required: true @@ -25,7 +25,7 @@ body: - type: input id: stopped-working-version attributes: - label: Stopped working in version + label: Slowed down in version placeholder: 2.x.x validations: required: true @@ -38,24 +38,6 @@ body: validations: required: true - - type: dropdown - id: os - attributes: - label: Operating system - options: - - Linux - - macOS - - Windows - validations: - required: false - - - type: input - id: os-version - attributes: - label: Operating system version (i.e. 20.04, 11.3, 10) - validations: - required: false - - type: textarea id: description attributes: diff --git a/.github/ISSUE_TEMPLATE/regression.yml b/.github/ISSUE_TEMPLATE/regression.yml deleted file mode 100644 index 4d51e43b9ed..00000000000 --- a/.github/ISSUE_TEMPLATE/regression.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: 💥 Regression Report -description: Report unexpected behavior that worked in previous versions - -body: - - type: checkboxes - id: prerequisites - attributes: - label: Prerequisites - options: - - label: I have written a descriptive issue title - required: true - - label: | - I have searched existing issues to ensure the regression has not already been reported - required: true - - - type: input - id: working-version - attributes: - label: Last working version - placeholder: 1.x.x - validations: - required: true - - - type: input - id: stopped-working-version - attributes: - label: Stopped working in version - placeholder: 2.x.x - validations: - required: true - - - type: input - id: node-version - attributes: - label: Node.js version - placeholder: 14.x - validations: - required: true - - - type: dropdown - id: os - attributes: - label: Operating system - options: - - Linux - - macOS - - Windows - validations: - required: false - - - type: input - id: os-version - attributes: - label: Operating system version (i.e. 20.04, 11.3, 10) - validations: - required: false - - - type: textarea - id: description - attributes: - label: 💥 Regression Report - description: A clear and concise description of what the regression is. - validations: - required: true - - - type: textarea - id: steps-to-reproduce - attributes: - label: Steps to Reproduce - description: | - List of steps, sample code, or a link to code or a project that reproduces the behavior. - Make sure you place a stack trace inside a [code (```) block](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) to avoid linking unrelated issues. - validations: - required: true - - - type: textarea - id: expected-behavior - attributes: - label: Expected Behavior - description: A clear and concise description of what you expected to happen. From 6cbbaef5ef18c280b5eeb7112f9bdea68beef3fc Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Tue, 17 May 2022 12:04:12 -0400 Subject: [PATCH 08/36] `_completeManyLean()` done, I think --- lib/query.js | 30 ++++++++++++++++++++++++------ test/query.test.js | 7 +++---- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/lib/query.js b/lib/query.js index be7d296d66e..5018118787f 100644 --- a/lib/query.js +++ b/lib/query.js @@ -2218,7 +2218,7 @@ Query.prototype._find = wrapThunk(function(callback) { // set a session on the document const completeManyOptions = Object.assign({}, { session: this && this.options && this.options.session || null, - transform: mongooseOptions.lean.transform || null + lean: mongooseOptions.lean || null }); const cb = (err, docs) => { @@ -4017,8 +4017,6 @@ function _completeOneLean(doc, res, opts, callback) { if (opts.lean && opts.lean.transform) { for (const key of Object.keys(doc)) { if (Array.isArray(doc[key])) { - // call completeOneLean or completeManyLean? - // if staying in completeOneLean doc[key].forEach((item) => { for (const k in item) { if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { @@ -4049,9 +4047,29 @@ function _completeOneLean(doc, res, opts, callback) { */ function _completeManyLean(model, docs, fields, userProvidedFields, opts, callback) { - docs.forEach(doc => { - opts.transform({}, doc); - }); + if (opts.lean && opts.lean.transform) { + docs.forEach(doc => { + for (const key of Object.keys(doc)) { + if (Array.isArray(doc[key])) { + doc[key].forEach((item) => { + for (const k in item) { + if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { + _completeOneLean(item[k], null, opts); + } + opts.lean.transform({}, item); + } + }); + } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { + _completeOneLean(doc[key], null, opts); + } + } + opts.lean.transform({}, doc); + }); + } + + if (!callback) { + return; + } return callback(null, docs); } diff --git a/test/query.test.js b/test/query.test.js index 765de6e8032..6d0a989f819 100644 --- a/test/query.test.js +++ b/test/query.test.js @@ -3976,19 +3976,18 @@ describe('Query', function() { }); const Test = db.model('gh10423', testSchema); await Test.create({ name: 'foo', foo: [{ sub: 'Test' }, { sub: 'Testerson' }], otherName: { nickName: 'Bar' } }); - /* const result = await Test.find().lean({ transform: (doc, ret) => { delete ret._id; return ret; } }); - console.log('The result', result); assert.equal(result[0]._id, undefined); - */ + assert.equal(result[0].otherName._id, undefined); + assert.equal(result[0].foo[0]._id, undefined); + assert.equal(result[0].foo[1]._id, undefined); const single = await Test.findOne().lean({ transform: (doc, ret) => { delete ret._id; return ret; } }); - console.log('The single document', single); assert.equal(single._id, undefined); assert.equal(single.otherName._id, undefined); assert.equal(single.foo[0]._id, undefined); From 0cd4171350257eaac4679a9f76217475c5a7d7e3 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Wed, 18 May 2022 15:51:47 +0200 Subject: [PATCH 09/36] try to fix isAtlas --- lib/helpers/topology/isAtlas.js | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/helpers/topology/isAtlas.js b/lib/helpers/topology/isAtlas.js index f0638d55136..8d4ae23c8c4 100644 --- a/lib/helpers/topology/isAtlas.js +++ b/lib/helpers/topology/isAtlas.js @@ -8,6 +8,19 @@ module.exports = function isAtlas(topologyDescription) { } const hostnames = Array.from(topologyDescription.servers.keys()); - return hostnames.length > 0 && - hostnames.every(host => host.endsWith('.mongodb.net:27017')); + + if (hostnames.length === 0) { + return false; + } + + for (let i = 0, il = hostnames.length; i < il; ++i) { + const url = new URL(hostnames[i]); + if ( + url.hostname.endsWith('.mongodb.net') === false || + url.port !== '27017' + ) { + return false; + } + } + return true; }; \ No newline at end of file From da4f04df76a702b957722a8982e06ed30a9655d5 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Wed, 18 May 2022 16:11:23 +0200 Subject: [PATCH 10/36] avoid prototype pollution by setDottedPath --- lib/helpers/path/setDottedPath.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/helpers/path/setDottedPath.js b/lib/helpers/path/setDottedPath.js index d276e164d3b..69cd19b773a 100644 --- a/lib/helpers/path/setDottedPath.js +++ b/lib/helpers/path/setDottedPath.js @@ -6,6 +6,15 @@ module.exports = function setDottedPath(obj, path, val) { return; } const parts = path.split('.'); + + if ( + parts.indexOf("__proto__") !== -1 || + parts.indexOf("constructor") !== -1 || + parts.indexOf("prototype") !== -1 + ) { + throw new Error('prototype, __proto__ or constructor can not be set by setDottedPath'); + } + const last = parts.pop(); let cur = obj; for (const part of parts) { From 033980636eb6f9677b95971d8fda04f2880e4941 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Wed, 18 May 2022 16:15:24 +0200 Subject: [PATCH 11/36] fix linting error --- lib/helpers/path/setDottedPath.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/helpers/path/setDottedPath.js b/lib/helpers/path/setDottedPath.js index 69cd19b773a..e54978ec7e6 100644 --- a/lib/helpers/path/setDottedPath.js +++ b/lib/helpers/path/setDottedPath.js @@ -8,9 +8,9 @@ module.exports = function setDottedPath(obj, path, val) { const parts = path.split('.'); if ( - parts.indexOf("__proto__") !== -1 || - parts.indexOf("constructor") !== -1 || - parts.indexOf("prototype") !== -1 + parts.indexOf('__proto__') !== -1 || + parts.indexOf('constructor') !== -1 || + parts.indexOf('prototype') !== -1 ) { throw new Error('prototype, __proto__ or constructor can not be set by setDottedPath'); } From c6de154c61fed16aff65d8ad27cd44d4b2d1f51b Mon Sep 17 00:00:00 2001 From: naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Thu, 19 May 2022 00:10:49 +0000 Subject: [PATCH 12/36] chore(deps): Included dependency review > Dependency Review GitHub Action in your repository to enforce dependency > reviews on your pull requests. > The action scans for vulnerable versions of dependencies introduced by package version > changes in pull requests, > and warns you about the associated security vulnerabilities. > This gives you better visibility of what's changing in a pull request, > and helps prevent vulnerabilities being added to your repository. https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement Signed-off-by: naveen <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/depsreview.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/depsreview.yaml diff --git a/.github/workflows/depsreview.yaml b/.github/workflows/depsreview.yaml new file mode 100644 index 00000000000..f2605b7a7e2 --- /dev/null +++ b/.github/workflows/depsreview.yaml @@ -0,0 +1,14 @@ +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v3 + - name: 'Dependency Review' + uses: actions/dependency-review-action@v1 From 73cc7420dc7dbb3bf9031716f3a45afc44007a64 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Fri, 20 May 2022 16:40:22 -0400 Subject: [PATCH 13/36] reworking solution --- lib/query.js | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/lib/query.js b/lib/query.js index 5018118787f..5ad0d3a7134 100644 --- a/lib/query.js +++ b/lib/query.js @@ -4048,23 +4048,43 @@ function _completeOneLean(doc, res, opts, callback) { function _completeManyLean(model, docs, fields, userProvidedFields, opts, callback) { if (opts.lean && opts.lean.transform) { - docs.forEach(doc => { - for (const key of Object.keys(doc)) { - if (Array.isArray(doc[key])) { - doc[key].forEach((item) => { + if (Array.isArray(docs)) { + docs.forEach(doc => { + for (const key of Object.keys(doc)) { + if (Array.isArray(doc[key])) { + doc[key].forEach((item) => { + for (const k in item) { + if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { + _completeManyLean(model, item[k], fields, userProvidedFields, opts); + } + console.log('item', item); + opts.lean.transform({}, item); + } + }); + } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { + _completeManyLean(model, doc[key], fields, userProvidedFields, opts); + } + } + console.log('doc', doc) + opts.lean.transform({}, doc); + }); + } else { + for (const key of Object.keys(docs)) { + if (Array.isArray(docs[key])) { + docs[key].forEach((item) => { for (const k in item) { if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { - _completeOneLean(item[k], null, opts); + _completeManyLean(model, item[k], fields, userProvidedFields, opts); } + console.log('else item', item) opts.lean.transform({}, item); } }); - } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { - _completeOneLean(doc[key], null, opts); + } else if (typeof docs[key] === 'object' && docs[key] != null && Object.keys(docs[key]).length) { + _completeManyLean(model, docs[key], fields, userProvidedFields, opts); } } - opts.lean.transform({}, doc); - }); + } } if (!callback) { From da0d878b82e841d31c0f0187b56af1f150e805cf Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Mon, 23 May 2022 11:51:44 -0400 Subject: [PATCH 14/36] independent `completeManyLean()` --- lib/query.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/query.js b/lib/query.js index 5ad0d3a7134..fed500f228d 100644 --- a/lib/query.js +++ b/lib/query.js @@ -4013,7 +4013,6 @@ Query.prototype._findAndModify = function(type, callback) { */ function _completeOneLean(doc, res, opts, callback) { - // need to recurse on subdocs like arrays and what not. if (opts.lean && opts.lean.transform) { for (const key of Object.keys(doc)) { if (Array.isArray(doc[key])) { @@ -4057,16 +4056,14 @@ function _completeManyLean(model, docs, fields, userProvidedFields, opts, callba if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { _completeManyLean(model, item[k], fields, userProvidedFields, opts); } - console.log('item', item); opts.lean.transform({}, item); } }); } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { _completeManyLean(model, doc[key], fields, userProvidedFields, opts); } - } - console.log('doc', doc) opts.lean.transform({}, doc); + } }); } else { for (const key of Object.keys(docs)) { @@ -4076,13 +4073,13 @@ function _completeManyLean(model, docs, fields, userProvidedFields, opts, callba if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { _completeManyLean(model, item[k], fields, userProvidedFields, opts); } - console.log('else item', item) opts.lean.transform({}, item); } }); } else if (typeof docs[key] === 'object' && docs[key] != null && Object.keys(docs[key]).length) { _completeManyLean(model, docs[key], fields, userProvidedFields, opts); } + opts.lean.transform({}, docs); } } } From 9babf9ccefb3be2107024d9ccdc237370619e1db Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Mon, 23 May 2022 12:01:23 -0400 Subject: [PATCH 15/36] lint fix --- lib/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/query.js b/lib/query.js index fed500f228d..6e355f0b6ee 100644 --- a/lib/query.js +++ b/lib/query.js @@ -4062,7 +4062,7 @@ function _completeManyLean(model, docs, fields, userProvidedFields, opts, callba } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { _completeManyLean(model, doc[key], fields, userProvidedFields, opts); } - opts.lean.transform({}, doc); + opts.lean.transform({}, doc); } }); } else { From 9c8098bd9e04278e134767bf12bea80346958754 Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Fri, 27 May 2022 15:37:15 -0500 Subject: [PATCH 16/36] Included the check within the test.yml as per the code recommendations. Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/depsreview.yaml | 14 -------------- .github/workflows/test.yml | 13 ++++++++++++- 2 files changed, 12 insertions(+), 15 deletions(-) delete mode 100644 .github/workflows/depsreview.yaml diff --git a/.github/workflows/depsreview.yaml b/.github/workflows/depsreview.yaml deleted file mode 100644 index f2605b7a7e2..00000000000 --- a/.github/workflows/depsreview.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: 'Dependency Review' -on: [pull_request] - -permissions: - contents: read - -jobs: - dependency-review: - runs-on: ubuntu-latest - steps: - - name: 'Checkout Repository' - uses: actions/checkout@v3 - - name: 'Dependency Review' - uses: actions/dependency-review-action@v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba37e65a9ca..6c597e4a31f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,4 +121,15 @@ jobs: uses: actions/upload-artifact@v3 with: name: coverage - path: coverage \ No newline at end of file + path: coverage + dependency-review: + name: Dependency Review + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Check out repo + uses: actions/checkout@v3 + - name: Dependency review + uses: actions/dependency-review-action@v1 From 0e7a7becc096d4114dd7f9fd27a810a5129946ca Mon Sep 17 00:00:00 2001 From: Eliott C Date: Sat, 28 May 2022 01:34:28 +0200 Subject: [PATCH 17/36] fix(document): fix ObjectId conversion for external schemas When dealing with external schemas, i.e. a mongoose schema created inside another module, instanceOf may not work. This results in the value of ObjectIds in the schema being converted to strings when saving to the database. --- lib/helpers/clone.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/helpers/clone.js b/lib/helpers/clone.js index 6936fa642fc..d5eb877dcbd 100644 --- a/lib/helpers/clone.js +++ b/lib/helpers/clone.js @@ -71,16 +71,14 @@ function clone(obj, options, isArrayChild) { return new objConstructor(+obj); case 'RegExp': return cloneRegExp(obj); + case 'ObjectId': + return new ObjectId(obj.id); default: // ignore break; } } - if (obj instanceof ObjectId) { - return new ObjectId(obj.id); - } - if (isBsonType(obj, 'Decimal128')) { if (options && options.flattenDecimals) { return obj.toJSON(); From d435d5dcf81025192f7a4f6203154616e1354f2e Mon Sep 17 00:00:00 2001 From: Eliott C Date: Sat, 28 May 2022 01:54:46 +0200 Subject: [PATCH 18/36] Update clone.js --- lib/helpers/clone.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/helpers/clone.js b/lib/helpers/clone.js index d5eb877dcbd..c593e01e11a 100644 --- a/lib/helpers/clone.js +++ b/lib/helpers/clone.js @@ -78,6 +78,10 @@ function clone(obj, options, isArrayChild) { break; } } + + if (obj instanceof ObjectId) { + return new ObjectId(obj.id); + } if (isBsonType(obj, 'Decimal128')) { if (options && options.flattenDecimals) { From aba95a2fe599d6cb0e37ecade22a7b43ddf6e3f7 Mon Sep 17 00:00:00 2001 From: Eliott C Date: Sat, 28 May 2022 01:55:24 +0200 Subject: [PATCH 19/36] Update clone.js --- lib/helpers/clone.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/helpers/clone.js b/lib/helpers/clone.js index c593e01e11a..5695ebf6989 100644 --- a/lib/helpers/clone.js +++ b/lib/helpers/clone.js @@ -78,7 +78,7 @@ function clone(obj, options, isArrayChild) { break; } } - + if (obj instanceof ObjectId) { return new ObjectId(obj.id); } From 9d30935d99e59158b49be418971057792a341a5d Mon Sep 17 00:00:00 2001 From: uzlopak Date: Sat, 28 May 2022 09:43:42 +0200 Subject: [PATCH 20/36] use specialParts for setDottedPath --- lib/helpers/path/setDottedPath.js | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/helpers/path/setDottedPath.js b/lib/helpers/path/setDottedPath.js index e54978ec7e6..c6544c655bf 100644 --- a/lib/helpers/path/setDottedPath.js +++ b/lib/helpers/path/setDottedPath.js @@ -1,23 +1,25 @@ 'use strict'; +const specialProperties = require('../specialProperties'); + + module.exports = function setDottedPath(obj, path, val) { if (path.indexOf('.') === -1) { + if (specialProperties.has(path)) { + return; + } + obj[path] = val; return; } const parts = path.split('.'); - if ( - parts.indexOf('__proto__') !== -1 || - parts.indexOf('constructor') !== -1 || - parts.indexOf('prototype') !== -1 - ) { - throw new Error('prototype, __proto__ or constructor can not be set by setDottedPath'); - } - const last = parts.pop(); let cur = obj; for (const part of parts) { + if (specialProperties.has(part)) { + continue; + } if (cur[part] == null) { cur[part] = {}; } @@ -25,5 +27,7 @@ module.exports = function setDottedPath(obj, path, val) { cur = cur[part]; } - cur[last] = val; + if (!specialProperties.has(last)) { + cur[last] = val; + } }; \ No newline at end of file From f6becebd93d054c52c71fe2b8ddaac549cc22fa4 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Sat, 28 May 2022 10:18:49 +0200 Subject: [PATCH 21/36] fix documention of error handling --- lib/error/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/error/index.js b/lib/error/index.js index 8a40f28176f..4f5b446736f 100644 --- a/lib/error/index.js +++ b/lib/error/index.js @@ -5,11 +5,11 @@ * Mongoose-specific errors. * * #### Example: - * const Model = mongoose.model('Test', new Schema({ answer: Number })); + * const Model = mongoose.model('Test', new mongoose.Schema({ answer: Number })); * const doc = new Model({ answer: 'not a number' }); * const err = doc.validateSync(); * - * err instanceof mongoose.Error; // true + * err instanceof mongoose.Error.ValidationError; // true * * @constructor Error * @param {String} msg Error message From 85a3d09b741ce9a7f6be78639bbe0820026c099e Mon Sep 17 00:00:00 2001 From: Eliott C Date: Sat, 28 May 2022 11:29:45 +0200 Subject: [PATCH 22/36] Update clone.js --- lib/helpers/clone.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/helpers/clone.js b/lib/helpers/clone.js index 5695ebf6989..70acc6481c5 100644 --- a/lib/helpers/clone.js +++ b/lib/helpers/clone.js @@ -71,8 +71,13 @@ function clone(obj, options, isArrayChild) { return new objConstructor(+obj); case 'RegExp': return cloneRegExp(obj); + // instanceof is not enough for schemas defined in another package case 'ObjectId': - return new ObjectId(obj.id); + if (obj._bsonType === "ObjectID") { + return new ObjectId(obj.id); + } else { + break; + } default: // ignore break; From cb63c7b1c2af5419146023eb54ccc8fc84719d29 Mon Sep 17 00:00:00 2001 From: Eliott C Date: Sat, 28 May 2022 11:32:15 +0200 Subject: [PATCH 23/36] Update clone.js --- lib/helpers/clone.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/helpers/clone.js b/lib/helpers/clone.js index 70acc6481c5..629daf85dce 100644 --- a/lib/helpers/clone.js +++ b/lib/helpers/clone.js @@ -73,7 +73,7 @@ function clone(obj, options, isArrayChild) { return cloneRegExp(obj); // instanceof is not enough for schemas defined in another package case 'ObjectId': - if (obj._bsonType === "ObjectID") { + if (obj._bsonType === 'ObjectID') { return new ObjectId(obj.id); } else { break; From 2d3d5b1b6a815fc8958c30f006ff218cbb58b2b0 Mon Sep 17 00:00:00 2001 From: Eliott C Date: Sat, 28 May 2022 11:32:46 +0200 Subject: [PATCH 24/36] Update clone.js --- lib/helpers/clone.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/helpers/clone.js b/lib/helpers/clone.js index 629daf85dce..3cd275e861a 100644 --- a/lib/helpers/clone.js +++ b/lib/helpers/clone.js @@ -75,9 +75,8 @@ function clone(obj, options, isArrayChild) { case 'ObjectId': if (obj._bsonType === 'ObjectID') { return new ObjectId(obj.id); - } else { - break; } + break; default: // ignore break; From 010b280def3e3fc4c71a421f1cc6a88c28fc4ea1 Mon Sep 17 00:00:00 2001 From: Eliott C Date: Sat, 28 May 2022 11:38:14 +0200 Subject: [PATCH 25/36] Update clone.js --- lib/helpers/clone.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/helpers/clone.js b/lib/helpers/clone.js index 3cd275e861a..379a74abeca 100644 --- a/lib/helpers/clone.js +++ b/lib/helpers/clone.js @@ -71,19 +71,13 @@ function clone(obj, options, isArrayChild) { return new objConstructor(+obj); case 'RegExp': return cloneRegExp(obj); - // instanceof is not enough for schemas defined in another package - case 'ObjectId': - if (obj._bsonType === 'ObjectID') { - return new ObjectId(obj.id); - } - break; default: // ignore break; } } - if (obj instanceof ObjectId) { + if (isBsonType(obj, 'ObjectID')) { return new ObjectId(obj.id); } From dda76d05e1a3162a6987d63c39b9b934da92e63b Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Sat, 28 May 2022 16:49:18 +0200 Subject: [PATCH 26/36] refacto: use isBsonType instead of instanceof for ObjectId & Decimal128 --- lib/helpers/common.js | 7 +++---- lib/helpers/discriminator/areDiscriminatorValuesEqual.js | 4 ++-- lib/schema/decimal128.js | 8 ++++---- lib/types/DocumentArray/methods/index.js | 6 +++--- lib/types/array/methods/index.js | 6 +++--- lib/types/map.js | 8 ++++---- lib/utils.js | 7 +++---- 7 files changed, 22 insertions(+), 24 deletions(-) diff --git a/lib/helpers/common.js b/lib/helpers/common.js index c38c88cf9c0..c1433ce4748 100644 --- a/lib/helpers/common.js +++ b/lib/helpers/common.js @@ -5,8 +5,7 @@ */ const Binary = require('../driver').get().Binary; -const Decimal128 = require('../types/decimal128'); -const ObjectId = require('../types/objectid'); +const isBsonType = require('./isBsonType'); const isMongooseObject = require('./isMongooseObject'); exports.flatten = flatten; @@ -99,9 +98,9 @@ function shouldFlatten(val) { return val && typeof val === 'object' && !(val instanceof Date) && - !(val instanceof ObjectId) && + !isBsonType(val, 'ObjectID') && (!Array.isArray(val) || val.length !== 0) && !(val instanceof Buffer) && - !(val instanceof Decimal128) && + !isBsonType(val, 'Decimal128') && !(val instanceof Binary); } diff --git a/lib/helpers/discriminator/areDiscriminatorValuesEqual.js b/lib/helpers/discriminator/areDiscriminatorValuesEqual.js index 87b2408e6db..52c7ee0eb0a 100644 --- a/lib/helpers/discriminator/areDiscriminatorValuesEqual.js +++ b/lib/helpers/discriminator/areDiscriminatorValuesEqual.js @@ -1,6 +1,6 @@ 'use strict'; -const ObjectId = require('../../types/objectid'); +const isBsonType = require('../isBsonType'); module.exports = function areDiscriminatorValuesEqual(a, b) { if (typeof a === 'string' && typeof b === 'string') { @@ -9,7 +9,7 @@ module.exports = function areDiscriminatorValuesEqual(a, b) { if (typeof a === 'number' && typeof b === 'number') { return a === b; } - if (a instanceof ObjectId && b instanceof ObjectId) { + if (isBsonType(a, 'ObjectID') && isBsonType(b, 'ObjectID')) { return a.toString() === b.toString(); } return false; diff --git a/lib/schema/decimal128.js b/lib/schema/decimal128.js index f20a452ccaf..3323df4e094 100644 --- a/lib/schema/decimal128.js +++ b/lib/schema/decimal128.js @@ -6,9 +6,9 @@ const SchemaType = require('../schematype'); const CastError = SchemaType.CastError; -const Decimal128Type = require('../types/decimal128'); const castDecimal128 = require('../cast/decimal128'); const utils = require('../utils'); +const isBsonType = require('../helpers/isBsonType'); /** * Decimal128 SchemaType constructor. @@ -105,7 +105,7 @@ Decimal128.cast = function cast(caster) { */ Decimal128._defaultCaster = v => { - if (v != null && !(v instanceof Decimal128Type)) { + if (v != null && !isBsonType(v, 'Decimal128')) { throw new Error(); } return v; @@ -115,7 +115,7 @@ Decimal128._defaultCaster = v => { * ignore */ -Decimal128._checkRequired = v => v instanceof Decimal128Type; +Decimal128._checkRequired = v => isBsonType(v, 'Decimal128'); /** * Override the function the required validator uses to check whether a string @@ -164,7 +164,7 @@ Decimal128.prototype.checkRequired = function checkRequired(value, doc) { Decimal128.prototype.cast = function(value, doc, init) { if (SchemaType._isRef(this, value, doc, init)) { - if (value instanceof Decimal128Type) { + if (isBsonType(value, 'Decimal128')) { return value; } diff --git a/lib/types/DocumentArray/methods/index.js b/lib/types/DocumentArray/methods/index.js index ac43d91bd3b..3a6a7c55d19 100644 --- a/lib/types/DocumentArray/methods/index.js +++ b/lib/types/DocumentArray/methods/index.js @@ -2,11 +2,11 @@ const ArrayMethods = require('../../array/methods'); const Document = require('../../../document'); -const ObjectId = require('../../objectid'); const castObjectId = require('../../../cast/objectid'); const getDiscriminatorByValue = require('../../../helpers/discriminator/getDiscriminatorByValue'); const internalToObjectOptions = require('../../../options').internalToObjectOptions; const utils = require('../../../utils'); +const isBsonType = require('../../../helpers/isBsonType'); const arrayParentSymbol = require('../../../helpers/symbols').arrayParentSymbol; const arrayPathSymbol = require('../../../helpers/symbols').arrayPathSymbol; @@ -66,7 +66,7 @@ const methods = { // only objects are permitted so we can safely assume that // non-objects are to be interpreted as _id if (Buffer.isBuffer(value) || - value instanceof ObjectId || !utils.isObject(value)) { + isBsonType(value, 'ObjectID') || !utils.isObject(value)) { value = { _id: value }; } @@ -134,7 +134,7 @@ const methods = { if (sid == _id._id) { return val; } - } else if (!(id instanceof ObjectId) && !(_id instanceof ObjectId)) { + } else if (!isBsonType(id, 'ObjectID') && !isBsonType(_id, 'ObjectID')) { if (id == _id || utils.deepEqual(id, _id)) { return val; } diff --git a/lib/types/array/methods/index.js b/lib/types/array/methods/index.js index 792d9b59e87..9de796d8aae 100644 --- a/lib/types/array/methods/index.js +++ b/lib/types/array/methods/index.js @@ -3,10 +3,10 @@ const Document = require('../../../document'); const ArraySubdocument = require('../../ArraySubdocument'); const MongooseError = require('../../../error/mongooseError'); -const ObjectId = require('../../objectid'); const cleanModifiedSubpaths = require('../../../helpers/document/cleanModifiedSubpaths'); const internalToObjectOptions = require('../../../options').internalToObjectOptions; const utils = require('../../../utils'); +const isBsonType = require('../../../helpers/isBsonType'); const arrayAtomicsSymbol = require('../../../helpers/symbols').arrayAtomicsSymbol; const arrayParentSymbol = require('../../../helpers/symbols').arrayParentSymbol; @@ -227,7 +227,7 @@ const methods = { // only objects are permitted so we can safely assume that // non-objects are to be interpreted as _id if (Buffer.isBuffer(value) || - value instanceof ObjectId || !utils.isObject(value)) { + isBsonType(value, 'ObjectID') || !utils.isObject(value)) { value = { _id: value }; } @@ -469,7 +469,7 @@ const methods = { */ indexOf(obj, fromIndex) { - if (obj instanceof ObjectId) { + if (isBsonType(obj, 'ObjectID')) { obj = obj.toString(); } diff --git a/lib/types/map.js b/lib/types/map.js index 903110e63ec..486c03f4e7f 100644 --- a/lib/types/map.js +++ b/lib/types/map.js @@ -1,13 +1,13 @@ 'use strict'; const Mixed = require('../schema/mixed'); -const ObjectId = require('./objectid'); const clone = require('../helpers/clone'); const deepEqual = require('../utils').deepEqual; const getConstructorName = require('../helpers/getConstructorName'); const handleSpreadDoc = require('../helpers/document/handleSpreadDoc'); const util = require('util'); const specialProperties = require('../helpers/specialProperties'); +const isBsonType = require('../helpers/isBsonType'); const populateModelSymbol = require('../helpers/symbols').populateModelSymbol; @@ -43,7 +43,7 @@ class MongooseMap extends Map { } get(key, options) { - if (key instanceof ObjectId) { + if (isBsonType(key, 'ObjectID')) { key = key.toString(); } @@ -55,7 +55,7 @@ class MongooseMap extends Map { } set(key, value) { - if (key instanceof ObjectId) { + if (isBsonType(key, 'ObjectID')) { key = key.toString(); } @@ -117,7 +117,7 @@ class MongooseMap extends Map { } delete(key) { - if (key instanceof ObjectId) { + if (isBsonType(key, 'ObjectID')) { key = key.toString(); } diff --git a/lib/utils.js b/lib/utils.js index 27992139393..7a6f129fe41 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -6,7 +6,6 @@ const ms = require('ms'); const mpath = require('mpath'); -const Decimal = require('./types/decimal128'); const ObjectId = require('./types/objectid'); const PopulateOptions = require('./options/PopulateOptions'); const clone = require('./helpers/clone'); @@ -307,7 +306,7 @@ exports.merge = function merge(to, from, options, path) { to[key] = from[key].clone(); } continue; - } else if (from[key] instanceof ObjectId) { + } else if (isBsonType(from[key], 'ObjectID')) { to[key] = new ObjectId(from[key]); continue; } @@ -481,7 +480,7 @@ exports.tick = function tick(callback) { */ exports.isMongooseType = function(v) { - return v instanceof ObjectId || v instanceof Decimal || v instanceof Buffer; + return isBsonType(v, 'ObjectID') || isBsonType(v, 'Decimal128') || v instanceof Buffer; }; exports.isMongooseObject = isMongooseObject; @@ -795,7 +794,7 @@ exports.array.unique = function(arr) { } ret.push(item); primitives.add(item); - } else if (item instanceof ObjectId) { + } else if (isBsonType(item, 'ObjectID')) { if (ids.has(item.toString())) { continue; } From 901b2119257bf6e96b31c9f9ac183d02c631a75e Mon Sep 17 00:00:00 2001 From: uzlopak Date: Sat, 28 May 2022 17:02:16 +0200 Subject: [PATCH 27/36] improve performance of isBsonType --- lib/helpers/isBsonType.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/helpers/isBsonType.js b/lib/helpers/isBsonType.js index 01435d3f285..54791cd02dc 100644 --- a/lib/helpers/isBsonType.js +++ b/lib/helpers/isBsonType.js @@ -1,13 +1,15 @@ 'use strict'; -const get = require('./get'); - /*! * Get the bson type, if it exists */ function isBsonType(obj, typename) { - return get(obj, '_bsontype', void 0) === typename; + return ( + typeof obj === 'object' && + obj !== null && + obj._bsontype === typename + ); } module.exports = isBsonType; From bcf382497c33c2709ac55965193dc1b43f4baac9 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Sat, 28 May 2022 17:06:32 +0200 Subject: [PATCH 28/36] improve tests --- test/helpers/isBsonType.test.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/helpers/isBsonType.test.js b/test/helpers/isBsonType.test.js index 54396787926..61ae5adc216 100644 --- a/test/helpers/isBsonType.test.js +++ b/test/helpers/isBsonType.test.js @@ -3,6 +3,9 @@ const assert = require('assert'); const isBsonType = require('../../lib/helpers/isBsonType'); +const Decimal128 = require('mongodb').Decimal128; +const ObjectId = require('mongodb').ObjectId; + describe('isBsonType', () => { it('true for any object with _bsontype property equal typename', () => { assert.ok(isBsonType({ _bsontype: 'MyType' }, 'MyType')); @@ -19,4 +22,12 @@ describe('isBsonType', () => { it('false for any object without _bsontype property', () => { assert.ok(!isBsonType({ }, 'OtherType')); }); + + it('true for Decimal128', () => { + assert.ok(isBsonType(new Decimal128('123'), 'Decimal128')); + }); + + it('true for ObjectId', () => { + assert.ok(isBsonType(new ObjectId(), 'ObjectID')); + }); }); \ No newline at end of file From 67af5b98c9f9746e0269928d86c995972852a7ed Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Sat, 28 May 2022 14:53:49 -0400 Subject: [PATCH 29/36] test(discriminator): add test coverage for discriminators on maps of subdocuments Fix #11720 --- test/model.discriminator.test.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/model.discriminator.test.js b/test/model.discriminator.test.js index 37d3ddd2013..337dae25d6d 100644 --- a/test/model.discriminator.test.js +++ b/test/model.discriminator.test.js @@ -1958,4 +1958,31 @@ describe('model', function() { assert(array.arrayEvent[0].element); }); + + it('handles discriminators on maps of subdocuments (gh-11720)', async function() { + const shapeSchema = Schema({ name: String }, { discriminatorKey: 'kind' }); + const schema = Schema({ shape: { type: Map, of: shapeSchema } }); + + schema.path('shape.$*').discriminator('Circle', Schema({ radius: String })); + schema.path('shape.$*').discriminator('Square', Schema({ side: Number })); + + const Test = db.model('Test', schema); + + let doc = new Test({ + shape: { + a: { kind: 'Circle', radius: 5 }, + b: { kind: 'Square', side: 10 } + } + }); + + assert.strictEqual(doc.shape.get('a').radius, '5'); + assert.strictEqual(doc.shape.get('b').side, 10); + + await doc.save(); + + doc = await Test.findById(doc); + + assert.strictEqual(doc.shape.get('a').radius, '5'); + assert.strictEqual(doc.shape.get('b').side, 10); + }); }); From 1fafd09bba82e7255c3494e218a3cfb1e39b5a86 Mon Sep 17 00:00:00 2001 From: uzlopak Date: Sun, 29 May 2022 00:25:09 +0200 Subject: [PATCH 30/36] fix schematypeoptions --- test/types/schemaTypeOptions.test.ts | 33 ++++++ types/index.d.ts | 143 +------------------------- types/schematypeoptions.d.ts | 144 +++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 142 deletions(-) create mode 100644 test/types/schemaTypeOptions.test.ts create mode 100644 types/schematypeoptions.d.ts diff --git a/test/types/schemaTypeOptions.test.ts b/test/types/schemaTypeOptions.test.ts new file mode 100644 index 00000000000..9c9c897e474 --- /dev/null +++ b/test/types/schemaTypeOptions.test.ts @@ -0,0 +1,33 @@ +import { + AnyArray, + Schema, + SchemaDefinition, + SchemaType, + SchemaTypeOptions, + Types, + ObjectId, + Unpacked, + + BooleanSchemaDefinition, + DateSchemaDefinition, + NumberSchemaDefinition, + ObjectIdSchemaDefinition, + StringSchemaDefinition +} from 'mongoose'; +import { expectType } from 'tsd'; + +(new SchemaTypeOptions()) instanceof SchemaTypeOptions; + +expectType(new SchemaTypeOptions().type); +expectType(new SchemaTypeOptions().type); +expectType(new SchemaTypeOptions().type); +expectType(new SchemaTypeOptions().type); +expectType | undefined>(new SchemaTypeOptions>().type); +expectType | undefined>(new SchemaTypeOptions().type); +expectType(new SchemaTypeOptions().type); +expectType | AnyArray> | undefined>(new SchemaTypeOptions().type); +expectType<(AnyArray> | AnyArray>> | AnyArray>>) | undefined>(new SchemaTypeOptions().type); +expectType | AnyArray> | undefined>(new SchemaTypeOptions().type); +expectType | AnyArray> | undefined>(new SchemaTypeOptions().type); +expectType | AnyArray> | undefined>(new SchemaTypeOptions().type); +expectType<(Function | typeof SchemaType | Schema | SchemaDefinition | Function | AnyArray) | undefined>(new SchemaTypeOptions().type); diff --git a/types/index.d.ts b/types/index.d.ts index d5f2e130297..2d90edcf312 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -6,6 +6,7 @@ /// /// /// +/// declare class NativeDate extends global.Date { } @@ -1032,148 +1033,6 @@ declare module 'mongoose' { type: typeof Schema.Types.Mixed; } - export interface SchemaTypeOptions { - type?: - T extends string ? StringSchemaDefinition : - T extends number ? NumberSchemaDefinition : - T extends boolean ? BooleanSchemaDefinition : - T extends NativeDate ? DateSchemaDefinition : - T extends Map ? SchemaDefinition : - T extends Buffer ? SchemaDefinition : - T extends Types.ObjectId ? ObjectIdSchemaDefinition : - T extends Types.ObjectId[] ? AnyArray | AnyArray> : - T extends object[] ? (AnyArray> | AnyArray>> | AnyArray>>) : - T extends string[] ? AnyArray | AnyArray> : - T extends number[] ? AnyArray | AnyArray> : - T extends boolean[] ? AnyArray | AnyArray> : - T extends Function[] ? AnyArray | AnyArray>> : - T | typeof SchemaType | Schema | SchemaDefinition | Function | AnyArray; - - /** Defines a virtual with the given name that gets/sets this path. */ - alias?: string; - - /** Function or object describing how to validate this schematype. See [validation docs](https://mongoosejs.com/docs/validation.html). */ - validate?: SchemaValidator | AnyArray>; - - /** Allows overriding casting logic for this individual path. If a string, the given string overwrites Mongoose's default cast error message. */ - cast?: string; - - /** - * If true, attach a required validator to this path, which ensures this path - * path cannot be set to a nullish value. If a function, Mongoose calls the - * function and only checks for nullish values if the function returns a truthy value. - */ - required?: boolean | (() => boolean) | [boolean, string] | [() => boolean, string]; - - /** - * The default value for this path. If a function, Mongoose executes the function - * and uses the return value as the default. - */ - default?: T extends Schema.Types.Mixed ? ({} | ((this: any, doc: any) => any)) : (ExtractMongooseArray | ((this: any, doc: any) => Partial>)); - - /** - * The model that `populate()` should use if populating this path. - */ - ref?: string | Model | ((this: any, doc: any) => string | Model); - - /** - * Whether to include or exclude this path by default when loading documents - * using `find()`, `findOne()`, etc. - */ - select?: boolean | number; - - /** - * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose will - * build an index on this path when the model is compiled. - */ - index?: boolean | number | IndexOptions | '2d' | '2dsphere' | 'hashed' | 'text'; - - /** - * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose - * will build a unique index on this path when the - * model is compiled. [The `unique` option is **not** a validator](/docs/validation.html#the-unique-option-is-not-a-validator). - */ - unique?: boolean | number; - - /** - * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose will - * disallow changes to this path once the document is saved to the database for the first time. Read more - * about [immutability in Mongoose here](http://thecodebarbarian.com/whats-new-in-mongoose-5-6-immutable-properties.html). - */ - immutable?: boolean | ((this: any, doc: any) => boolean); - - /** - * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose will - * build a sparse index on this path. - */ - sparse?: boolean | number; - - /** - * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose - * will build a text index on this path. - */ - text?: boolean | number | any; - - /** - * Define a transform function for this individual schema type. - * Only called when calling `toJSON()` or `toObject()`. - */ - transform?: (this: any, val: T) => any; - - /** defines a custom getter for this property using [`Object.defineProperty()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty). */ - get?: (value: any, doc?: this) => T; - - /** defines a custom setter for this property using [`Object.defineProperty()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty). */ - set?: (value: any, priorVal?: T, doc?: this) => any; - - /** array of allowed values for this path. Allowed for strings, numbers, and arrays of strings */ - enum?: Array | ReadonlyArray | { values: Array | ReadonlyArray, message?: string } | { [path: string]: string | number | null }; - - /** The default [subtype](http://bsonspec.org/spec.html) associated with this buffer when it is stored in MongoDB. Only allowed for buffer paths */ - subtype?: number; - - /** The minimum value allowed for this path. Only allowed for numbers and dates. */ - min?: number | NativeDate | [number, string] | [NativeDate, string] | readonly [number, string] | readonly [NativeDate, string]; - - /** The maximum value allowed for this path. Only allowed for numbers and dates. */ - max?: number | NativeDate | [number, string] | [NativeDate, string] | readonly [number, string] | readonly [NativeDate, string]; - - /** Defines a TTL index on this path. Only allowed for dates. */ - expires?: string | number; - - /** If `true`, Mongoose will skip gathering indexes on subpaths. Only allowed for subdocuments and subdocument arrays. */ - excludeIndexes?: boolean; - - /** If set, overrides the child schema's `_id` option. Only allowed for subdocuments and subdocument arrays. */ - _id?: boolean; - - /** If set, specifies the type of this map's values. Mongoose will cast this map's values to the given type. */ - of?: Function | SchemaDefinitionProperty; - - /** If true, uses Mongoose's default `_id` settings. Only allowed for ObjectIds */ - auto?: boolean; - - /** Attaches a validator that succeeds if the data string matches the given regular expression, and fails otherwise. */ - match?: RegExp | [RegExp, string] | readonly [RegExp, string]; - - /** If truthy, Mongoose will add a custom setter that lowercases this string using JavaScript's built-in `String#toLowerCase()`. */ - lowercase?: boolean; - - /** If truthy, Mongoose will add a custom setter that removes leading and trailing whitespace using JavaScript's built-in `String#trim()`. */ - trim?: boolean; - - /** If truthy, Mongoose will add a custom setter that uppercases this string using JavaScript's built-in `String#toUpperCase()`. */ - uppercase?: boolean; - - /** If set, Mongoose will add a custom validator that ensures the given string's `length` is at least the given number. */ - minlength?: number | [number, string] | readonly [number, string]; - - /** If set, Mongoose will add a custom validator that ensures the given string's `length` is at most the given number. */ - maxlength?: number | [number, string] | readonly [number, string]; - - [other: string]: any; - } - export type RefType = | number | string diff --git a/types/schematypeoptions.d.ts b/types/schematypeoptions.d.ts new file mode 100644 index 00000000000..49d8b248926 --- /dev/null +++ b/types/schematypeoptions.d.ts @@ -0,0 +1,144 @@ +declare module 'mongoose' { + + export class SchemaTypeOptions { + type?: + T extends string ? StringSchemaDefinition : + T extends number ? NumberSchemaDefinition : + T extends boolean ? BooleanSchemaDefinition : + T extends NativeDate ? DateSchemaDefinition : + T extends Map ? SchemaDefinition : + T extends Buffer ? SchemaDefinition : + T extends Types.ObjectId ? ObjectIdSchemaDefinition : + T extends Types.ObjectId[] ? AnyArray | AnyArray> : + T extends object[] ? (AnyArray> | AnyArray>> | AnyArray>>) : + T extends string[] ? AnyArray | AnyArray> : + T extends number[] ? AnyArray | AnyArray> : + T extends boolean[] ? AnyArray | AnyArray> : + T extends Function[] ? AnyArray | AnyArray>> : + T | typeof SchemaType | Schema | SchemaDefinition | Function | AnyArray; + + /** Defines a virtual with the given name that gets/sets this path. */ + alias?: string; + + /** Function or object describing how to validate this schematype. See [validation docs](https://mongoosejs.com/docs/validation.html). */ + validate?: SchemaValidator | AnyArray>; + + /** Allows overriding casting logic for this individual path. If a string, the given string overwrites Mongoose's default cast error message. */ + cast?: string; + + /** + * If true, attach a required validator to this path, which ensures this path + * path cannot be set to a nullish value. If a function, Mongoose calls the + * function and only checks for nullish values if the function returns a truthy value. + */ + required?: boolean | (() => boolean) | [boolean, string] | [() => boolean, string]; + + /** + * The default value for this path. If a function, Mongoose executes the function + * and uses the return value as the default. + */ + default?: T extends Schema.Types.Mixed ? ({} | ((this: any, doc: any) => any)) : (ExtractMongooseArray | ((this: any, doc: any) => Partial>)); + + /** + * The model that `populate()` should use if populating this path. + */ + ref?: string | Model | ((this: any, doc: any) => string | Model); + + /** + * Whether to include or exclude this path by default when loading documents + * using `find()`, `findOne()`, etc. + */ + select?: boolean | number; + + /** + * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose will + * build an index on this path when the model is compiled. + */ + index?: boolean | number | IndexOptions | '2d' | '2dsphere' | 'hashed' | 'text'; + + /** + * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose + * will build a unique index on this path when the + * model is compiled. [The `unique` option is **not** a validator](/docs/validation.html#the-unique-option-is-not-a-validator). + */ + unique?: boolean | number; + + /** + * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose will + * disallow changes to this path once the document is saved to the database for the first time. Read more + * about [immutability in Mongoose here](http://thecodebarbarian.com/whats-new-in-mongoose-5-6-immutable-properties.html). + */ + immutable?: boolean | ((this: any, doc: any) => boolean); + + /** + * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose will + * build a sparse index on this path. + */ + sparse?: boolean | number; + + /** + * If [truthy](https://masteringjs.io/tutorials/fundamentals/truthy), Mongoose + * will build a text index on this path. + */ + text?: boolean | number | any; + + /** + * Define a transform function for this individual schema type. + * Only called when calling `toJSON()` or `toObject()`. + */ + transform?: (this: any, val: T) => any; + + /** defines a custom getter for this property using [`Object.defineProperty()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty). */ + get?: (value: any, doc?: this) => T; + + /** defines a custom setter for this property using [`Object.defineProperty()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty). */ + set?: (value: any, priorVal?: T, doc?: this) => any; + + /** array of allowed values for this path. Allowed for strings, numbers, and arrays of strings */ + enum?: Array | ReadonlyArray | { values: Array | ReadonlyArray, message?: string } | { [path: string]: string | number | null }; + + /** The default [subtype](http://bsonspec.org/spec.html) associated with this buffer when it is stored in MongoDB. Only allowed for buffer paths */ + subtype?: number; + + /** The minimum value allowed for this path. Only allowed for numbers and dates. */ + min?: number | NativeDate | [number, string] | [NativeDate, string] | readonly [number, string] | readonly [NativeDate, string]; + + /** The maximum value allowed for this path. Only allowed for numbers and dates. */ + max?: number | NativeDate | [number, string] | [NativeDate, string] | readonly [number, string] | readonly [NativeDate, string]; + + /** Defines a TTL index on this path. Only allowed for dates. */ + expires?: string | number; + + /** If `true`, Mongoose will skip gathering indexes on subpaths. Only allowed for subdocuments and subdocument arrays. */ + excludeIndexes?: boolean; + + /** If set, overrides the child schema's `_id` option. Only allowed for subdocuments and subdocument arrays. */ + _id?: boolean; + + /** If set, specifies the type of this map's values. Mongoose will cast this map's values to the given type. */ + of?: Function | SchemaDefinitionProperty; + + /** If true, uses Mongoose's default `_id` settings. Only allowed for ObjectIds */ + auto?: boolean; + + /** Attaches a validator that succeeds if the data string matches the given regular expression, and fails otherwise. */ + match?: RegExp | [RegExp, string] | readonly [RegExp, string]; + + /** If truthy, Mongoose will add a custom setter that lowercases this string using JavaScript's built-in `String#toLowerCase()`. */ + lowercase?: boolean; + + /** If truthy, Mongoose will add a custom setter that removes leading and trailing whitespace using JavaScript's built-in `String#trim()`. */ + trim?: boolean; + + /** If truthy, Mongoose will add a custom setter that uppercases this string using JavaScript's built-in `String#toUpperCase()`. */ + uppercase?: boolean; + + /** If set, Mongoose will add a custom validator that ensures the given string's `length` is at least the given number. */ + minlength?: number | [number, string] | readonly [number, string]; + + /** If set, Mongoose will add a custom validator that ensures the given string's `length` is at most the given number. */ + maxlength?: number | [number, string] | readonly [number, string]; + + [other: string]: any; + } +} From 74a3b6b58ec4285d0f0c25c72e2f92c8fda81001 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Sat, 28 May 2022 21:54:30 -0400 Subject: [PATCH 31/36] fix(document): avoid manually populating documents that are manually populated in another doc with different unpopulatedValue Fix #11442 --- lib/document.js | 2 +- test/document.test.js | 62 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/lib/document.js b/lib/document.js index ad216686859..431406aa9d4 100644 --- a/lib/document.js +++ b/lib/document.js @@ -1378,7 +1378,7 @@ Document.prototype.$set = function $set(path, val, type, options) { })(); let didPopulate = false; - if (refMatches && val instanceof Document) { + if (refMatches && val instanceof Document && (!val.$__.wasPopulated || utils.deepEqual(val.$__.wasPopulated.value, val._id))) { const unpopulatedValue = (schema && schema.$isSingleNested) ? schema.cast(val, this) : val._id; this.$populated(path, unpopulatedValue, { [populateModelSymbol]: val.constructor }); val.$__.wasPopulated = { value: unpopulatedValue }; diff --git a/test/document.test.js b/test/document.test.js index 0950dd4fc6a..115d87a10d8 100644 --- a/test/document.test.js +++ b/test/document.test.js @@ -11392,4 +11392,66 @@ describe('document', function() { doc = await Test.findById(doc); assert.strictEqual(doc.list[0].toObject().quantity, null); }); + + it('avoids manually populating document that is manually populated in another doc with different unpopulatedValue (gh-11442) (gh-11008)', async function() { + const BarSchema = new Schema({ + name: String, + more: String + }); + const Bar = db.model('Bar', BarSchema); + + // Denormalised Bar schema with just the name, for use on the Foo model + const BarNameSchema = new Schema({ + _id: { + type: Schema.Types.ObjectId, + ref: 'Bar' + }, + name: String + }); + + // Foo model, which contains denormalized bar data (just the name) + const FooSchema = new Schema({ + something: String, + other: Number, + bar: { + type: BarNameSchema, + ref: 'Bar' + } + }); + const Foo = db.model('Foo', FooSchema); + + const Baz = db.model('Baz', new Schema({ bar: { type: 'ObjectId', ref: 'Bar' } })); + + const bar = await Bar.create({ + name: 'I am another Bar', + more: 'With even more data' + }); + const foo = await Foo.create({ + something: 'I am another Foo', + other: 4 + }); + foo.bar = bar; + const baz = await Baz.create({}); + baz.bar = bar; + + assert.ok(foo.populated('bar')); + assert.ok(!baz.populated('bar')); + + let res = foo.toObject({ depopulate: true }); + assert.strictEqual(res.bar._id.toString(), bar._id.toString()); + assert.strictEqual(res.bar.name, 'I am another Bar'); + + res = baz.toObject({ depopulate: true }); + assert.strictEqual(res.bar.toString(), bar._id.toString()); + + const bar2 = await Bar.create({ + name: 'test2' + }); + baz.bar = bar2; + assert.ok(baz.populated('bar')); + + const baz2 = await Baz.create({}); + baz2.bar = bar2; + assert.ok(baz.populated('bar')); + }); }); From 283c47267fb1af91c0290ce31a98bdb74c3d7844 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Sat, 28 May 2022 22:21:11 -0400 Subject: [PATCH 32/36] docs: correct issue number re: #6796 --- lib/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/connection.js b/lib/connection.js index cd36386bc00..f03f42a67c4 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -548,7 +548,7 @@ Connection.prototype.dropDatabase = _wrapConnHelper(function dropDatabase(cb) { // If `dropDatabase()` is called, this model's collection will not be // init-ed. It is sufficiently common to call `dropDatabase()` after // `mongoose.connect()` but before creating models that we want to - // support this. See gh-6967 + // support this. See gh-6796 for (const name of Object.keys(this.models)) { delete this.models[name].$init; } From d082047d98bbd2e6138d30ccc7ca5c5600524652 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Thu, 2 Jun 2022 18:01:56 -0400 Subject: [PATCH 33/36] refining the logic to requests made --- lib/query.js | 42 ++++++++++++++++++++++++++++++++++-------- test/query.test.js | 16 +++++++++------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/lib/query.js b/lib/query.js index 6e355f0b6ee..f26930d424e 100644 --- a/lib/query.js +++ b/lib/query.js @@ -27,6 +27,7 @@ const immediate = require('./helpers/immediate'); const isExclusive = require('./helpers/projection/isExclusive'); const isInclusive = require('./helpers/projection/isInclusive'); const isSubpath = require('./helpers/projection/isSubpath'); +const mpath = require('mpath'); const mquery = require('mquery'); const parseProjection = require('./helpers/projection/parseProjection'); const removeUnusedArrayFilters = require('./helpers/update/removeUnusedArrayFilters'); @@ -2243,9 +2244,9 @@ Query.prototype._find = wrapThunk(function(callback) { } return mongooseOptions.lean ? // call _completeManyLean here? - _completeManyLean(_this.model, docs, fields, userProvidedFields, completeManyOptions, callback) : + _completeManyLean(_this.model.schema, docs, null, completeManyOptions, callback) : // callback(null, docs) : - completeMany(_this.model, docs, fields, userProvidedFields, completeManyOptions, callback); + completeMany(_this.model.schema, docs, null, completeManyOptions, callback); } const pop = helpers.preparePopulationOptionsMQ(_this, mongooseOptions); @@ -4021,14 +4022,14 @@ function _completeOneLean(doc, res, opts, callback) { if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { _completeOneLean(item[k], res, opts); } - opts.lean.transform({}, item); + item = opts.lean.transform(item); } }); } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { _completeOneLean(doc[key], res, opts); } } - opts.lean.transform({}, doc); + doc = opts.lean.transform(doc); if (callback) { return callback(null, doc); } else { @@ -4045,7 +4046,9 @@ function _completeOneLean(doc, res, opts, callback) { * ignore */ +/* function _completeManyLean(model, docs, fields, userProvidedFields, opts, callback) { + console.log('this', model.schema.childSchemas); if (opts.lean && opts.lean.transform) { if (Array.isArray(docs)) { docs.forEach(doc => { @@ -4056,13 +4059,13 @@ function _completeManyLean(model, docs, fields, userProvidedFields, opts, callba if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { _completeManyLean(model, item[k], fields, userProvidedFields, opts); } - opts.lean.transform({}, item); + item = opts.lean.transform(item); } }); } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { _completeManyLean(model, doc[key], fields, userProvidedFields, opts); } - opts.lean.transform({}, doc); + doc = opts.lean.transform(doc); } }); } else { @@ -4073,13 +4076,13 @@ function _completeManyLean(model, docs, fields, userProvidedFields, opts, callba if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { _completeManyLean(model, item[k], fields, userProvidedFields, opts); } - opts.lean.transform({}, item); + item = opts.lean.transform(item); } }); } else if (typeof docs[key] === 'object' && docs[key] != null && Object.keys(docs[key]).length) { _completeManyLean(model, docs[key], fields, userProvidedFields, opts); } - opts.lean.transform({}, docs); + docs = opts.lean.transform(docs); } } } @@ -4089,7 +4092,30 @@ function _completeManyLean(model, docs, fields, userProvidedFields, opts, callba } return callback(null, docs); } +*/ +function _completeManyLean(schema, docs, path, opts, callback) { + if (opts.lean && opts.lean.transform) { + for (let i = 0; i < schema.childSchemas.length; i++) { + const childPath = path ? path + '.' + schema.childSchemas[i].model.path : schema.childSchemas[i].model.path; + const _schema = schema.childSchemas[i].schema; + console.log('childPath', childPath, 'schema', _schema); + let doc = mpath.get(childPath, docs); + console.log('before if', doc) + if (doc == null) { + continue; + } + doc = opts.lean.transform(doc[0]); + console.log('doc', doc); + _completeManyLean(_schema, doc, childPath, opts); + } + } + + if (!callback) { + return; + } + return callback(null, docs); +} /*! * Override mquery.prototype._mergeUpdate to handle mongoose objects in * updates. diff --git a/test/query.test.js b/test/query.test.js index 6d0a989f819..83d0431cba0 100644 --- a/test/query.test.js +++ b/test/query.test.js @@ -3976,17 +3976,19 @@ describe('Query', function() { }); const Test = db.model('gh10423', testSchema); await Test.create({ name: 'foo', foo: [{ sub: 'Test' }, { sub: 'Testerson' }], otherName: { nickName: 'Bar' } }); - const result = await Test.find().lean({ transform: (doc, ret) => { - delete ret._id; - return ret; + const result = await Test.find().lean({ transform: (doc) => { + delete doc._id; + return doc; } }); - assert.equal(result[0]._id, undefined); + // only OtherName and foo should have _id stripped + console.log(result[0]) + assert(result[0]._id); assert.equal(result[0].otherName._id, undefined); assert.equal(result[0].foo[0]._id, undefined); assert.equal(result[0].foo[1]._id, undefined); - const single = await Test.findOne().lean({ transform: (doc, ret) => { - delete ret._id; - return ret; + const single = await Test.findOne().lean({ transform: (doc) => { + delete doc._id; + return doc; } }); assert.equal(single._id, undefined); assert.equal(single.otherName._id, undefined); From 4c0ecbe945341cec840ca9d208de7139360032ff Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Fri, 3 Jun 2022 10:57:29 -0400 Subject: [PATCH 34/36] reworked `completeManyLean()` to be in line with requested change --- lib/query.js | 56 ++++------------------------------------------ test/query.test.js | 2 +- 2 files changed, 5 insertions(+), 53 deletions(-) diff --git a/lib/query.js b/lib/query.js index f26930d424e..65c2f696a67 100644 --- a/lib/query.js +++ b/lib/query.js @@ -4046,67 +4046,19 @@ function _completeOneLean(doc, res, opts, callback) { * ignore */ -/* -function _completeManyLean(model, docs, fields, userProvidedFields, opts, callback) { - console.log('this', model.schema.childSchemas); - if (opts.lean && opts.lean.transform) { - if (Array.isArray(docs)) { - docs.forEach(doc => { - for (const key of Object.keys(doc)) { - if (Array.isArray(doc[key])) { - doc[key].forEach((item) => { - for (const k in item) { - if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { - _completeManyLean(model, item[k], fields, userProvidedFields, opts); - } - item = opts.lean.transform(item); - } - }); - } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { - _completeManyLean(model, doc[key], fields, userProvidedFields, opts); - } - doc = opts.lean.transform(doc); - } - }); - } else { - for (const key of Object.keys(docs)) { - if (Array.isArray(docs[key])) { - docs[key].forEach((item) => { - for (const k in item) { - if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { - _completeManyLean(model, item[k], fields, userProvidedFields, opts); - } - item = opts.lean.transform(item); - } - }); - } else if (typeof docs[key] === 'object' && docs[key] != null && Object.keys(docs[key]).length) { - _completeManyLean(model, docs[key], fields, userProvidedFields, opts); - } - docs = opts.lean.transform(docs); - } - } - } - - if (!callback) { - return; - } - return callback(null, docs); -} -*/ - function _completeManyLean(schema, docs, path, opts, callback) { if (opts.lean && opts.lean.transform) { for (let i = 0; i < schema.childSchemas.length; i++) { const childPath = path ? path + '.' + schema.childSchemas[i].model.path : schema.childSchemas[i].model.path; const _schema = schema.childSchemas[i].schema; - console.log('childPath', childPath, 'schema', _schema); let doc = mpath.get(childPath, docs); - console.log('before if', doc) if (doc == null) { continue; } - doc = opts.lean.transform(doc[0]); - console.log('doc', doc); + doc = doc.flat(); + for (let i = 0; i < doc.length; i++) { + opts.lean.transform(doc[i]); + } _completeManyLean(_schema, doc, childPath, opts); } } diff --git a/test/query.test.js b/test/query.test.js index 83d0431cba0..c92111e2040 100644 --- a/test/query.test.js +++ b/test/query.test.js @@ -3981,7 +3981,7 @@ describe('Query', function() { return doc; } }); // only OtherName and foo should have _id stripped - console.log(result[0]) + console.log(result[0]); assert(result[0]._id); assert.equal(result[0].otherName._id, undefined); assert.equal(result[0].foo[0]._id, undefined); From d11c511cb4ad88792b91b6b2cdc0fa7dee075b73 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Fri, 3 Jun 2022 11:19:02 -0400 Subject: [PATCH 35/36] reworked `_completeOneLean()` to be in line with requested changes --- lib/query.js | 33 +++++++++++++++++---------------- test/query.test.js | 4 +--- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/lib/query.js b/lib/query.js index 65c2f696a67..c3e426298a5 100644 --- a/lib/query.js +++ b/lib/query.js @@ -2435,7 +2435,7 @@ Query.prototype._completeOne = function(doc, res, callback) { } } return mongooseOptions.lean ? - _completeOneLean(doc, res, options, callback) : + _completeOneLean(model.schema, doc, null, res, options, callback) : completeOne(model, doc, res, options, projection, userProvidedFields, null, callback); } @@ -2446,7 +2446,7 @@ Query.prototype._completeOne = function(doc, res, callback) { if (err != null) { return callback(err); } - _completeOneLean(doc, res, options, callback); + _completeOneLean(model.schema, doc, null, res, options, callback); }); } @@ -4013,23 +4013,24 @@ Query.prototype._findAndModify = function(type, callback) { * ignore */ -function _completeOneLean(doc, res, opts, callback) { +function _completeOneLean(schema, doc, path, res, opts, callback) { if (opts.lean && opts.lean.transform) { - for (const key of Object.keys(doc)) { - if (Array.isArray(doc[key])) { - doc[key].forEach((item) => { - for (const k in item) { - if (typeof item[k] === 'object' && item[k] != null && Object.keys(item[k]).length) { - _completeOneLean(item[k], res, opts); - } - item = opts.lean.transform(item); - } - }); - } else if (typeof doc[key] === 'object' && doc[key] != null && Object.keys(doc[key]).length) { - _completeOneLean(doc[key], res, opts); + for (let i = 0; i < schema.childSchemas.length; i++) { + const childPath = path ? path + '.' + schema.childSchemas[i].model.path : schema.childSchemas[i].model.path; + const _schema = schema.childSchemas[i].schema; + const obj = mpath.get(childPath, doc); + if (obj == null) { + continue; + } + if (Array.isArray(obj)) { + for (let i = 0; i < obj.length; i++) { + opts.lean.transform(obj[i]); + } + } else { + opts.lean.transform(obj); } + _completeOneLean(_schema, obj, childPath, res, opts); } - doc = opts.lean.transform(doc); if (callback) { return callback(null, doc); } else { diff --git a/test/query.test.js b/test/query.test.js index c92111e2040..d09a4135bdc 100644 --- a/test/query.test.js +++ b/test/query.test.js @@ -3980,8 +3980,6 @@ describe('Query', function() { delete doc._id; return doc; } }); - // only OtherName and foo should have _id stripped - console.log(result[0]); assert(result[0]._id); assert.equal(result[0].otherName._id, undefined); assert.equal(result[0].foo[0]._id, undefined); @@ -3990,7 +3988,7 @@ describe('Query', function() { delete doc._id; return doc; } }); - assert.equal(single._id, undefined); + assert(single._id); assert.equal(single.otherName._id, undefined); assert.equal(single.foo[0]._id, undefined); assert.equal(single.foo[0]._id, undefined); From 1c7696e84e07267a91e94d9f1aa181a980c7fed0 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Fri, 3 Jun 2022 12:28:52 -0400 Subject: [PATCH 36/36] should work now --- lib/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/query.js b/lib/query.js index c3e426298a5..ace76a7f9b6 100644 --- a/lib/query.js +++ b/lib/query.js @@ -2246,7 +2246,7 @@ Query.prototype._find = wrapThunk(function(callback) { // call _completeManyLean here? _completeManyLean(_this.model.schema, docs, null, completeManyOptions, callback) : // callback(null, docs) : - completeMany(_this.model.schema, docs, null, completeManyOptions, callback); + completeMany(_this.model, docs, fields, userProvidedFields, completeManyOptions, callback); } const pop = helpers.preparePopulationOptionsMQ(_this, mongooseOptions);