feat: add jsonParse flag to mongooseAdapter that preserves existing, …

…untracked MongoDB data types (#7338)

## Description

Preserves external data structures stored in MongoDB by avoiding the use
of `JSON.parse(JSON.stringify(mongooseDoc))`.

- [x] I have read and understand the
[CONTRIBUTING.md](https://github.com/payloadcms/payload/blob/main/CONTRIBUTING.md)
document in this repository.

## Type of change

- [x] New feature (non-breaking change which adds functionality)

## Checklist:

- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] Existing test suite passes locally with my changes

docs/database/mongodb.mdx

@@ -34,6 +34,7 @@ export default buildConfig({
 |----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `autoPluralization`  | Tell Mongoose to auto-pluralize any collection names if it encounters any singular words used as collection `slug`s.                                                                                                                                                                          |
 | `schemaOptions`      | Customize schema options for all Mongoose schemas created internally.   |
+| `jsonParse`          | Set to false to disable the automatic JSON stringify/parse of data queried by MongoDB. For example, if you have data not tracked by Payload such as `Date` fields and similar, you can use this option to ensure that existing `Date` properties remain as `Date` and not strings. |
 | `collections`        | Options on a collection-by-collection basis. [More](#collections-options)  |
 | `globals`            | Options for the Globals collection created by Payload. [More](#globals-options) |
 | `connectOptions`     | Customize MongoDB connection options. Payload will connect to your MongoDB database using default options which you can override and extend to include all the [options](https://mongoosejs.com/docs/connections.html#options) available to mongoose.                                         |
@@ -74,4 +75,28 @@ const db = mongooseAdapter({
 
 ### Global Options
 
-Payload automatically creates a single `globals` collection that correspond with any Payload globals that you define. When you initialize the `mongooseAdapter`, you can specify settings here for your globals in a similar manner to how you can for collections above. Right now, the only property available is `schemaOptions` but more may be added in the future.
+Payload automatically creates a single `globals` collection that correspond with any Payload globals that you define. When you initialize the `mongooseAdapter`, you can specify settings here for your globals in a similar manner to how you can for collections above. Right now, the only property available is `schemaOptions` but more may be added in the future.
+
+### Preserving externally managed data
+
+You can use Payload in conjunction with an existing MongoDB database, where you might have some fields "tracked" in Payload via corresponding field configs, and other fields completely unknown to Payload.
+
+If you have external field data in existing MongoDB collections which you'd like to use in combination with Payload, and you don't want to lose those external fields, you can configure Payload to "preserve" that data while it makes updates to your existing documents.
+
+To do this, the first step is to configure Mongoose's `strict` property, which tells Mongoose to write all data that it receives (and not disregard any data that it does not know about).
+
+The second step is to disable Payload's automatic JSON parsing of documents it receives from MongoDB.
+
+Here's an example for how to configure your Mongoose adapter to preserve external collection fields that are not tracked by Payload:
+
+```ts
+mongooseAdapter({
+  url: process.env.DATABASE_URI,
+  // Disable the JSON parsing that Payload performs
+  jsonParse: false,
+  // Disable strict mode for Mongoose
+  schemaOptions: {
+    strict: false,
+  },
+})
+```

packages/db-mongodb/src/create.ts

@@ -1,9 +1,10 @@
 import type { Create } from 'payload/database'
-import type { Document, PayloadRequest } from 'payload/types'
+import type { PayloadRequest } from 'payload/types'
 
 import type { MongooseAdapter } from '.'
 
 import handleError from './utilities/handleError'
+import sanitizeInternalFields from './utilities/sanitizeInternalFields'
 import { withSession } from './withSession'
 
 export const create: Create = async function create(
@@ -19,15 +20,13 @@ export const create: Create = async function create(
     handleError(error, req)
   }
 
-  // doc.toJSON does not do stuff like converting ObjectIds to string, or date strings to date objects. That's why we use JSON.parse/stringify here
-  const result: Document = JSON.parse(JSON.stringify(doc))
+  const result = this.jsonParse ? JSON.parse(JSON.stringify(doc)) : doc.toObject()
+
   const verificationToken = doc._verificationToken
 
-  // custom id type reset
-  result.id = result._id
   if (verificationToken) {
     result._verificationToken = verificationToken
   }
 
-  return result
+  return sanitizeInternalFields(result)
 }

packages/db-mongodb/src/createGlobal.ts

@@ -19,10 +19,8 @@ export const createGlobal: CreateGlobal = async function createGlobal(
 
   let [result] = (await Model.create([global], options)) as any
 
-  result = JSON.parse(JSON.stringify(result))
+  result = this.jsonParse ? JSON.parse(JSON.stringify(result)) : result.toObject()
 
-  // custom id type reset
-  result.id = result._id
   result = sanitizeInternalFields(result)
 
   return result

packages/db-mongodb/src/createGlobalVersion.ts

@@ -1,9 +1,9 @@
 import type { CreateGlobalVersion } from 'payload/database'
 import type { PayloadRequest } from 'payload/types'
-import type { Document } from 'payload/types'
 
 import type { MongooseAdapter } from '.'
 
+import sanitizeInternalFields from './utilities/sanitizeInternalFields'
 import { withSession } from './withSession'
 
 export const createGlobalVersion: CreateGlobalVersion = async function createGlobalVersion(
@@ -52,13 +52,12 @@ export const createGlobalVersion: CreateGlobalVersion = async function createGlo
     options,
   )
 
-  const result: Document = JSON.parse(JSON.stringify(doc))
+  const result = this.jsonParse ? JSON.parse(JSON.stringify(doc)) : doc.toObject()
+
   const verificationToken = doc._verificationToken
 
-  // custom id type reset
-  result.id = result._id
   if (verificationToken) {
     result._verificationToken = verificationToken
   }
-  return result
+  return sanitizeInternalFields(result)
 }

packages/db-mongodb/src/createVersion.ts

@@ -1,9 +1,9 @@
 import type { CreateVersion } from 'payload/database'
 import type { PayloadRequest } from 'payload/types'
-import type { Document } from 'payload/types'
 
 import type { MongooseAdapter } from '.'
 
+import sanitizeInternalFields from './utilities/sanitizeInternalFields'
 import { withSession } from './withSession'
 
 export const createVersion: CreateVersion = async function createVersion(
@@ -60,13 +60,13 @@ export const createVersion: CreateVersion = async function createVersion(
     options,
   )
 
-  const result: Document = JSON.parse(JSON.stringify(doc))
+  const result = this.jsonParse ? JSON.parse(JSON.stringify(doc)) : doc.toObject()
+
   const verificationToken = doc._verificationToken
 
-  // custom id type reset
-  result.id = result._id
   if (verificationToken) {
     result._verificationToken = verificationToken
   }
-  return result
+
+  return sanitizeInternalFields(result)
 }

packages/db-mongodb/src/deleteOne.ts

@@ -1,6 +1,5 @@
 import type { DeleteOne } from 'payload/database'
 import type { PayloadRequest } from 'payload/types'
-import type { Document } from 'payload/types'
 
 import type { MongooseAdapter } from '.'
 
@@ -19,13 +18,9 @@ export const deleteOne: DeleteOne = async function deleteOne(
     where,
   })
 
-  const doc = await Model.findOneAndDelete(query, options).lean()
+  let doc = await Model.findOneAndDelete(query, options).lean()
 
-  let result: Document = JSON.parse(JSON.stringify(doc))
+  doc = this.jsonParse ? JSON.parse(JSON.stringify(doc)) : doc
 
-  // custom id type reset
-  result.id = result._id
-  result = sanitizeInternalFields(result)
-
-  return result
+  return sanitizeInternalFields(doc)
 }

packages/db-mongodb/src/find.ts

@@ -86,13 +86,12 @@ export const find: Find = async function find(
   }
 
   const result = await Model.paginate(query, paginationOptions)
-  const docs = JSON.parse(JSON.stringify(result.docs))
+
+  const docs = this.jsonParse ? JSON.parse(JSON.stringify(result.docs)) : result.docs
 
   return {
     ...result,
     docs: docs.map((doc) => {
-      // eslint-disable-next-line no-param-reassign
-      doc.id = doc._id
       return sanitizeInternalFields(doc)
     }),
   }

packages/db-mongodb/src/findGlobal.ts

@@ -30,12 +30,16 @@ export const findGlobal: FindGlobal = async function findGlobal(
   if (!doc) {
     return null
   }
+
+  if (this.jsonParse) {
+    doc = JSON.parse(JSON.stringify(doc))
+  }
+
   if (doc._id) {
-    doc.id = doc._id
+    doc.id = JSON.parse(JSON.stringify(doc._id))
     delete doc._id
   }
 
-  doc = JSON.parse(JSON.stringify(doc))
   doc = sanitizeInternalFields(doc)
 
   return doc

packages/db-mongodb/src/findGlobalVersions.ts

@@ -105,13 +105,12 @@ export const findGlobalVersions: FindGlobalVersions = async function findGlobalV
   }
 
   const result = await Model.paginate(query, paginationOptions)
-  const docs = JSON.parse(JSON.stringify(result.docs))
+
+  const docs = this.jsonParse ? JSON.parse(JSON.stringify(result.docs)) : result.docs
 
   return {
     ...result,
     docs: docs.map((doc) => {
-      // eslint-disable-next-line no-param-reassign
-      doc.id = doc._id
       return sanitizeInternalFields(doc)
     }),
   }

packages/db-mongodb/src/findOne.ts

@@ -1,7 +1,6 @@
 import type { MongooseQueryOptions } from 'mongoose'
 import type { FindOne } from 'payload/database'
 import type { PayloadRequest } from 'payload/types'
-import type { Document } from 'payload/types'
 
 import type { MongooseAdapter } from '.'
 
@@ -24,17 +23,15 @@ export const findOne: FindOne = async function findOne(
     where,
   })
 
-  const doc = await Model.findOne(query, {}, options)
+  let doc = await Model.findOne(query, {}, options)
 
   if (!doc) {
     return null
   }
 
-  let result: Document = JSON.parse(JSON.stringify(doc))
+  doc = this.jsonParse ? JSON.parse(JSON.stringify(doc)) : doc
 
-  // custom id type reset
-  result.id = result._id
-  result = sanitizeInternalFields(result)
+  doc = sanitizeInternalFields(doc)
 
-  return result
+  return doc
 }

packages/db-mongodb/src/findVersions.ts

@@ -101,13 +101,12 @@ export const findVersions: FindVersions = async function findVersions(
   }
 
   const result = await Model.paginate(query, paginationOptions)
-  const docs = JSON.parse(JSON.stringify(result.docs))
+
+  const docs = this.jsonParse ? JSON.parse(JSON.stringify(result.docs)) : result.docs
 
   return {
     ...result,
     docs: docs.map((doc) => {
-      // eslint-disable-next-line no-param-reassign
-      doc.id = doc._id
       return sanitizeInternalFields(doc)
     }),
   }

packages/db-mongodb/src/index.ts

@@ -56,13 +56,16 @@ export interface Args {
     /** Set false to disable $facet aggregation in non-supporting databases, Defaults to true */
     useFacet?: boolean
   }
+
   /** Set to true to disable hinting to MongoDB to use 'id' as index. This is currently done when counting documents for pagination. Disabling this optimization might fix some problems with AWS DocumentDB. Defaults to false */
   disableIndexHints?: boolean
   /** Define Mongoose options for the globals collection.
    */
   globals?: {
     schemaOptions?: SchemaOptions
   }
+  /** Set to false to disable the automatic JSON stringify/parse of data queried by MongoDB. For example, if you have data not tracked by Payload such as `Date` fields and similar, you can use this option to ensure that existing `Date` properties remain as `Date` and not strings. */
+  jsonParse?: boolean
   migrationDir?: string
   /** Define default Mongoose schema options for all schemas created.
    */
@@ -87,6 +90,7 @@ export type MongooseAdapter = BaseDatabaseAdapter &
     globalsOptions: {
       schemaOptions?: SchemaOptions
     }
+    jsonParse: boolean
     mongoMemoryServer: any
     schemaOptions?: SchemaOptions
     sessions: Record<number | string, ClientSession>
@@ -114,6 +118,7 @@ declare module 'payload' {
     globalsOptions: {
       schemaOptions?: SchemaOptions
     }
+    jsonParse: boolean
     mongoMemoryServer: any
     schemaOptions?: SchemaOptions
 
@@ -131,6 +136,7 @@ export function mongooseAdapter({
   connectOptions,
   disableIndexHints = false,
   globals,
+  jsonParse = true,
   migrationDir: migrationDirArg,
   schemaOptions,
   transactionOptions = {},
@@ -153,6 +159,7 @@ export function mongooseAdapter({
       disableIndexHints,
       globals: undefined,
       globalsOptions: globals || {},
+      jsonParse,
       mongoMemoryServer: undefined,
       schemaOptions: schemaOptions || {},
       sessions: {},

packages/db-mongodb/src/queryDrafts.ts

@@ -87,12 +87,14 @@ export const queryDrafts: QueryDrafts = async function queryDrafts(
   }
 
   const result = await VersionModel.paginate(versionQuery, paginationOptions)
-  const docs = JSON.parse(JSON.stringify(result.docs))
+
+  const docs = this.jsonParse ? JSON.parse(JSON.stringify(result.docs)) : result.docs
 
   return {
     ...result,
     docs: docs.map((doc) => {
       // eslint-disable-next-line no-param-reassign
+
       doc = {
         _id: doc.parent,
         id: doc.parent,

packages/db-mongodb/src/updateGlobal.ts

@@ -21,10 +21,8 @@ export const updateGlobal: UpdateGlobal = async function updateGlobal(
 
   result = await Model.findOneAndUpdate({ globalType: slug }, data, options)
 
-  result = JSON.parse(JSON.stringify(result))
+  result = this.jsonParse ? JSON.parse(JSON.stringify(result)) : result
 
-  // custom id type reset
-  result.id = result._id
   result = sanitizeInternalFields(result)
 
   return result

packages/db-mongodb/src/updateGlobalVersion.ts

@@ -3,6 +3,7 @@ import type { PayloadRequest, TypeWithID } from 'payload/types'
 
 import type { MongooseAdapter } from '.'
 
+import sanitizeInternalFields from './utilities/sanitizeInternalFields'
 import { withSession } from './withSession'
 
 export async function updateGlobalVersion<T extends TypeWithID>(
@@ -30,16 +31,14 @@ export async function updateGlobalVersion<T extends TypeWithID>(
     where: whereToUse,
   })
 
-  const doc = await VersionModel.findOneAndUpdate(query, versionData, options)
+  let doc = await VersionModel.findOneAndUpdate(query, versionData, options)
 
-  const result = JSON.parse(JSON.stringify(doc))
+  doc = this.jsonParse ? JSON.parse(JSON.stringify(doc)) : doc
 
   const verificationToken = doc._verificationToken
 
-  // custom id type reset
-  result.id = result._id
   if (verificationToken) {
-    result._verificationToken = verificationToken
+    doc._verificationToken = verificationToken
   }
-  return result
+  return sanitizeInternalFields(doc)
 }

packages/db-mongodb/src/updateOne.ts

@@ -32,9 +32,7 @@ export const updateOne: UpdateOne = async function updateOne(
     handleError(error, req)
   }
 
-  result = JSON.parse(JSON.stringify(result))
-  result.id = result._id
-  result = sanitizeInternalFields(result)
+  result = this.jsonParse ? JSON.parse(JSON.stringify(result)) : result
 
-  return result
+  return sanitizeInternalFields(result)
 }