refactor: rewrite the whole parser (AGAIN) (#23)

* feat: add new @scalar/resolver package

* chore: clean up

* test: improve test:prepare error message

* refactor: remove resolver package, integrate into the openapi parser (wip)

* fix: references with whitespace

* fix: invalid reference error message

* test: use new invalid reference error message

* test: disable file system tests

* chore: clean up

* chore: try to add some more tests

* chore: add diff testing

* chore: add failing test case

* chore: make the example even simpler

* chore: add some comments

* feat: recursively resolve pointer reference before pushing to object

* fix: pointers with white space can’t be resolved

* chore: disable old test

* fix: can’t access $ref of undefined

* fix: references inside of arrays can’t be found

* chore: disable old test

* chore: test more real-world examples

* fix: original properties are overwritten with the referenced content

* chore: diff testing for more files

* fix: existing properties are overwritten with referenced content

* fix: referenced content loses reference to original object

* fix: throws exception when the pointer does not exist

* fix: $ref is not a string

* chore: add failing max call stack example file (wip)

* refactor: migrate checkReferences method

* chore: rewrite resolve function to use less memory wip

* refactor: rename basic test file

* chore: add another test

* refactor: write resolve function from scratch (wip)

* refactor: write resolve function from scratch (wip)

* refactor: rename findReference to resolveUri

* fix: detect circular references and throw error

* chore: resolve circular refs wip

* fix: resolve circular references

* chore: docstrings and cleanup

* chore: update diff test invalid files list

* chore: improve circular reference test

* chore: rewrite the resolve function from scratch (again)

* refactor: rewrite the resolve method *again* (wip)

* refactor: minor bug fixes

* test: output errors (if they appear)

* chore: clean up

* docs(changeset): refactor: rewrote the whole parser (again)

* docs(changeset): fix: circular references can not be resolved

* docs(changeset): fix: references inside references are not resolved

* docs: update README

* docs: add mentions to the README

* chore: clean up

* fix: ts errors

---------

Co-authored-by: tmastrom <thomas.mastromonaco@gmail.com>

.changeset/heavy-tools-peel.md

@@ -0,0 +1,5 @@
+---
+"@scalar/openapi-parser": minor
+---
+
+fix: circular references can not be resolved

.changeset/plenty-steaks-shave.md

@@ -0,0 +1,5 @@
+---
+"@scalar/openapi-parser": minor
+---
+
+fix: references inside references are not resolved

.changeset/violet-guests-smash.md

@@ -0,0 +1,5 @@
+---
+"@scalar/openapi-parser": minor
+---
+
+refactor: rewrote the whole parser (again)

README.md

@@ -15,7 +15,7 @@ Modern OpenAPI parser written in TypeScript with support for OpenAPI 3.1, OpenAP
 - [x] Written in TypeScript
 - [x] Runs in Node.js and in the browser (without any polyfills or configuration)
 - [x] Tested with hundreds of real world examples
-- [x] Amazing error output
+- [ ] Amazing error output
 - [ ] Support for OpenAPI 4.0 👀
 
 ## Limitations
@@ -98,6 +98,15 @@ const specification = openapi()
 
 We are API nerds. You too? Let’s chat on Discord: <https://discord.gg/8HeZcRGPFS>
 
+## Thank you!
+
+Thanks a ton for all the help and inspiration:
+
+- [@philsturgeon](https://github.com/philsturgeon) to make sure we build something we won’t hate.
+- We took a lot of inspiration from [@seriousme]https://github.com/seriousme and his package [openapi-schema-validator](https://github.com/seriousme/openapi-schema-validator) early-on.
+- You could consider this package the modern successor of [@apidevtools/swagger-parser](https://github.com/APIDevTools/swagger-parser), we even test against it to make sure we’re getting the same results (where intended).
+- We stole a lot of example specification from [@mermade](https://github.com/mermade) to test against.
+
 ## Contributors
 
 <!-- readme: collaborators,contributors -start -->

packages/openapi-parser/package.json

@@ -36,21 +36,21 @@
     "test": "vitest"
   },
   "devDependencies": {
+    "@apidevtools/swagger-parser": "^10.1.0",
     "@modyfi/vite-plugin-yaml": "^1.1.0",
     "@types/js-yaml": "^4.0.9",
+    "glob": "^10.3.10",
+    "just-diff": "^6.0.2",
+    "vite": "^5.1.4",
     "vite-plugin-dts": "^3.7.3"
   },
   "dependencies": {
     "@humanwhocodes/momoa": "^3.0.1",
-    "@types/node": "^20.11.24",
-    "@hyperjump/browser": "^1.1.3",
-    "@hyperjump/json-schema": "^1.7.2",
+    "@types/node": "^20.11.20",
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",
-    "glob": "^10.3.10",
     "js-yaml": "^4.1.0",
-    "json-to-ast": "^2.1.0",
     "jsonpointer": "^5.0.1",
     "leven": "^4.0.0",
     "openapi-types": "^12.1.3",

packages/openapi-parser/src/configuration.ts

@@ -24,7 +24,9 @@ export const ERRORS = {
   // URI_MUST_BE_STRING: 'uri parameter or $id attribute must be a string',
   OPENAPI_VERSION_NOT_SUPPORTED:
     'Cannot find supported Swagger/OpenAPI version in specification, version must be a string.',
-  INVALID_REFERENCE: 'Can’t resolve $ref: %s',
+  INVALID_REFERENCE: 'Can’t resolve URI: %s',
+  EXTERNAL_REFERENCE_NOT_SUPPORTED:
+    'External references are not supported yet: %s',
 } as const
 
 export type VALIDATOR_ERROR = keyof typeof ERRORS

packages/openapi-parser/src/lib/Validator/Validator.ts

@@ -37,7 +37,9 @@ export class Validator {
   public specification: AnyObject
 
   resolveReferences(filesystem?: Filesystem) {
-    return resolveReferences(filesystem, true)
+    return resolveReferences(
+      filesystem.find((file) => file.isEntrypoint === true).specification,
+    )
   }
 
   /**
@@ -89,7 +91,7 @@ export class Validator {
 
       // Check if the references are valid
       if (schemaResult) {
-        return checkReferences(filesystem)
+        return checkReferences(entrypoint.specification)
       }
 
       // Error handling

packages/openapi-parser/src/lib/Validator/resolveFromFilesystem.ts → packages/openapi-parser/src/lib/Validator/_backup/resolveFromFilesystem.ts

@@ -1,5 +1,6 @@
-import { path } from '../../polyfills'
-import type { Filesystem, FilesystemEntry } from '../../types'
+// @ts-nocheck
+import { path } from '../../../polyfills'
+import type { Filesystem, FilesystemEntry } from '../../../types'
 import { resolveReferences } from './resolveReferences'
 
 export function resolveFromFilesystem(

packages/openapi-parser/src/lib/Validator/resolveUri.test.ts → packages/openapi-parser/src/lib/Validator/_backup/resolveUri.test.ts

@@ -1,14 +1,15 @@
+// @ts-nocheck
 import { describe, expect, it } from 'vitest'
 
-import { getListOfReferences, loadFiles, resolve, validate } from '../../../src'
-import { relativePath } from '../../../tests/utils'
+import { getListOfReferences, loadFiles, resolve, validate } from '../../..'
+import { relativePath } from '../../../../tests/utils'
 
 const EXAMPLE_FILE = relativePath(
   import.meta.url,
-  '../../tests/resolveUri/invalid/openapi.yaml',
+  '../../../tests/resolveUri/invalid/openapi.yaml',
 )
 
-describe('resolveUri', async () => {
+describe.skip('resolveUri', async () => {
   it('resolves internal references', async () => {
     const specification = {
       openapi: '3.0.3',
@@ -48,6 +49,7 @@ describe('resolveUri', async () => {
 
     const result = await resolve(specification)
 
+    expect(result.errors).toBe(undefined)
     expect(result.valid).toBe(true)
     expect(
       result.schema?.paths?.['/upload']?.post?.responses?.[401]?.content[
@@ -78,7 +80,7 @@ describe('resolveUri', async () => {
     expect(result.valid).toBe(false)
     expect(result.errors?.length).toEqual(1)
     expect(result.errors?.[0].error).toEqual(
-      'Can’t resolve $ref: #/foo/bar/does_not_exist',
+      'Can’t resolve URI: #/foo/bar/does_not_exist',
     )
   })
 
@@ -90,11 +92,11 @@ describe('resolveUri', async () => {
     expect(result.valid).toBe(false)
     expect(result.errors.length).toBe(1)
     expect(result.errors[0].error).toBe(
-      `Can’t resolve $ref: schemas/does-not-exist.yaml`,
+      `Can’t resolve URI: schemas/does-not-exist.yaml`,
     )
   })
 
-  it('resolves file references', async () => {
+  it.todo('resolves file references', async () => {
     const baseFile = {
       openapi: '3.0.3',
       info: {
@@ -145,6 +147,7 @@ describe('resolveUri', async () => {
       },
     ])
 
+    expect(result.errors).toBe(undefined)
     expect(result.valid).toBe(true)
     expect(
       result.schema?.paths?.['/upload']?.post?.responses?.[401]?.content[
@@ -153,7 +156,7 @@ describe('resolveUri', async () => {
     ).toEqual('string')
   })
 
-  it('resolves file references with pointers', async () => {
+  it.todo('resolves file references with pointers', async () => {
     const baseFile = {
       openapi: '3.0.3',
       info: {
@@ -209,6 +212,7 @@ describe('resolveUri', async () => {
       },
     ])
 
+    expect(result.errors).toBe(undefined)
     expect(result.valid).toBe(true)
     expect(
       result.schema?.paths?.['/upload']?.post?.responses?.[401]?.content[

packages/openapi-parser/src/lib/Validator/resolveUri.ts → packages/openapi-parser/src/lib/Validator/_backup/resolveUri.ts

@@ -1,7 +1,8 @@
+// @ts-nocheck
 import { unescapeJsonPointer } from 'ajv/dist/compile/util'
 
-import { ERRORS } from '../../configuration'
-import type { AnyObject, Filesystem, FilesystemEntry } from '../../types'
+import { ERRORS } from '../../../configuration'
+import type { AnyObject, Filesystem, FilesystemEntry } from '../../../types'
 import { resolveFromFilesystem } from './resolveFromFilesystem'
 
 /**

packages/openapi-parser/src/lib/Validator/checkReferences.test.ts

@@ -1,57 +1,81 @@
 import { describe, expect, it } from 'vitest'
 
-import { makeFilesystem } from '../../utils/makeFilesystem'
 import { checkReferences } from './checkReferences'
 
 describe('checkReferences', () => {
   it('returns true for a simple internal reference', () => {
-    const specification = `openapi: 3.1.0
-info:
-  title: Hello World
-  version: 2.0.0
-paths:
+    const specification = {
+      openapi: '3.1.0',
+      info: {
+        title: 'Hello World',
+        version: '2.0.0',
+      },
+      paths: {
+        '/foobar': {
+          post: {
+            description: 'Example',
+            requestBody: {
+              content: {
+                'application/json': {
+                  schema: {
+                    $ref: '#/components/schemas/Foobar',
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+      components: {
+        schemas: {
+          Foobar: {
+            type: 'string',
+            example: 'Hello World!',
+          },
+        },
+      },
+    }
 
-  '/foobar':
-    post:
-      description: 'Example'
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/Foobar'
-components:
-  schemas:
-    Foobar:
-      type: string
-      example: 'Hello World!'`
-
-    const result = checkReferences(makeFilesystem(specification))
+    const result = checkReferences(specification)
+    expect(result.errors).toBe(undefined)
     expect(result.valid).toBe(true)
     expect(result.errors).toBeUndefined()
   })
 
   it('returns false for a broken internal reference', () => {
-    const specification = `openapi: 3.1.0
-info:
-  title: Hello World
-  version: 2.0.0
-paths:
-
-  '/foobar':
-    post:
-      description: 'Example'
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: '#/components/schemas/Barfoo'
-components:
-  schemas:
-    Foobar:
-      type: string
-      example: 'Hello World!'`
+    const specification = {
+      openapi: '3.1.0',
+      info: {
+        title: 'Hello World',
+        version: '2.0.0',
+      },
+      paths: {
+        '/foobar': {
+          post: {
+            description: 'Example',
+            requestBody: {
+              content: {
+                'application/json': {
+                  schema: {
+                    $ref: '#/components/schemas/Barfoo',
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+      components: {
+        schemas: {
+          Foobar: {
+            type: 'string',
+            example: 'Hello World!',
+          },
+        },
+      },
+    }
 
-    const result = checkReferences(makeFilesystem(specification))
+    const result = checkReferences(specification)
 
     expect(result.valid).toBe(false)
     expect(result.errors).not.toBeUndefined()

packages/openapi-parser/src/lib/Validator/checkReferences.ts

@@ -1,20 +1,19 @@
-import type { Filesystem } from '../../types'
+import { AnyObject } from '../../types'
 import { resolveReferences } from './resolveReferences'
 import { transformErrors } from './transformErrors'
 
-export function checkReferences(filesystem: Filesystem) {
-  const entrypoint = filesystem.find((file) => file.isEntrypoint === true)
-
+// TODO: Adapat for the new function
+export function checkReferences(specification: AnyObject) {
   try {
-    resolveReferences(filesystem, false)
+    resolveReferences(specification)
 
     return {
       valid: true,
     }
-  } catch (err) {
+  } catch (error) {
     return {
       valid: false,
-      errors: transformErrors(entrypoint, err.message),
+      errors: transformErrors(specification, error.message),
     }
   }
 }

packages/openapi-parser/src/lib/Validator/escapeJsonPointer.test.ts

@@ -2,4 +2,14 @@ import { describe, expect, it } from 'vitest'
 
 import { escapeJsonPointer } from './escapeJsonPointer'
 
-describe.todo('escapeJsonPointer', async () => {})
+describe('escapeJsonPointer', async () => {
+  it('should escape a slash', () => {
+    expect(escapeJsonPointer('application/json')).toBe('application~1json')
+  })
+
+  it('should escape multiple slashes', () => {
+    expect(escapeJsonPointer('/api/users/{id}/reports')).toBe(
+      '~1api~1users~1{id}~1reports',
+    )
+  })
+})

packages/openapi-parser/src/lib/Validator/index.ts

@@ -1 +1,2 @@
 export * from './Validator'
+export * from './resolveReferences'