diff --git a/packages/eslint-mdx/src/parser.ts b/packages/eslint-mdx/src/parser.ts
index 6b5d8d21..7c122a37 100644
--- a/packages/eslint-mdx/src/parser.ts
+++ b/packages/eslint-mdx/src/parser.ts
@@ -24,7 +24,8 @@ import {
   ParserOptions,
 } from './types'
 
-export const mdxProcessor = unified().use(remarkParse).use(remarkMdx).freeze()
+export const mdProcessor = unified().use(remarkParse).freeze()
+export const mdxProcessor = mdProcessor().use(remarkMdx).freeze()
 
 export const AST_PROPS = ['body', 'comments', 'tokens'] as const
 export const ES_NODE_TYPES: readonly string[] = ['export', 'import', 'jsx']
@@ -161,7 +162,7 @@ export class Parser {
       return this._eslintParse(code, options)
     }
 
-    const root = mdxProcessor.parse(code) as Parent
+    const root = (isMdx ? mdxProcessor : mdProcessor).parse(code) as Parent
 
     this._ast = {
       ...normalizePosition(root.position),
diff --git a/packages/eslint-plugin-mdx/src/rules/helper.ts b/packages/eslint-plugin-mdx/src/rules/helper.ts
index aed892bc..5daf3623 100644
--- a/packages/eslint-plugin-mdx/src/rules/helper.ts
+++ b/packages/eslint-plugin-mdx/src/rules/helper.ts
@@ -41,7 +41,7 @@ export const requirePkg = <T>(
 let searchSync: (searchFrom?: string) => CosmiconfigResult
 let remarkProcessor: Processor
 
-export const getRemarkProcessor = (searchFrom: string) => {
+export const getRemarkProcessor = (searchFrom: string, isMdx: boolean) => {
   if (!searchSync) {
     searchSync = cosmiconfigSync('remark', {
       packageProp: 'remarkConfig',
@@ -66,6 +66,12 @@ export const getRemarkProcessor = (searchFrom: string) => {
     // just ignore if the package does not exist
   }
 
+  const initProcessor = remarkProcessor().use({ settings }).use(remarkStringify)
+
+  if (isMdx) {
+    initProcessor.use(remarkMdx)
+  }
+
   return plugins
     .reduce((processor, pluginWithSettings) => {
       const [plugin, ...pluginSettings] = Array.isArray(pluginWithSettings)
@@ -78,6 +84,6 @@ export const getRemarkProcessor = (searchFrom: string) => {
           : plugin,
         ...pluginSettings,
       )
-    }, remarkProcessor().use({ settings }).use(remarkStringify).use(remarkMdx))
+    }, initProcessor)
     .freeze()
 }
diff --git a/packages/eslint-plugin-mdx/src/rules/remark.ts b/packages/eslint-plugin-mdx/src/rules/remark.ts
index ddc9a105..583c249e 100644
--- a/packages/eslint-plugin-mdx/src/rules/remark.ts
+++ b/packages/eslint-plugin-mdx/src/rules/remark.ts
@@ -23,21 +23,21 @@ export const remark: Rule.RuleModule = {
     const filename = context.getFilename()
     const extname = path.extname(filename)
     const sourceCode = context.getSourceCode()
-    const extensions = new Set(
-      DEFAULT_EXTENSIONS.concat(
-        context.parserOptions.extensions || [],
-        MARKDOWN_EXTENSIONS,
-        context.parserOptions.markdownExtensions || [],
-      ),
+    const options = context.parserOptions
+    const isMdx = DEFAULT_EXTENSIONS.concat(options.extensions || []).includes(
+      extname,
     )
+    const isMarkdown = MARKDOWN_EXTENSIONS.concat(
+      options.markdownExtensions || [],
+    ).includes(extname)
     return {
       Program(node) {
         /* istanbul ignore if */
-        if (!extensions.has(extname)) {
+        if (!isMdx && !isMarkdown) {
           return
         }
         const sourceText = sourceCode.getText(node)
-        const remarkProcessor = getRemarkProcessor(filename)
+        const remarkProcessor = getRemarkProcessor(filename, isMdx)
         const file = vfile({
           path: filename,
           contents: sourceText,