Markdownz: replace remark with rehype

Replace `remark` with `markdownz`, to parse Zooniverse markdown to HTML, and `rehype`, to render the parsed HTML as a React component tree. Bump `@zooniverse/react-components` to 1.7.0. Our `markdown-it` plugins rely on Node modules. I've added fallbacks to the dev classifier and project Webpack builds, where they require them.
zooniverse · Sep 18, 2023 · 89905b6 · 89905b6
1 parent 0d9b932
commit 89905b6
Show file tree

Hide file tree

Showing 14 changed files with 320 additions and 914 deletions.
diff --git a/package.json b/package.json
@@ -15,9 +15,6 @@
     "nyc": "~15.1.0",
     "snazzy": "~9.0.0"
   },
-  "resolutions": {
-    "**/trim": "~1.0"
-  },
   "scripts": {
     "bootstrap": "./bin/bootstrap.sh",
     "bootstrap:es6": "./bin/bootstrap:es6.sh",

diff --git a/packages/app-content-pages/package.json b/packages/app-content-pages/package.json
@@ -20,7 +20,7 @@
     "@zooniverse/async-states": "~0.0.1",
     "@zooniverse/grommet-theme": "~3.1.0",
     "@zooniverse/panoptes-js": "~0.4.0",
-    "@zooniverse/react-components": "~1.6.0",
+    "@zooniverse/react-components": "~1.7.0",
     "contentful": "~10.5.0",
     "dotenv": "~16.3.0",
     "dotenv-webpack": "~8.0.0",

diff --git a/packages/app-project/next.config.js b/packages/app-project/next.config.js
@@ -104,6 +104,10 @@ const nextConfig = {
       alias: {
         ...config.resolve.alias,
         ...webpackConfig.resolve.alias
+      },
+      fallback: {
+        ...config.resolve.fallback,
+        ...webpackConfig.resolve.fallback
       }
     }
     return config

diff --git a/packages/app-project/package.json b/packages/app-project/package.json
@@ -29,7 +29,7 @@
     "@zooniverse/classifier": "^0.0.1",
     "@zooniverse/grommet-theme": "~3.1.0",
     "@zooniverse/panoptes-js": "~0.4.0",
-    "@zooniverse/react-components": "~1.6.0",
+    "@zooniverse/react-components": "~1.7.0",
     "cookie": "~0.5.0",
     "d3": "~6.7.0",
     "engine.io-client": "~6.5.0",

diff --git a/packages/app-project/webpack.config.js b/packages/app-project/webpack.config.js
@@ -11,6 +11,9 @@ module.exports = {
       '@stores': path.resolve(__dirname, 'stores'),
       '@test': path.resolve(__dirname, 'test'),
       'sentry.edge.config.js': path.resolve(__dirname, 'sentry.edge.config.js')
+    },
+    fallback: {
+      fs: false
     }
   }
 }
diff --git a/packages/lib-classifier/package.json b/packages/lib-classifier/package.json
@@ -79,7 +79,7 @@
     "@wojtekmaj/enzyme-adapter-react-17": "~0.8.0",
     "@zooniverse/grommet-theme": "~3.1.0",
     "@zooniverse/panoptes-js": "~0.4.0",
-    "@zooniverse/react-components": "~1.6.0",
+    "@zooniverse/react-components": "~1.7.0",
     "@zooniverse/standard": "~2.0.0",
     "babel-loader": "~9.1.0",
     "babel-plugin-module-resolver": "~5.0.0",
@@ -104,6 +104,7 @@
     "mst-middlewares": "~5.1.0",
     "nock": "~13.3.0",
     "panoptes-client": "~5.5.1",
+    "path-browserify": "~1.0.1",
     "polished": "~4.2.2",
     "process": "~0.11.10",
     "prop-types": "^15.8.1",
@@ -118,6 +119,8 @@
     "style-loader": "~3.3.1",
     "styled-components": "~5.3.3",
     "superagent": "~8.1.0",
+    "url": "~0.11.2",
+    "util": "~0.12.5",
     "webpack": "~5.88.0",
     "webpack-cli": "~5.1.0",
     "webpack-dev-server": "~4.15.0"

diff --git a/packages/lib-classifier/webpack.dev.js b/packages/lib-classifier/webpack.dev.js
@@ -53,7 +53,12 @@ module.exports = {
       '@viewers': path.resolve(__dirname, 'src/components/Classifier/components/SubjectViewer')
     },
     fallback: {
-      url: false,
+      fs: false,
+      // for markdown-it plugins
+      path: require.resolve("path-browserify"),
+      util: require.resolve("util"),
+      url: require.resolve("url"),
+      process: false,
     }
   },
   module: {

diff --git a/packages/lib-react-components/CHANGELOG.md b/packages/lib-react-components/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.7.0] 2023-09-17
+
+### Changed
+- `Markdownz`: replace `remark` with `markdownz` and `rehype`.
+
 ## [1.6.0] 2023-09-12
 
 ### Changed

diff --git a/packages/lib-react-components/package.json b/packages/lib-react-components/package.json
@@ -3,7 +3,7 @@
   "description": "Zooniverse React Components",
   "license": "Apache-2.0",
   "author": "Zooniverse <contact@zooniverse.org> (https://www.zooniverse.org/)",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "main": "dist/cjs/index.js",
   "exports": {
     ".": {
@@ -44,19 +44,15 @@
     "cuid": "~3.0.0",
     "formik": "~2.4.0",
     "i18next": "~23.5.0",
+    "markdownz": "~8.1.1",
     "mime": "~3.0.0",
     "polished": "~4.2.2",
     "prop-types": "~15.8.1",
     "react-i18next": "~13.2.0",
     "react-resize-detector": "~9.1.0",
     "react-rnd": "10.4.1",
-    "remark": "~12.0.1",
-    "remark-emoji": "~2.0.2",
-    "remark-external-links": "~3.1.1",
-    "remark-footnotes": "2.0.0",
-    "remark-react": "~8.0.0",
-    "remark-sub-super": "~1.0.16",
-    "remark-toc": "~5.1.1",
+    "rehype": "~11.0.0",
+    "rehype-react": "~6.2.1",
     "unist-util-visit": "~1.4.0"
   },
   "peerDependencies": {

diff --git a/packages/lib-react-components/src/Markdownz/Markdownz.js b/packages/lib-react-components/src/Markdownz/Markdownz.js
@@ -1,4 +1,4 @@
-import { Fragment, useCallback } from 'react'
+import { createElement, Fragment, useCallback } from 'react'
 import PropTypes from 'prop-types'
 import {
   Anchor,
@@ -13,27 +13,17 @@ import {
   Text
 } from 'grommet'
 
-import remark from 'remark'
-import remark2react from 'remark-react'
-import emoji from 'remark-emoji'
-import remarkSubSuper from 'remark-sub-super'
-import externalLinks from 'remark-external-links'
-import toc from 'remark-toc'
-import ping from './lib/ping'
-import footnotes from 'remark-footnotes'
+import { utils } from 'markdownz'
+import rehype from 'rehype'
+import rehype2react from 'rehype-react'
 import Media from '../Media'
 import withThemeContext from '../helpers/withThemeContext'
 import theme from './theme'
 
-const HASHTAG = '#'
-const AT = '@'
-const SUBJECT_SYMBOL = '^S'
-
-export const RESTRICTED_USERNAMES = ['admins', 'moderators', 'researchers', 'scientists', 'team', 'support']
-
 // Support image resizing, video, and audio using markdown's image syntax
 export function renderMedia(nodeProps) {
-  let width, height
+  let width = nodeProps.width
+  let height = nodeProps.height
   const imgSizeRegex = /=(\d+(%|px|em|rem|vw)?)x(\d+(%|px|em|rem|vh)?)?/
   let alt = nodeProps.alt
   const src = nodeProps.src
@@ -71,33 +61,11 @@ const componentMappings = {
   ul: ({ children }) => <ul style={{ fontSize: '14px', marginTop: 0 }}>{children}</ul>
 }
 
-export function buildResourceURL(baseURI, projectSlug, resource, symbol) {
-  if (!resource) return ''
-  const baseURL = (projectSlug) ? `${baseURI}/projects/${projectSlug}` : baseURI
-
-  if (symbol === HASHTAG) return (projectSlug) ? `${baseURL}/talk/tag/${resource}` : `${baseURL}/talk/search?query=%23${resource}`
-  if (symbol === AT) return `${baseURL}/users/${resource}`
-  if (symbol === SUBJECT_SYMBOL && projectSlug) return `${baseURL}/talk/subjects/${resource}`
-
-  return ''
-}
-
-export function shouldResourceBeLinkable(restrictedUserNames, projectSlug, resource, symbol) {
-  if (symbol === AT) return !restrictedUserNames.includes(resource)
-  if (symbol === SUBJECT_SYMBOL) return !!projectSlug
-  return true
-}
-
-export function replaceImageString(img, altText, imageURL, imageSize) {
-  return `![${altText} ${imageSize}](${imageURL})`
-}
-
 function Markdownz({
   baseURI = '',
   children,
   components = {},
   projectSlug = '',
-  restrictedUserNames = RESTRICTED_USERNAMES,
   settings = {}
 }) {
   if (!children) return null
@@ -106,42 +74,32 @@ function Markdownz({
     console.warn('Overriding the rendering function for the img tag may break the syntax support for image resizing and using image markup for video and audio. Are you sure you want to do this?')
   }
 
-  const imageRegex = /!\[([^\]]+?)\]\((https:\/\/[^)]+?) (=\d+?(|%|px|em|rem|vw)x\d*?(|%|px|em|rem|vh))\)/g
-  const newChildren = children.replace(imageRegex, replaceImageString)
-
-  const remarkReactComponents = { ...componentMappings, ...components }
-  const remarkSettings = { ...settings }
-
-  const pingCallback = useCallback(
-    (resource, symbol) => shouldResourceBeLinkable(restrictedUserNames, projectSlug, resource, symbol),
-    [restrictedUserNames, projectSlug]
-  )
-  const resourceURL = useCallback(
-    (resource, symbol) => buildResourceURL(baseURI, projectSlug, resource, symbol),
-    [baseURI, projectSlug]
-  )
-  let markdown = null
+  const rehypeReactComponents = { ...componentMappings, ...components }
+  const rehypeSettings = { ...settings }
+
+  const html = utils.getHtml({
+    baseURI,
+    content: children,
+    project: {
+      slug: projectSlug
+    }
+  })
+  let parsedHTML = null
   try {
-    markdown = remark()
-      .data('settings', remarkSettings)
-      .use(emoji)
-      .use(remarkSubSuper)
-      .use(externalLinks)
-      .use(footnotes, { inlineNotes: true })
-      .use(ping, {
-        ping: pingCallback,
-        pingSymbols: [AT, HASHTAG, SUBJECT_SYMBOL],
-        resourceURL
+    parsedHTML = rehype()
+      .data('settings', rehypeSettings)
+      .use(rehype2react, {
+        Fragment,
+        createElement,
+        components: rehypeReactComponents
       })
-      .use(toc)
-      .use(remark2react, { remarkReactComponents })
-      .processSync(newChildren).result
+      .processSync(html).result
   } catch (error) {
-    markdown = error.message
+    parsedHTML = error.message
   }
   return (
     <Fragment>
-      {markdown}
+      {parsedHTML}
     </Fragment>
   );
 }
@@ -151,7 +109,6 @@ Markdownz.propTypes = {
   children: PropTypes.string.isRequired,
   components: PropTypes.object,
   projectSlug: PropTypes.string,
-  restrictedUserNames: PropTypes.arrayOf(PropTypes.string),
   settings: PropTypes.object
 }