From 2cc033f77b03b64f9bb5aed60bb292aefb292b64 Mon Sep 17 00:00:00 2001 From: Alexey Pyltsyn Date: Thu, 30 Apr 2020 15:38:23 +0300 Subject: [PATCH 1/4] fix(v2): remove Markdown syntax from excerpt --- packages/docusaurus-utils/package.json | 4 +- .../src/__tests__/index.test.ts | 57 +++++++++++++++++++ packages/docusaurus-utils/src/index.ts | 38 ++++++++++--- yarn.lock | 10 ++++ 4 files changed, 99 insertions(+), 10 deletions(-) diff --git a/packages/docusaurus-utils/package.json b/packages/docusaurus-utils/package.json index 064af699524d..8d749ec0cc1f 100644 --- a/packages/docusaurus-utils/package.json +++ b/packages/docusaurus-utils/package.json @@ -12,11 +12,13 @@ }, "license": "MIT", "dependencies": { + "@types/remove-markdown": "^0.1.1", "escape-string-regexp": "^2.0.0", "fs-extra": "^8.1.0", "gray-matter": "^4.0.2", "lodash.camelcase": "^4.3.0", - "lodash.kebabcase": "^4.1.1" + "lodash.kebabcase": "^4.1.1", + "remove-markdown": "^0.3.0" }, "engines": { "node": ">=10.9.0" diff --git a/packages/docusaurus-utils/src/__tests__/index.test.ts b/packages/docusaurus-utils/src/__tests__/index.test.ts index 5e22171c9126..d0e7bde431c9 100644 --- a/packages/docusaurus-utils/src/__tests__/index.test.ts +++ b/packages/docusaurus-utils/src/__tests__/index.test.ts @@ -17,6 +17,7 @@ import { posixPath, objectWithKeySorted, aliasedSitePath, + createExcerpt, } from '../index'; describe('load utils', () => { @@ -292,4 +293,60 @@ describe('load utils', () => { `"Url must be a string. Received undefined"`, ); }); + + test('createExcerpt', () => { + const asserts = [ + // Regular content + { + input: ` + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum ex urna, molestie et sagittis ut, varius ac justo. + + Nunc porttitor libero nec vulputate venenatis. Nam nec rhoncus mauris. Morbi tempus est et nibh maximus, tempus venenatis arcu lobortis. + `, + output: + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum ex urna, molestie et sagittis ut, varius ac justo.', + }, + // Content with imports declarations and Markdown markup, as well as Emoji + { + input: ` + import Component from '@site/src/components/Component'; + import Component from '@site/src/components/Component' + + Lorem **ipsum** dolor sit \`amet\`, consectetur _adipiscing_ elit. [**Vestibulum**](#) ex urna, molestie et sagittis ut, varius ac justo :wink:. + + Nunc porttitor libero nec vulputate venenatis. Nam nec rhoncus mauris. Morbi tempus est et nibh maximus, tempus venenatis arcu lobortis. + `, + output: + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum ex urna, molestie et sagittis ut, varius ac justo.', + }, + // Content beginning with admonitions + { + input: ` + import Component from '@site/src/components/Component' + + :::caution + + Lorem ipsum dolor sit amet, consectetur adipiscing elit. + + ::: + + Nunc porttitor libero nec vulputate venenatis. Nam nec rhoncus mauris. Morbi tempus est et nibh maximus, tempus venenatis arcu lobortis. + `, + output: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.', + }, + // Content beginning with heading + { + input: ` + # Lorem ipsum dolor sit amet + + Nunc porttitor libero nec vulputate venenatis. Nam nec rhoncus mauris. Morbi tempus est et nibh maximus, tempus venenatis arcu lobortis. + `, + output: 'Lorem ipsum dolor sit amet', + }, + ]; + + asserts.forEach((testCase) => { + expect(createExcerpt(testCase.input)).toEqual(testCase.output); + }); + }); }); diff --git a/packages/docusaurus-utils/src/index.ts b/packages/docusaurus-utils/src/index.ts index fb40e69a802f..ef35e64b2ff3 100644 --- a/packages/docusaurus-utils/src/index.ts +++ b/packages/docusaurus-utils/src/index.ts @@ -12,6 +12,7 @@ import camelCase from 'lodash.camelcase'; import kebabCase from 'lodash.kebabcase'; import escapeStringRegexp from 'escape-string-regexp'; import fs from 'fs-extra'; +import removeMd from 'remove-markdown'; const fileHash = new Map(); export async function generate( @@ -185,6 +186,33 @@ export function getSubFolder(file: string, refDir: string): string | null { // Regex for an import statement. const importRegexString = '^(.*import){1}(.+){0,1}\\s[\'"](.+)[\'"];?'; +export function createExcerpt(fileString: string): string | undefined { + let fileContent = fileString.trimLeft(); + + if (RegExp(importRegexString).test(fileContent)) { + fileContent = fileContent + .replace(RegExp(importRegexString, 'gm'), '') + .trimLeft(); + } + + const fileLines = fileContent.split('\n'); + + for (let fileLine of fileLines) { + const cleanedLine = removeMd(fileLine) + // Remove definition of admonition. + .replace(/(:{3}.*)/, '') + // Remove Emoji names within colons include preceding whitespace. + .replace(/\s?(:(::|[^:\n])+:)/g, '') + .trim(); + + if (cleanedLine) { + return cleanedLine; + } + } + + return undefined; +} + export function parse( fileString: string, ): { @@ -196,18 +224,10 @@ export function parse( } { const options: {} = { excerpt: (file: matter.GrayMatterFile): void => { - let fileContent = file.content.trimLeft(); - // Hacky way of stripping out import statements from the excerpt // TODO: Find a better way to do so, possibly by compiling the Markdown content, // stripping out HTML tags and obtaining the first line. - if (RegExp(importRegexString).test(fileContent)) { - fileContent = fileContent - .replace(RegExp(importRegexString, 'gm'), '') - .trimLeft(); - } - - file.excerpt = fileContent.split('\n', 1).shift(); + file.excerpt = createExcerpt(file.content); }, }; diff --git a/yarn.lock b/yarn.lock index f4f4ce60c24e..3536853f968e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2725,6 +2725,11 @@ resolved "https://registry.yarnpkg.com/@types/relateurl/-/relateurl-0.2.28.tgz#6bda7db8653fa62643f5ee69e9f69c11a392e3a6" integrity sha1-a9p9uGU/piZD9e5p6facEaOS46Y= +"@types/remove-markdown@^0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@types/remove-markdown/-/remove-markdown-0.1.1.tgz#c79d3000df412526186b2af3808b85bee68bc907" + integrity sha512-SCYOFMHUqyiJU5M0V2gBB6UDdBhPwma34j0vYX0JgWaqp/74ila2Ops1jt5tB/C1UQXVXqK+is61884bITn3LQ== + "@types/semver@^7.1.0": version "7.1.0" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.1.0.tgz#c8c630d4c18cd326beff77404887596f96408408" @@ -14559,6 +14564,11 @@ remarkable@^2.0.0: argparse "^1.0.10" autolinker "^3.11.0" +remove-markdown@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/remove-markdown/-/remove-markdown-0.3.0.tgz#5e4b667493a93579728f3d52ecc1db9ca505dc98" + integrity sha1-XktmdJOpNXlyjz1S7MHbnKUF3Jg= + remove-trailing-separator@^1.0.1: version "1.1.0" resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" From 3ec437bb6427249f56843a82390470b9c775a300 Mon Sep 17 00:00:00 2001 From: Alexey Pyltsyn Date: Thu, 30 Apr 2020 15:55:47 +0300 Subject: [PATCH 2/4] Update snapshots --- .../src/__tests__/index.test.ts | 8 ++++---- .../src/__tests__/metadata.test.ts | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/index.test.ts b/packages/docusaurus-plugin-content-docs/src/__tests__/index.test.ts index d727bf6a29a5..a2387b55dda9 100644 --- a/packages/docusaurus-plugin-content-docs/src/__tests__/index.test.ts +++ b/packages/docusaurus-plugin-content-docs/src/__tests__/index.test.ts @@ -298,7 +298,7 @@ describe('versioned website', () => { permalink: '/docs/next/foo/bar', source: path.join('@site', routeBasePath, 'foo', 'bar.md'), title: 'bar', - description: 'This is `next` version of bar.', + description: 'This is next version of bar.', version: 'next', sidebar: 'docs', next: { @@ -311,7 +311,7 @@ describe('versioned website', () => { permalink: '/docs/next/hello', source: path.join('@site', routeBasePath, 'hello.md'), title: 'hello', - description: 'Hello `next` !', + description: 'Hello next !', version: 'next', sidebar: 'docs', previous: { @@ -329,7 +329,7 @@ describe('versioned website', () => { 'hello.md', ), title: 'hello', - description: 'Hello `1.0.1` !', + description: 'Hello 1.0.1 !', version: '1.0.1', sidebar: 'version-1.0.1/docs', previous: { @@ -349,7 +349,7 @@ describe('versioned website', () => { ), title: 'baz', description: - 'Baz `1.0.0` ! This will be deleted in next subsequent versions.', + 'Baz 1.0.0 ! This will be deleted in next subsequent versions.', version: '1.0.0', sidebar: 'version-1.0.0/docs', next: { diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/metadata.test.ts b/packages/docusaurus-plugin-content-docs/src/__tests__/metadata.test.ts index 06ebb93c7970..aeff67339a36 100644 --- a/packages/docusaurus-plugin-content-docs/src/__tests__/metadata.test.ts +++ b/packages/docusaurus-plugin-content-docs/src/__tests__/metadata.test.ts @@ -84,7 +84,7 @@ describe('simple site', () => { title: 'baz', editUrl: 'https://github.com/facebook/docusaurus/edit/master/website/docs/foo/baz.md', - description: '## Images', + description: 'Images', }); }); @@ -228,7 +228,7 @@ describe('versioned site', () => { permalink: '/docs/next/foo/bar', source: path.join('@site', routeBasePath, sourceA), title: 'bar', - description: 'This is `next` version of bar.', + description: 'This is next version of bar.', version: 'next', }); expect(dataB).toEqual({ @@ -236,7 +236,7 @@ describe('versioned site', () => { permalink: '/docs/next/hello', source: path.join('@site', routeBasePath, sourceB), title: 'hello', - description: 'Hello `next` !', + description: 'Hello next !', version: 'next', }); }); @@ -286,7 +286,7 @@ describe('versioned site', () => { permalink: '/docs/1.0.0/foo/bar', source: path.join('@site', path.relative(siteDir, versionedDir), sourceA), title: 'bar', - description: 'Bar `1.0.0` !', + description: 'Bar 1.0.0 !', version: '1.0.0', }); expect(dataB).toEqual({ @@ -294,7 +294,7 @@ describe('versioned site', () => { permalink: '/docs/1.0.0/hello', source: path.join('@site', path.relative(siteDir, versionedDir), sourceB), title: 'hello', - description: 'Hello `1.0.0` !', + description: 'Hello 1.0.0 !', version: '1.0.0', }); expect(dataC).toEqual({ @@ -302,7 +302,7 @@ describe('versioned site', () => { permalink: '/docs/foo/bar', source: path.join('@site', path.relative(siteDir, versionedDir), sourceC), title: 'bar', - description: 'Bar `1.0.1` !', + description: 'Bar 1.0.1 !', version: '1.0.1', }); expect(dataD).toEqual({ @@ -310,7 +310,7 @@ describe('versioned site', () => { permalink: '/docs/hello', source: path.join('@site', path.relative(siteDir, versionedDir), sourceD), title: 'hello', - description: 'Hello `1.0.1` !', + description: 'Hello 1.0.1 !', version: '1.0.1', }); }); From c0432dbdbf201ed2f18050f01cb9f050486731ee Mon Sep 17 00:00:00 2001 From: Alexey Pyltsyn Date: Thu, 30 Apr 2020 19:10:40 +0300 Subject: [PATCH 3/4] Use Remark --- packages/docusaurus-utils/package.json | 4 +++- packages/docusaurus-utils/src/index.ts | 8 +++++++- packages/docusaurus-utils/types.d.ts | 1 + yarn.lock | 5 +++++ 4 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 packages/docusaurus-utils/types.d.ts diff --git a/packages/docusaurus-utils/package.json b/packages/docusaurus-utils/package.json index 8d749ec0cc1f..5de945cceadb 100644 --- a/packages/docusaurus-utils/package.json +++ b/packages/docusaurus-utils/package.json @@ -18,7 +18,9 @@ "gray-matter": "^4.0.2", "lodash.camelcase": "^4.3.0", "lodash.kebabcase": "^4.1.1", - "remove-markdown": "^0.3.0" + "remark": "^12.0.0", + "remove-markdown": "^0.3.0", + "strip-markdown": "^3.1.2" }, "engines": { "node": ">=10.9.0" diff --git a/packages/docusaurus-utils/src/index.ts b/packages/docusaurus-utils/src/index.ts index ef35e64b2ff3..7d51e4bcd106 100644 --- a/packages/docusaurus-utils/src/index.ts +++ b/packages/docusaurus-utils/src/index.ts @@ -12,7 +12,9 @@ import camelCase from 'lodash.camelcase'; import kebabCase from 'lodash.kebabcase'; import escapeStringRegexp from 'escape-string-regexp'; import fs from 'fs-extra'; -import removeMd from 'remove-markdown'; +// import removeMd from 'remove-markdown'; +import remark from 'remark'; +import strip from 'strip-markdown'; const fileHash = new Map(); export async function generate( @@ -186,6 +188,10 @@ export function getSubFolder(file: string, refDir: string): string | null { // Regex for an import statement. const importRegexString = '^(.*import){1}(.+){0,1}\\s[\'"](.+)[\'"];?'; +function removeMd(markdownString: string): string { + return remark().use(strip).processSync(markdownString.trim()).toString(); +} + export function createExcerpt(fileString: string): string | undefined { let fileContent = fileString.trimLeft(); diff --git a/packages/docusaurus-utils/types.d.ts b/packages/docusaurus-utils/types.d.ts new file mode 100644 index 000000000000..3cf1cb2b1abb --- /dev/null +++ b/packages/docusaurus-utils/types.d.ts @@ -0,0 +1 @@ +declare module 'strip-markdown'; diff --git a/yarn.lock b/yarn.lock index 3536853f968e..f1a353a11a50 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15907,6 +15907,11 @@ strip-json-comments@~2.0.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= +strip-markdown@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/strip-markdown/-/strip-markdown-3.1.2.tgz#172f6f89f9a98896e65a65422e0507f2bbac1667" + integrity sha512-NjwW6CEefesmHQPs7lof/lgnSriqUnRNOWpnrNPq9A7/yOCdnhaB7DcxlhYuN7WiiRUe349aitAsTQ/ajM9Dmw== + strip-outer@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/strip-outer/-/strip-outer-1.0.1.tgz#b2fd2abf6604b9d1e6013057195df836b8a9d631" From 3ec542d7c7a91aeac6606d99faf5dba19dc40efd Mon Sep 17 00:00:00 2001 From: Alexey Pyltsyn Date: Sun, 17 May 2020 19:52:59 +0300 Subject: [PATCH 4/4] Switch to own solution --- packages/docusaurus-utils/package.json | 6 +---- .../src/__tests__/index.test.ts | 4 +-- packages/docusaurus-utils/src/index.ts | 27 ++++++++++++------- packages/docusaurus-utils/types.d.ts | 1 - yarn.lock | 15 ----------- 5 files changed, 21 insertions(+), 32 deletions(-) delete mode 100644 packages/docusaurus-utils/types.d.ts diff --git a/packages/docusaurus-utils/package.json b/packages/docusaurus-utils/package.json index 5de945cceadb..064af699524d 100644 --- a/packages/docusaurus-utils/package.json +++ b/packages/docusaurus-utils/package.json @@ -12,15 +12,11 @@ }, "license": "MIT", "dependencies": { - "@types/remove-markdown": "^0.1.1", "escape-string-regexp": "^2.0.0", "fs-extra": "^8.1.0", "gray-matter": "^4.0.2", "lodash.camelcase": "^4.3.0", - "lodash.kebabcase": "^4.1.1", - "remark": "^12.0.0", - "remove-markdown": "^0.3.0", - "strip-markdown": "^3.1.2" + "lodash.kebabcase": "^4.1.1" }, "engines": { "node": ">=10.9.0" diff --git a/packages/docusaurus-utils/src/__tests__/index.test.ts b/packages/docusaurus-utils/src/__tests__/index.test.ts index d0e7bde431c9..2e4f75fd329a 100644 --- a/packages/docusaurus-utils/src/__tests__/index.test.ts +++ b/packages/docusaurus-utils/src/__tests__/index.test.ts @@ -312,7 +312,7 @@ describe('load utils', () => { import Component from '@site/src/components/Component'; import Component from '@site/src/components/Component' - Lorem **ipsum** dolor sit \`amet\`, consectetur _adipiscing_ elit. [**Vestibulum**](#) ex urna, molestie et sagittis ut, varius ac justo :wink:. + Lorem **ipsum** dolor sit \`amet\`, consectetur _adipiscing_ elit. [**Vestibulum**](https://wiktionary.org/wiki/vestibulum) ex urna, ~molestie~ et sagittis ut, varius ac justo :wink:. Nunc porttitor libero nec vulputate venenatis. Nam nec rhoncus mauris. Morbi tempus est et nibh maximus, tempus venenatis arcu lobortis. `, @@ -337,7 +337,7 @@ describe('load utils', () => { // Content beginning with heading { input: ` - # Lorem ipsum dolor sit amet + ## Lorem ipsum dolor sit amet Nunc porttitor libero nec vulputate venenatis. Nam nec rhoncus mauris. Morbi tempus est et nibh maximus, tempus venenatis arcu lobortis. `, diff --git a/packages/docusaurus-utils/src/index.ts b/packages/docusaurus-utils/src/index.ts index 7d51e4bcd106..8e80e3fcfbd8 100644 --- a/packages/docusaurus-utils/src/index.ts +++ b/packages/docusaurus-utils/src/index.ts @@ -12,9 +12,6 @@ import camelCase from 'lodash.camelcase'; import kebabCase from 'lodash.kebabcase'; import escapeStringRegexp from 'escape-string-regexp'; import fs from 'fs-extra'; -// import removeMd from 'remove-markdown'; -import remark from 'remark'; -import strip from 'strip-markdown'; const fileHash = new Map(); export async function generate( @@ -188,10 +185,6 @@ export function getSubFolder(file: string, refDir: string): string | null { // Regex for an import statement. const importRegexString = '^(.*import){1}(.+){0,1}\\s[\'"](.+)[\'"];?'; -function removeMd(markdownString: string): string { - return remark().use(strip).processSync(markdownString.trim()).toString(); -} - export function createExcerpt(fileString: string): string | undefined { let fileContent = fileString.trimLeft(); @@ -204,8 +197,24 @@ export function createExcerpt(fileString: string): string | undefined { const fileLines = fileContent.split('\n'); for (let fileLine of fileLines) { - const cleanedLine = removeMd(fileLine) - // Remove definition of admonition. + const cleanedLine = fileLine + // Remove HTML tags. + .replace(/<[^>]*>/g, '') + // Remove ATX-style headers. + .replace(/^\#{1,6}\s*([^#]*)\s*(\#{1,6})?/gm, '$1') + // Remove emphasis and strikethroughs. + .replace(/([\*_~]{1,3})(\S.*?\S{0,1})\1/g, '$2') + // Remove inline links. + .replace(/\[(.*?)\][\[\(].*?[\]\)]/g, '$1') + // Remove inline code. + .replace(/`(.+?)`/g, '$1') + // Remove images. + .replace(/\!\[(.*?)\][\[\(].*?[\]\)]/g, '') + // Remove blockquotes. + .replace(/^\s{0,3}>\s?/g, '') + // Remove footnotes. + .replace(/\[\^.+?\](\: .*?$)?/g, '') + // Remove admonition definition. .replace(/(:{3}.*)/, '') // Remove Emoji names within colons include preceding whitespace. .replace(/\s?(:(::|[^:\n])+:)/g, '') diff --git a/packages/docusaurus-utils/types.d.ts b/packages/docusaurus-utils/types.d.ts deleted file mode 100644 index 3cf1cb2b1abb..000000000000 --- a/packages/docusaurus-utils/types.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare module 'strip-markdown'; diff --git a/yarn.lock b/yarn.lock index f1a353a11a50..f4f4ce60c24e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2725,11 +2725,6 @@ resolved "https://registry.yarnpkg.com/@types/relateurl/-/relateurl-0.2.28.tgz#6bda7db8653fa62643f5ee69e9f69c11a392e3a6" integrity sha1-a9p9uGU/piZD9e5p6facEaOS46Y= -"@types/remove-markdown@^0.1.1": - version "0.1.1" - resolved "https://registry.yarnpkg.com/@types/remove-markdown/-/remove-markdown-0.1.1.tgz#c79d3000df412526186b2af3808b85bee68bc907" - integrity sha512-SCYOFMHUqyiJU5M0V2gBB6UDdBhPwma34j0vYX0JgWaqp/74ila2Ops1jt5tB/C1UQXVXqK+is61884bITn3LQ== - "@types/semver@^7.1.0": version "7.1.0" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.1.0.tgz#c8c630d4c18cd326beff77404887596f96408408" @@ -14564,11 +14559,6 @@ remarkable@^2.0.0: argparse "^1.0.10" autolinker "^3.11.0" -remove-markdown@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/remove-markdown/-/remove-markdown-0.3.0.tgz#5e4b667493a93579728f3d52ecc1db9ca505dc98" - integrity sha1-XktmdJOpNXlyjz1S7MHbnKUF3Jg= - remove-trailing-separator@^1.0.1: version "1.1.0" resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" @@ -15907,11 +15897,6 @@ strip-json-comments@~2.0.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= -strip-markdown@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/strip-markdown/-/strip-markdown-3.1.2.tgz#172f6f89f9a98896e65a65422e0507f2bbac1667" - integrity sha512-NjwW6CEefesmHQPs7lof/lgnSriqUnRNOWpnrNPq9A7/yOCdnhaB7DcxlhYuN7WiiRUe349aitAsTQ/ajM9Dmw== - strip-outer@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/strip-outer/-/strip-outer-1.0.1.tgz#b2fd2abf6604b9d1e6013057195df836b8a9d631"