From 9eca52f43bdba74d4ef4ebffc6d1e9fe1a665b66 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 20 Jul 2021 16:05:19 +0300 Subject: [PATCH 1/6] fix(github): Better massageMarkdown implementation --- .../github/__snapshots__/index.spec.ts.snap | 5 +- lib/platform/github/index.ts | 7 +-- .../github/massage-markdown-links.spec.ts | 35 ++++++++++++++ lib/platform/github/massage-markdown-links.ts | 48 +++++++++++++++++++ package.json | 3 ++ yarn.lock | 23 +++++++++ 6 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 lib/platform/github/massage-markdown-links.spec.ts create mode 100644 lib/platform/github/massage-markdown-links.ts diff --git a/lib/platform/github/__snapshots__/index.spec.ts.snap b/lib/platform/github/__snapshots__/index.spec.ts.snap index eaec3c89ba6dd4..3ac903a828457b 100644 --- a/lib/platform/github/__snapshots__/index.spec.ts.snap +++ b/lib/platform/github/__snapshots__/index.spec.ts.snap @@ -7665,7 +7665,10 @@ Array [ ] `; -exports[`platform/github/index massageMarkdown(input) returns updated pr body 1`] = `"https://github.com/foo/bar/issues/5 plus also [a link](https://togithub.com/foo/bar/issues/5)"`; +exports[`platform/github/index massageMarkdown(input) returns updated pr body 1`] = ` +"[https://github.com/foo/bar/issues/5](https://togithub.com/foo/bar/issues/5) plus also [a link](https://togithub.com/foo/bar/issues/5) +" +`; exports[`platform/github/index mergePr(prNo) - autodetection should give up 1`] = ` Array [ diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts index 2f16a60a44a4d3..1cfc8ef9fb4821 100644 --- a/lib/platform/github/index.ts +++ b/lib/platform/github/index.ts @@ -60,6 +60,7 @@ import { PrList, } from './types'; import { UserDetails, getUserDetails, getUserEmail } from './user'; +import { massageMarkdownLinks } from './massage-markdown-links'; const githubApi = new githubHttp.GithubHttp(); @@ -1572,11 +1573,7 @@ export function massageMarkdown(input: string): string { if (config.isGhe) { return smartTruncate(input, 60000); } - const massagedInput = input - // to be safe, replace all github.com links with renovatebot redirector - .replace(/href="https?:\/\/github.com\//g, 'href="https://togithub.com/') - .replace(/]\(https:\/\/github\.com\//g, '](https://togithub.com/') - .replace(/]: https:\/\/github\.com\//g, ']: https://togithub.com/'); + const massagedInput = massageMarkdownLinks(input); return smartTruncate(massagedInput, 60000); } diff --git a/lib/platform/github/massage-markdown-links.spec.ts b/lib/platform/github/massage-markdown-links.spec.ts new file mode 100644 index 00000000000000..9908a7615bd6ea --- /dev/null +++ b/lib/platform/github/massage-markdown-links.spec.ts @@ -0,0 +1,35 @@ +import { getName } from '../../../test/util'; +import { massageMarkdownLinks } from './massage-markdown-links'; + +describe(getName(), () => { + const table = [ + ['github.com', '[github.com](togithub.com)'], + ['www.github.com', '[www.github.com](www.togithub.com)'], + ['api.github.com', 'api.github.com'], + ['togithub.com', 'togithub.com'], + ['foobartogithub.com', 'foobartogithub.com'], + ['[github.com](github.com)', '[github.com](togithub.com)'], + ['[github.com](www.github.com)', '[github.com](www.togithub.com)'], + [ + '[github.com](https://github.com/foo/bar)', + '[github.com](https://togithub.com/foo/bar)', + ], + [ + 'https://github.com/foo/bar', + '[https://github.com/foo/bar](https://togithub.com/foo/bar)', + ], + [ + '[foobar](https://github.com/foo/bar)', + '[foobar](https://togithub.com/foo/bar)', + ], + [ + '[https://github.com/foo/bar](https://github.com/foo/bar)', + '[https://github.com/foo/bar](https://togithub.com/foo/bar)', + ], + ]; + + test.each(table)('%s -> %s', (input, output) => { + const res = massageMarkdownLinks(input).trimRight(); + expect(res).toEqual(output); + }); +}); diff --git a/lib/platform/github/massage-markdown-links.ts b/lib/platform/github/massage-markdown-links.ts new file mode 100644 index 00000000000000..0fec2f97e9e94b --- /dev/null +++ b/lib/platform/github/massage-markdown-links.ts @@ -0,0 +1,48 @@ +import findAndReplace from 'mdast-util-find-and-replace'; +import remark from 'remark'; +import type { Plugin, Transformer } from 'unified'; +// eslint-disable-next-line import/no-unresolved +import type { Node } from 'unist'; +import visit from 'unist-util-visit'; + +const urlRegex = + /(?:https?:)?(?:\/\/)?(?:www\.)?(? { + if (node?.url) { + // eslint-disable-next-line no-param-reassign + node.url = massageLink(node.url); + } + }); +} + +const githubExtra: Plugin = (): Transformer => transformer; + +export function massageMarkdownLinks(content: string): string { + const output = remark().use(githubExtra).processSync(content); + return output.toString(); +} diff --git a/package.json b/package.json index 9f886304c2c50f..33c0efc0a2f308 100644 --- a/package.json +++ b/package.json @@ -167,6 +167,8 @@ "luxon": "2.0.1", "markdown-it": "12.1.0", "markdown-table": "2.0.0", + "mdast-util-find-and-replace": "1.0.0", + "mdast-util-to-string": "1.0.0", "minimatch": "3.0.4", "moo": "0.5.1", "node-html-parser": "3.3.6", @@ -187,6 +189,7 @@ "simple-git": "2.41.2", "slugify": "1.6.0", "traverse": "0.6.6", + "unist-util-visit": "2.0.0", "upath": "2.0.1", "url-join": "4.0.1", "validate-npm-package-name": "3.0.0", diff --git a/yarn.lock b/yarn.lock index 870ea300cbd589..91d1dc1e8abd0a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6501,6 +6501,15 @@ matcher@^3.0.0: dependencies: escape-string-regexp "^4.0.0" +mdast-util-find-and-replace@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/mdast-util-find-and-replace/-/mdast-util-find-and-replace-1.0.0.tgz#ce33ca0e720ce6ba16cf49dea441cd5f277270a1" + integrity sha512-3ArHEP8jGEtgcAyM1kUDhsuZwEVfabBjAQz9ohRWKHkheaBFdOElA0uzHKYVpuh6flnDTkg7cqWBs0pHSfPLqQ== + dependencies: + escape-string-regexp "^4.0.0" + unist-util-is "^4.0.0" + unist-util-visit-parents "^3.0.0" + mdast-util-find-and-replace@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/mdast-util-find-and-replace/-/mdast-util-find-and-replace-1.1.1.tgz#b7db1e873f96f66588c321f1363069abf607d1b5" @@ -6533,6 +6542,11 @@ mdast-util-to-markdown@^0.6.0: repeat-string "^1.0.0" zwitch "^1.0.0" +mdast-util-to-string@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-1.0.0.tgz#4f74267b39353df5f49727cd9c3d108b1196845f" + integrity sha1-T3Qmezk1PfX0lyfNnD0QixGWhF8= + mdast-util-to-string@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-1.1.0.tgz#27055500103f51637bd07d01da01eb1967a43527" @@ -9449,6 +9463,15 @@ unist-util-visit-parents@^3.0.0: "@types/unist" "^2.0.0" unist-util-is "^4.0.0" +unist-util-visit@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-2.0.0.tgz#1fdae5ea88251651bfe49b7e84390d664fc227c5" + integrity sha512-kiTpWKsF54u/78L/UU/i7lxrnqGiEWBgqCpaIZBYP0gwUC+Akq0Ajm4U8JiNIoQNfAioBdsyarnOcTEAb9mLeQ== + dependencies: + "@types/unist" "^2.0.0" + unist-util-is "^4.0.0" + unist-util-visit-parents "^3.0.0" + unist-util-visit@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-2.0.3.tgz#c3703893146df47203bb8a9795af47d7b971208c" From 57d1dddedf3271af2718f334ae0f1981e0ee3071 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 6 Aug 2021 14:06:33 +0300 Subject: [PATCH 2/6] Fixes --- lib/platform/github/index.ts | 2 +- lib/platform/github/massage-markdown-links.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts index 1cfc8ef9fb4821..3a5c621ef8de75 100644 --- a/lib/platform/github/index.ts +++ b/lib/platform/github/index.ts @@ -48,6 +48,7 @@ import { repoInfoQuery, vulnerabilityAlertsQuery, } from './graphql'; +import { massageMarkdownLinks } from './massage-markdown-links'; import { BranchProtection, CombinedBranchStatus, @@ -60,7 +61,6 @@ import { PrList, } from './types'; import { UserDetails, getUserDetails, getUserEmail } from './user'; -import { massageMarkdownLinks } from './massage-markdown-links'; const githubApi = new githubHttp.GithubHttp(); diff --git a/lib/platform/github/massage-markdown-links.ts b/lib/platform/github/massage-markdown-links.ts index 0fec2f97e9e94b..93426e670e7d52 100644 --- a/lib/platform/github/massage-markdown-links.ts +++ b/lib/platform/github/massage-markdown-links.ts @@ -11,7 +11,8 @@ const urlRegex = function massageLink(input: string): string { return urlRegex.test(input) ? input.replace(/(?:to)?github\.com/, 'togithub.com') - : input; + : /* istanbul ignore next */ + input; } function linkifyText(url: string): Node | boolean { From 4a1f3e2df766cbe057c2f0b6709a04ae3be64f7c Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 6 Aug 2021 14:18:48 +0300 Subject: [PATCH 3/6] Preserve old replacements just in case --- lib/platform/github/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts index 3a5c621ef8de75..469e7b5493b64f 100644 --- a/lib/platform/github/index.ts +++ b/lib/platform/github/index.ts @@ -1573,7 +1573,11 @@ export function massageMarkdown(input: string): string { if (config.isGhe) { return smartTruncate(input, 60000); } - const massagedInput = massageMarkdownLinks(input); + const massagedInput = massageMarkdownLinks(input) + // to be safe, replace all github.com links with renovatebot redirector + .replace(/href="https?:\/\/github.com\//g, 'href="https://togithub.com/') + .replace(/]\(https:\/\/github\.com\//g, '](https://togithub.com/') + .replace(/]: https:\/\/github\.com\//g, ']: https://togithub.com/'); return smartTruncate(massagedInput, 60000); } From a9e6bcad95eb95dc66563639b45db7f21209507e Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 6 Aug 2021 14:20:02 +0300 Subject: [PATCH 4/6] Wrap with try-catch --- lib/platform/github/massage-markdown-links.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/platform/github/massage-markdown-links.ts b/lib/platform/github/massage-markdown-links.ts index 93426e670e7d52..73c69b57521173 100644 --- a/lib/platform/github/massage-markdown-links.ts +++ b/lib/platform/github/massage-markdown-links.ts @@ -44,6 +44,10 @@ function transformer(tree: Node): void { const githubExtra: Plugin = (): Transformer => transformer; export function massageMarkdownLinks(content: string): string { - const output = remark().use(githubExtra).processSync(content); - return output.toString(); + try { + const output = remark().use(githubExtra).processSync(content); + return output.toString(); + } catch (_e) /* istanbul ignore next */ { + return content; + } } From 8aca427e42fbc5d4f0b1d88cdf66eb8751c4352b Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 6 Aug 2021 14:21:00 +0300 Subject: [PATCH 5/6] Logger --- lib/platform/github/massage-markdown-links.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/platform/github/massage-markdown-links.ts b/lib/platform/github/massage-markdown-links.ts index 73c69b57521173..ccec71852d1d1f 100644 --- a/lib/platform/github/massage-markdown-links.ts +++ b/lib/platform/github/massage-markdown-links.ts @@ -4,6 +4,7 @@ import type { Plugin, Transformer } from 'unified'; // eslint-disable-next-line import/no-unresolved import type { Node } from 'unist'; import visit from 'unist-util-visit'; +import { logger } from '../../logger'; const urlRegex = /(?:https?:)?(?:\/\/)?(?:www\.)?(? Date: Fri, 6 Aug 2021 17:40:10 +0300 Subject: [PATCH 6/6] Bump dep versions --- package.json | 6 +++--- yarn.lock | 35 ++++++----------------------------- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/package.json b/package.json index 33c0efc0a2f308..7384f020b89422 100644 --- a/package.json +++ b/package.json @@ -167,8 +167,8 @@ "luxon": "2.0.1", "markdown-it": "12.1.0", "markdown-table": "2.0.0", - "mdast-util-find-and-replace": "1.0.0", - "mdast-util-to-string": "1.0.0", + "mdast-util-find-and-replace": "1.1.1", + "mdast-util-to-string": "2.0.0", "minimatch": "3.0.4", "moo": "0.5.1", "node-html-parser": "3.3.6", @@ -189,7 +189,7 @@ "simple-git": "2.41.2", "slugify": "1.6.0", "traverse": "0.6.6", - "unist-util-visit": "2.0.0", + "unist-util-visit": "2.0.3", "upath": "2.0.1", "url-join": "4.0.1", "validate-npm-package-name": "3.0.0", diff --git a/yarn.lock b/yarn.lock index 91d1dc1e8abd0a..16a7f053a96264 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6501,16 +6501,7 @@ matcher@^3.0.0: dependencies: escape-string-regexp "^4.0.0" -mdast-util-find-and-replace@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/mdast-util-find-and-replace/-/mdast-util-find-and-replace-1.0.0.tgz#ce33ca0e720ce6ba16cf49dea441cd5f277270a1" - integrity sha512-3ArHEP8jGEtgcAyM1kUDhsuZwEVfabBjAQz9ohRWKHkheaBFdOElA0uzHKYVpuh6flnDTkg7cqWBs0pHSfPLqQ== - dependencies: - escape-string-regexp "^4.0.0" - unist-util-is "^4.0.0" - unist-util-visit-parents "^3.0.0" - -mdast-util-find-and-replace@^1.0.0: +mdast-util-find-and-replace@1.1.1, mdast-util-find-and-replace@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/mdast-util-find-and-replace/-/mdast-util-find-and-replace-1.1.1.tgz#b7db1e873f96f66588c321f1363069abf607d1b5" integrity sha512-9cKl33Y21lyckGzpSmEQnIDjEfeeWelN5s1kUW1LwdB0Fkuq2u+4GdqcGEygYxJE8GVqCl0741bYXHgamfWAZA== @@ -6542,21 +6533,16 @@ mdast-util-to-markdown@^0.6.0: repeat-string "^1.0.0" zwitch "^1.0.0" -mdast-util-to-string@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-1.0.0.tgz#4f74267b39353df5f49727cd9c3d108b1196845f" - integrity sha1-T3Qmezk1PfX0lyfNnD0QixGWhF8= +mdast-util-to-string@2.0.0, mdast-util-to-string@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-2.0.0.tgz#b8cfe6a713e1091cb5b728fc48885a4767f8b97b" + integrity sha512-AW4DRS3QbBayY/jJmD8437V1Gombjf8RSOUCMFBuo5iHi58AGEgVCKQ+ezHkZZDpAQS75hcBMpLqjpJTjtUL7w== mdast-util-to-string@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-1.1.0.tgz#27055500103f51637bd07d01da01eb1967a43527" integrity sha512-jVU0Nr2B9X3MU4tSK7JP1CMkSvOj7X5l/GboG1tKRw52lLF1x2Ju92Ms9tNetCcbfX3hzlM73zYo2NKkWSfF/A== -mdast-util-to-string@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-2.0.0.tgz#b8cfe6a713e1091cb5b728fc48885a4767f8b97b" - integrity sha512-AW4DRS3QbBayY/jJmD8437V1Gombjf8RSOUCMFBuo5iHi58AGEgVCKQ+ezHkZZDpAQS75hcBMpLqjpJTjtUL7w== - mdurl@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e" @@ -9463,16 +9449,7 @@ unist-util-visit-parents@^3.0.0: "@types/unist" "^2.0.0" unist-util-is "^4.0.0" -unist-util-visit@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-2.0.0.tgz#1fdae5ea88251651bfe49b7e84390d664fc227c5" - integrity sha512-kiTpWKsF54u/78L/UU/i7lxrnqGiEWBgqCpaIZBYP0gwUC+Akq0Ajm4U8JiNIoQNfAioBdsyarnOcTEAb9mLeQ== - dependencies: - "@types/unist" "^2.0.0" - unist-util-is "^4.0.0" - unist-util-visit-parents "^3.0.0" - -unist-util-visit@^2.0.0: +unist-util-visit@2.0.3, unist-util-visit@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-2.0.3.tgz#c3703893146df47203bb8a9795af47d7b971208c" integrity sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==