From 3c69bbe76ef4b1cf52bfd33ee8a5c44267f6dccb Mon Sep 17 00:00:00 2001 From: Mateusz Titz Date: Sat, 21 Dec 2024 15:16:08 +0100 Subject: [PATCH 1/3] Add handling possible short mentions tags to ExpensiMark --- .eslintrc.js | 3 +++ __tests__/ExpensiMark-HTML-test.js | 15 +++++++++++++-- lib/ExpensiMark.ts | 23 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index b3a9bd5a..124d2546 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,6 +1,9 @@ module.exports = { extends: ['expensify', 'prettier'], parser: '@typescript-eslint/parser', + env: { + jest: true, + }, overrides: [ { files: ['*.js', '*.jsx'], diff --git a/__tests__/ExpensiMark-HTML-test.js b/__tests__/ExpensiMark-HTML-test.js index 3eef3952..ceb64398 100644 --- a/__tests__/ExpensiMark-HTML-test.js +++ b/__tests__/ExpensiMark-HTML-test.js @@ -1,4 +1,4 @@ -/* eslint-disable max-len */ +/* eslint-disable max-len,no-useless-concat */ import ExpensiMark from '../lib/ExpensiMark'; const parser = new ExpensiMark(); @@ -931,6 +931,7 @@ test('Test urls autolinks correctly', () => { }, ]; + // Fixme @expensify.com is a now correct "possible" short mention testCases.forEach((testCase) => { expect(parser.replace(testCase.testString)).toBe(testCase.resultString); }); @@ -1324,6 +1325,16 @@ test('Test for user mention with @username@domain.com', () => { const resultString = '@username@expensify.com'; expect(parser.replace(testString)).toBe(resultString); }); +// Todo +// popraw psujący się @here testy +// Todo wszystkie edge kejsy w których short mention jest ambiguous +// short-mention -> possible-short-mention + +test('Test for short mention mention with @username@domain.com', () => { + const testString = '@john.doe'; + const resultString = '@john.doe'; + expect(parser.replace(testString)).toBe(resultString); +}); test('Test for user mention with @@username@domain.com', () => { const testString = '@@username@expensify.com'; @@ -1376,7 +1387,7 @@ test('Test for user mention without leading whitespace', () => { test('Test for user mention with @username@expensify', () => { const testString = '@username@expensify'; - const resultString = '@username@expensify'; + const resultString = '@username@expensify'; expect(parser.replace(testString)).toBe(resultString); }); diff --git a/lib/ExpensiMark.ts b/lib/ExpensiMark.ts index 61413e0d..3ce43635 100644 --- a/lib/ExpensiMark.ts +++ b/lib/ExpensiMark.ts @@ -417,6 +417,29 @@ export default class ExpensiMark { }, }, + { + name: 'shortMentions', + regex: new RegExp( + "(@here|[a-zA-Z0-9.!$%&+=?^\\`{|}-]?)(@(?=((?=[\\w]+[\\w'#%+-]+(?:\\.[\\w'#%+-]+)*)[\\w\\.'#%+-]{1,64}(?= |_|\\b))(?!([:\\/\\\\]))(?.*))\\S{3,254}(?=\\k$))(?!((?:(?!|[^<]*(<\\/pre>|<\\/code>|<\\/mention-user>|<\\/mention-here>))", + 'gim', + ), + replacement: (_extras, match, g1, g2) => { + if (!Str.isValidMention(match)) { + return match; + } + return `${g1}${g2}`; + }, + // rawInputReplacement: (_extras, match, g1, g2) => { + // const phoneNumberRegex = new RegExp(`^${Constants.CONST.REG_EXP.PHONE_PART}$`); + // const mention = g2.slice(1); + // const mentionWithoutSMSDomain = str_1.default.removeSMSDomain(mention); + // if (!str_1.default.isValidMention(match) || (phoneNumberRegex.test(mentionWithoutSMSDomain) && !str_1.default.isValidPhoneNumber(mentionWithoutSMSDomain))) { + // return match; + // } + // return `${g1}${g2}`; + // }, + }, + { name: 'quote', From bb4cc9746c0565e56942e6adc333aae7b182d0d8 Mon Sep 17 00:00:00 2001 From: Mateusz Titz Date: Mon, 30 Dec 2024 10:44:24 +0100 Subject: [PATCH 2/3] Cleanup and fix short-mentions tests --- __tests__/ExpensiMark-HTML-test.js | 10 +++---- __tests__/ExpensiMark-test.js | 1 - lib/ExpensiMark.ts | 47 +++++++++++++++--------------- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/__tests__/ExpensiMark-HTML-test.js b/__tests__/ExpensiMark-HTML-test.js index ceb64398..b18aecca 100644 --- a/__tests__/ExpensiMark-HTML-test.js +++ b/__tests__/ExpensiMark-HTML-test.js @@ -697,6 +697,7 @@ test('Test url replacements', () => { '@test.com test.com ' + '@test.com @test.com '; + // Fixme [short-mention] this errors on "test.com @test.com @test.com { }, ]; - // Fixme @expensify.com is a now correct "possible" short mention + // Fixme [short-mention] @expensify.com should now be considered a short-mention "candidate" testCases.forEach((testCase) => { expect(parser.replace(testCase.testString)).toBe(testCase.resultString); }); @@ -1325,12 +1326,8 @@ test('Test for user mention with @username@domain.com', () => { const resultString = '@username@expensify.com'; expect(parser.replace(testString)).toBe(resultString); }); -// Todo -// popraw psujący się @here testy -// Todo wszystkie edge kejsy w których short mention jest ambiguous -// short-mention -> possible-short-mention -test('Test for short mention mention with @username@domain.com', () => { +test('Test for short mention mention with @username', () => { const testString = '@john.doe'; const resultString = '@john.doe'; expect(parser.replace(testString)).toBe(resultString); @@ -1519,6 +1516,7 @@ test('Test for @here mention with italic, bold and strikethrough styles', () => ' @here!' + ' @here?'; + // Fixme [short-mention] these should now be short-mention candidates const resultString = '@here' + ' @here' + diff --git a/__tests__/ExpensiMark-test.js b/__tests__/ExpensiMark-test.js index 41c0a2c9..befe1958 100644 --- a/__tests__/ExpensiMark-test.js +++ b/__tests__/ExpensiMark-test.js @@ -1,7 +1,6 @@ /* eslint-disable max-len */ import ExpensiMark from '../lib/ExpensiMark'; import * as Utils from '../lib/utils'; -import {any, string} from "prop-types"; const parser = new ExpensiMark(); diff --git a/lib/ExpensiMark.ts b/lib/ExpensiMark.ts index 3ce43635..d468f936 100644 --- a/lib/ExpensiMark.ts +++ b/lib/ExpensiMark.ts @@ -417,29 +417,6 @@ export default class ExpensiMark { }, }, - { - name: 'shortMentions', - regex: new RegExp( - "(@here|[a-zA-Z0-9.!$%&+=?^\\`{|}-]?)(@(?=((?=[\\w]+[\\w'#%+-]+(?:\\.[\\w'#%+-]+)*)[\\w\\.'#%+-]{1,64}(?= |_|\\b))(?!([:\\/\\\\]))(?.*))\\S{3,254}(?=\\k$))(?!((?:(?!|[^<]*(<\\/pre>|<\\/code>|<\\/mention-user>|<\\/mention-here>))", - 'gim', - ), - replacement: (_extras, match, g1, g2) => { - if (!Str.isValidMention(match)) { - return match; - } - return `${g1}${g2}`; - }, - // rawInputReplacement: (_extras, match, g1, g2) => { - // const phoneNumberRegex = new RegExp(`^${Constants.CONST.REG_EXP.PHONE_PART}$`); - // const mention = g2.slice(1); - // const mentionWithoutSMSDomain = str_1.default.removeSMSDomain(mention); - // if (!str_1.default.isValidMention(match) || (phoneNumberRegex.test(mentionWithoutSMSDomain) && !str_1.default.isValidPhoneNumber(mentionWithoutSMSDomain))) { - // return match; - // } - // return `${g1}${g2}`; - // }, - }, - { name: 'quote', @@ -507,6 +484,30 @@ export default class ExpensiMark { rawInputReplacement: '$1$2', }, + { + name: 'shortMentions', + + regex: new RegExp( + "(@here|[a-zA-Z0-9.!$%&+=?^\\`{|}-]?)(@(?=((?=[\\w]+[\\w'#%+-]+(?:\\.[\\w'#%+-]+)*)[\\w\\.'#%+-]{1,64}(?= |_|\\b))(?!([:\\/\\\\]))(?.*))\\S{3,254}(?=\\k$))(?!((?:(?!|[^<]*(<\\/pre>|<\\/code>|<\\/mention-user>|<\\/mention-here>))", + 'gim', + ), + replacement: (_extras, match, g1, g2) => { + if (!Str.isValidMention(match)) { + return match; + } + return `${g1}${g2}`; + }, + // rawInputReplacement: (_extras, match, g1, g2) => { + // const phoneNumberRegex = new RegExp(`^${Constants.CONST.REG_EXP.PHONE_PART}$`); + // const mention = g2.slice(1); + // const mentionWithoutSMSDomain = str_1.default.removeSMSDomain(mention); + // if (!str_1.default.isValidMention(match) || (phoneNumberRegex.test(mentionWithoutSMSDomain) && !str_1.default.isValidPhoneNumber(mentionWithoutSMSDomain))) { + // return match; + // } + // return `${g1}${g2}`; + // }, + }, + { // Use \B in this case because \b doesn't match * or ~. // \B will match everything that \b doesn't, so it works From f7e27c7a95c8557385cf13ec8176462e14c43639 Mon Sep 17 00:00:00 2001 From: Mateusz Titz Date: Wed, 8 Jan 2025 13:51:16 +0100 Subject: [PATCH 3/3] Improve short mentions regex and make parser tests pass --- __tests__/ExpensiMark-HTML-test.js | 41 +++++++++++++++++++----------- lib/CONST.ts | 5 ++++ lib/ExpensiMark.ts | 33 ++++++++++++++++-------- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/__tests__/ExpensiMark-HTML-test.js b/__tests__/ExpensiMark-HTML-test.js index b18aecca..3266ce07 100644 --- a/__tests__/ExpensiMark-HTML-test.js +++ b/__tests__/ExpensiMark-HTML-test.js @@ -693,11 +693,10 @@ test('Test url replacements', () => { 'http://example.com/foo/*/bar/*/test.txt ' + 'test-.com ' + '-test.com ' + - '@test.com ' + - '@test.com test.com ' + - '@test.com @test.com '; + '@test.com ' + + '@test.com test.com ' + + '@test.com @test.com '; - // Fixme [short-mention] this errors on "test.com @test.com @test.com { { testString: 'expensify.com -expensify.com @expensify.com', resultString: - 'expensify.com -expensify.com @expensify.com', + 'expensify.com -expensify.com @expensify.com', }, { testString: 'https//www.expensify.com', @@ -932,7 +931,6 @@ test('Test urls autolinks correctly', () => { }, ]; - // Fixme [short-mention] @expensify.com should now be considered a short-mention "candidate" testCases.forEach((testCase) => { expect(parser.replace(testCase.testString)).toBe(testCase.resultString); }); @@ -1327,12 +1325,6 @@ test('Test for user mention with @username@domain.com', () => { expect(parser.replace(testString)).toBe(resultString); }); -test('Test for short mention mention with @username', () => { - const testString = '@john.doe'; - const resultString = '@john.doe'; - expect(parser.replace(testString)).toBe(resultString); -}); - test('Test for user mention with @@username@domain.com', () => { const testString = '@@username@expensify.com'; const resultString = '@@username@expensify.com'; @@ -1460,6 +1452,26 @@ test('Test for @here mention with inlineCodeBlock style', () => { expect(parser.replace(testString)).toBe(resultString); }); +describe('Tests for short mentions', () => { + test('short mentions should work for @username', () => { + const testString = '@johnny'; + const resultString = '@johnny'; + expect(parser.replace(testString)).toBe(resultString); + }); + + test('short mentions should work for @firstname.lastname', () => { + const testString = '@john.doe'; + const resultString = '@john.doe'; + expect(parser.replace(testString)).toBe(resultString); + }); + + test('short mentions should work and not break @here after mention', () => { + const testString = '@john.doe@here'; + const resultString = '@john.doe@here'; + expect(parser.replace(testString)).toBe(resultString); + }); +}); + // Examples that should match for here mentions: test('Test for here mention with @here', () => { const testString = '@here'; @@ -1516,7 +1528,6 @@ test('Test for @here mention with italic, bold and strikethrough styles', () => ' @here!' + ' @here?'; - // Fixme [short-mention] these should now be short-mention candidates const resultString = '@here' + ' @here' + @@ -1659,13 +1670,13 @@ test('Skip rendering invalid markdown', () => { test('Test for email with test+1@gmail.com@gmail.com', () => { const testString = 'test+1@gmail.com@gmail.com'; - const resultString = 'test+1@gmail.com@gmail.com'; + const resultString = 'test+1@gmail.com@gmail.com'; expect(parser.replace(testString)).toBe(resultString); }); test('Test for email with test@gmail.com@gmail.com', () => { const testString = 'test@gmail.com@gmail.com'; - const resultString = 'test@gmail.com@gmail.com'; + const resultString = 'test@gmail.com@gmail.com'; expect(parser.replace(testString)).toBe(resultString); }); diff --git a/lib/CONST.ts b/lib/CONST.ts index 431b7b4b..7c920706 100644 --- a/lib/CONST.ts +++ b/lib/CONST.ts @@ -419,6 +419,11 @@ const CONST = { */ EMOJI_RULE: /[\p{Extended_Pictographic}](\u200D[\p{Extended_Pictographic}]|[\u{1F3FB}-\u{1F3FF}]|[\u{E0020}-\u{E007F}]|\uFE0F|\u20E3)*|[\u{1F1E6}-\u{1F1FF}]{2}|[#*0-9]\uFE0F?\u20E3/gu, + + /** + * Regex to match a piece of text or @here, needed for both shortMention and userMention + */ + PRE_MENTION_TEXT_PART: '(@here|[a-zA-Z0-9.!$%&+=?^\\`{|}-]?)', }, REPORT: { diff --git a/lib/ExpensiMark.ts b/lib/ExpensiMark.ts index d468f936..543c0f50 100644 --- a/lib/ExpensiMark.ts +++ b/lib/ExpensiMark.ts @@ -365,7 +365,7 @@ export default class ExpensiMark { { name: 'userMentions', regex: new RegExp( - `(@here|[a-zA-Z0-9.!$%&+=?^\`{|}-]?)(@${Constants.CONST.REG_EXP.EMAIL_PART}|@${Constants.CONST.REG_EXP.PHONE_PART})(?!((?:(?!|[^<]*(<\\/pre>|<\\/code>))`, + `${Constants.CONST.REG_EXP.PRE_MENTION_TEXT_PART}(@${Constants.CONST.REG_EXP.EMAIL_PART}|@${Constants.CONST.REG_EXP.PHONE_PART})(?!((?:(?!|[^<]*(<\\/pre>|<\\/code>))`, 'gim', ), replacement: (_extras, match, g1, g2) => { @@ -484,11 +484,25 @@ export default class ExpensiMark { rawInputReplacement: '$1$2', }, + /** + * This regex matches a short user mention in a string. + * A short-mention is a string that starts with the '@' symbol and is followed by a valid user's primary login without the email domain part + * Ex: @john.doe, @user12345, but NOT @user@email.com + * + * Notes: + * Phone is not a valid short mention. + * In reality these "short-mentions" are just possible candidates, because the parser has no way of verifying if there exists a user named ex: @john.examplename. + * The actual verification whether these mentions are pointing to real users is done in specific projects using ExpensiMark. + * Nevertheless, "@john.examplename" is a correct possible short-mention, and so would be parsed. + * This behaviour is similar to treating every user@something as valid user login. + * + * This regex will correctly preserve any @here mentions, the same way as "userMention" rule. + */ { name: 'shortMentions', regex: new RegExp( - "(@here|[a-zA-Z0-9.!$%&+=?^\\`{|}-]?)(@(?=((?=[\\w]+[\\w'#%+-]+(?:\\.[\\w'#%+-]+)*)[\\w\\.'#%+-]{1,64}(?= |_|\\b))(?!([:\\/\\\\]))(?.*))\\S{3,254}(?=\\k$))(?!((?:(?!|[^<]*(<\\/pre>|<\\/code>|<\\/mention-user>|<\\/mention-here>))", + `${Constants.CONST.REG_EXP.PRE_MENTION_TEXT_PART}(@(?=((?=[\\w]+[\\w'#%+-]+(?:\\.[\\w'#%+-]+)*)[\\w\\.'#%+-]{1,64}(?= |_|\\b))(?!([:\\/\\\\]))(?.*))(?!here)\\S{3,254}(?=\\k$))(?!((?:(?!|[^<]*(<\\/pre>|<\\/code>|<\\/mention-user>|<\\/mention-here>))`, 'gim', ), replacement: (_extras, match, g1, g2) => { @@ -497,15 +511,12 @@ export default class ExpensiMark { } return `${g1}${g2}`; }, - // rawInputReplacement: (_extras, match, g1, g2) => { - // const phoneNumberRegex = new RegExp(`^${Constants.CONST.REG_EXP.PHONE_PART}$`); - // const mention = g2.slice(1); - // const mentionWithoutSMSDomain = str_1.default.removeSMSDomain(mention); - // if (!str_1.default.isValidMention(match) || (phoneNumberRegex.test(mentionWithoutSMSDomain) && !str_1.default.isValidPhoneNumber(mentionWithoutSMSDomain))) { - // return match; - // } - // return `${g1}${g2}`; - // }, + }, + + { + name: 'hereMentionAfterShortMentions', + regex: /(<\/mention-short>)(@here)(?=\b)/gm, + replacement: '$1$2', }, {