Merge pull request #31259 from software-mansion-labs/@kosmydel/invisi…

…ble-characters-2

Handle invisible characters in forms v2

src/CONST.ts

@@ -1366,6 +1366,10 @@ const CONST = {
         ILLEGAL_FILENAME_CHARACTERS: /\/|<|>|\*|"|:|\?|\\|\|/g,
 
         ENCODE_PERCENT_CHARACTER: /%(25)+/g,
+
+        INVISIBLE_CHARACTERS_GROUPS: /[\p{C}\p{Z}]/gu,
+
+        OTHER_INVISIBLE_CHARACTERS: /[\u3164]/g,
     },
 
     PRONOUNS: {

src/components/Form.js

@@ -6,6 +6,7 @@ import {withOnyx} from 'react-native-onyx';
 import _ from 'underscore';
 import compose from '@libs/compose';
 import * as ErrorUtils from '@libs/ErrorUtils';
+import * as ValidationUtils from '@libs/ValidationUtils';
 import Visibility from '@libs/Visibility';
 import stylePropTypes from '@styles/stylePropTypes';
 import styles from '@styles/styles';
@@ -126,14 +127,8 @@ function Form(props) {
      */
     const onValidate = useCallback(
         (values, shouldClearServerError = true) => {
-            const trimmedStringValues = {};
-            _.each(values, (inputValue, inputID) => {
-                if (_.isString(inputValue)) {
-                    trimmedStringValues[inputID] = inputValue.trim();
-                } else {
-                    trimmedStringValues[inputID] = inputValue;
-                }
-            });
+            // Trim all string values
+            const trimmedStringValues = ValidationUtils.prepareValues(values);
 
             if (shouldClearServerError) {
                 FormActions.setErrors(props.formID, null);
@@ -191,7 +186,7 @@ function Form(props) {
 
             return touchedInputErrors;
         },
-        [errors, touchedInputs, props.formID, validate],
+        [props.formID, validate, errors],
     );
 
     useEffect(() => {
@@ -228,11 +223,14 @@ function Form(props) {
             return;
         }
 
+        // Trim all string values
+        const trimmedStringValues = ValidationUtils.prepareValues(inputValues);
+
         // Touches all form inputs so we can validate the entire form
         _.each(inputRefs.current, (inputRef, inputID) => (touchedInputs.current[inputID] = true));
 
         // Validate form and return early if any errors are found
-        if (!_.isEmpty(onValidate(inputValues))) {
+        if (!_.isEmpty(onValidate(trimmedStringValues))) {
             return;
         }
 
@@ -242,8 +240,8 @@ function Form(props) {
         }
 
         // Call submit handler
-        onSubmit(inputValues);
-    }, [props.formState, onSubmit, inputRefs, inputValues, onValidate, touchedInputs, props.network.isOffline, props.enabledWhenOffline]);
+        onSubmit(trimmedStringValues);
+    }, [props.formState.isLoading, props.network.isOffline, props.enabledWhenOffline, inputValues, onValidate, onSubmit]);
 
     /**
      * Loops over Form's children and automatically supplies Form props to them

src/components/Form/FormProvider.js

@@ -6,6 +6,7 @@ import _ from 'underscore';
 import networkPropTypes from '@components/networkPropTypes';
 import {withNetwork} from '@components/OnyxProvider';
 import compose from '@libs/compose';
+import * as ValidationUtils from '@libs/ValidationUtils';
 import Visibility from '@libs/Visibility';
 import stylePropTypes from '@styles/stylePropTypes';
 import * as FormActions from '@userActions/FormActions';
@@ -108,14 +109,7 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC
 
     const onValidate = useCallback(
         (values, shouldClearServerError = true) => {
-            const trimmedStringValues = {};
-            _.each(values, (inputValue, inputID) => {
-                if (_.isString(inputValue)) {
-                    trimmedStringValues[inputID] = inputValue.trim();
-                } else {
-                    trimmedStringValues[inputID] = inputValue;
-                }
-            });
+            const trimmedStringValues = ValidationUtils.prepareValues(values);
 
             if (shouldClearServerError) {
                 FormActions.setErrors(formID, null);
@@ -186,11 +180,14 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC
             return;
         }
 
+        // Prepare values before submitting
+        const trimmedStringValues = ValidationUtils.prepareValues(inputValues);
+
         // Touches all form inputs so we can validate the entire form
         _.each(inputRefs.current, (inputRef, inputID) => (touchedInputs.current[inputID] = true));
 
         // Validate form and return early if any errors are found
-        if (!_.isEmpty(onValidate(inputValues))) {
+        if (!_.isEmpty(onValidate(trimmedStringValues))) {
             return;
         }
 
@@ -199,7 +196,7 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC
             return;
         }
 
-        onSubmit(inputValues);
+        onSubmit(trimmedStringValues);
     }, [enabledWhenOffline, formState.isLoading, inputValues, network.isOffline, onSubmit, onValidate]);
 
     const registerInput = useCallback(

src/libs/StringUtils.ts

@@ -10,4 +10,57 @@ function sanitizeString(str: string): string {
     return _.deburr(str).toLowerCase().replaceAll(CONST.REGEX.NON_ALPHABETIC_AND_NON_LATIN_CHARS, '');
 }
 
-export default {sanitizeString};
+/**
+ *  Check if the string would be empty if all invisible characters were removed.
+ */
+function isEmptyString(value: string): boolean {
+    // \p{C} matches all 'Other' characters
+    // \p{Z} matches all separators (spaces etc.)
+    // Source: http://www.unicode.org/reports/tr18/#General_Category_Property
+    let transformed = value.replace(CONST.REGEX.INVISIBLE_CHARACTERS_GROUPS, '');
+
+    // Remove other invisible characters that are not in the above unicode categories
+    transformed = transformed.replace(CONST.REGEX.OTHER_INVISIBLE_CHARACTERS, '');
+
+    // Check if after removing invisible characters the string is empty
+    return transformed === '';
+}
+
+/**
+ *  Remove invisible characters from a string except for spaces and format characters for emoji, and trim it.
+ */
+function removeInvisibleCharacters(value: string): string {
+    let result = value;
+
+    // Remove spaces:
+    // - \u200B: zero-width space
+    // - \u00A0: non-breaking space
+    // - \u2060: word joiner
+    result = result.replace(/[\u200B\u00A0\u2060]/g, '');
+
+    // Temporarily replace all newlines with non-breaking spaces
+    // It is necessary because the next step removes all newlines because they are in the (Cc) category
+    result = result.replace(/\n/g, '\u00A0');
+
+    // Remove all characters from the 'Other' (C) category except for format characters (Cf)
+    // because some of them are used for emojis
+    result = result.replace(/[\p{Cc}\p{Cs}\p{Co}\p{Cn}]/gu, '');
+
+    // Replace all non-breaking spaces with newlines
+    result = result.replace(/\u00A0/g, '\n');
+
+    // Remove characters from the (Cf) category that are not used for emojis
+    result = result.replace(/[\u200E-\u200F]/g, '');
+
+    // Remove all characters from the 'Separator' (Z) category except for Space Separator (Zs)
+    result = result.replace(/[\p{Zl}\p{Zp}]/gu, '');
+
+    // If the result consist of only invisible characters, return an empty string
+    if (isEmptyString(result)) {
+        return '';
+    }
+
+    return result.trim();
+}
+
+export default {sanitizeString, isEmptyString, removeInvisibleCharacters};

src/libs/ValidationUtils.ts

@@ -9,6 +9,7 @@ import {Report} from '@src/types/onyx';
 import * as OnyxCommon from '@src/types/onyx/OnyxCommon';
 import * as CardUtils from './CardUtils';
 import * as LoginUtils from './LoginUtils';
+import StringUtils from './StringUtils';
 
 /**
  * Implements the Luhn Algorithm, a checksum formula used to validate credit card
@@ -73,7 +74,7 @@ function isValidPastDate(date: string | Date): boolean {
  */
 function isRequiredFulfilled(value: string | Date | unknown[] | Record<string, unknown>): boolean {
     if (typeof value === 'string') {
-        return value.trim().length > 0;
+        return !StringUtils.isEmptyString(value);
     }
 
     if (isDate(value)) {
@@ -352,6 +353,25 @@ function isValidAccountRoute(accountID: number): boolean {
     return CONST.REGEX.NUMBER.test(String(accountID)) && accountID > 0;
 }
 
+type ValuesType = Record<string, unknown>;
+
+/**
+ * This function is used to remove invisible characters from strings before validation and submission.
+ */
+function prepareValues(values: ValuesType): ValuesType {
+    const trimmedStringValues: ValuesType = {};
+
+    for (const [inputID, inputValue] of Object.entries(values)) {
+        if (typeof inputValue === 'string') {
+            trimmedStringValues[inputID] = StringUtils.removeInvisibleCharacters(inputValue);
+        } else {
+            trimmedStringValues[inputID] = inputValue;
+        }
+    }
+
+    return trimmedStringValues;
+}
+
 export {
     meetsMinimumAgeRequirement,
     meetsMaximumAgeRequirement,
@@ -385,4 +405,5 @@ export {
     isNumeric,
     isValidAccountRoute,
     isValidRecoveryCode,
+    prepareValues,
 };

src/pages/workspace/WorkspaceSettingsPage.js

@@ -20,6 +20,7 @@ import compose from '@libs/compose';
 import Navigation from '@libs/Navigation/Navigation';
 import * as ReportUtils from '@libs/ReportUtils';
 import * as UserUtils from '@libs/UserUtils';
+import * as ValidationUtils from '@libs/ValidationUtils';
 import styles from '@styles/styles';
 import * as Policy from '@userActions/Policy';
 import CONST from '@src/CONST';
@@ -77,7 +78,7 @@ function WorkspaceSettingsPage({policy, currencyList, windowWidth, route}) {
         const errors = {};
         const name = values.name.trim();
 
-        if (!name || !name.length) {
+        if (!ValidationUtils.isRequiredFulfilled(name)) {
             errors.name = 'workspace.editor.nameIsRequiredError';
         } else if ([...name].length > CONST.WORKSPACE_NAME_CHARACTER_LIMIT) {
             // Uses the spread syntax to count the number of Unicode code points instead of the number of UTF-16

src/stories/Form.stories.js

@@ -10,6 +10,7 @@ import StatePicker from '@components/StatePicker';
 import Text from '@components/Text';
 import TextInput from '@components/TextInput';
 import NetworkConnection from '@libs/NetworkConnection';
+import * as ValidationUtils from '@libs/ValidationUtils';
 import styles from '@styles/styles';
 import * as FormActions from '@userActions/FormActions';
 import CONST from '@src/CONST';
@@ -177,28 +178,28 @@ const defaultArgs = {
     submitButtonText: 'Submit',
     validate: (values) => {
         const errors = {};
-        if (!values.routingNumber) {
+        if (!ValidationUtils.isRequiredFulfilled(values.routingNumber)) {
             errors.routingNumber = 'Please enter a routing number';
         }
-        if (!values.accountNumber) {
+        if (!ValidationUtils.isRequiredFulfilled(values.accountNumber)) {
             errors.accountNumber = 'Please enter an account number';
         }
-        if (!values.street) {
+        if (!ValidationUtils.isRequiredFulfilled(values.street)) {
             errors.street = 'Please enter an address';
         }
-        if (!values.dob) {
+        if (!ValidationUtils.isRequiredFulfilled(values.dob)) {
             errors.dob = 'Please enter your date of birth';
         }
-        if (!values.pickFruit) {
+        if (!ValidationUtils.isRequiredFulfilled(values.pickFruit)) {
             errors.pickFruit = 'Please select a fruit';
         }
-        if (!values.pickAnotherFruit) {
+        if (!ValidationUtils.isRequiredFulfilled(values.pickAnotherFruit)) {
             errors.pickAnotherFruit = 'Please select a fruit';
         }
-        if (!values.state) {
+        if (!ValidationUtils.isRequiredFulfilled(values.state)) {
             errors.state = 'Please select a state';
         }
-        if (!values.checkbox) {
+        if (!ValidationUtils.isRequiredFulfilled(values.checkbox)) {
             errors.checkbox = 'You must accept the Terms of Service to continue';
         }
         return errors;

tests/unit/isEmptyString.js

@@ -0,0 +1,92 @@
+import _ from 'underscore';
+import enEmojis from '../../assets/emojis/en';
+import StringUtils from '../../src/libs/StringUtils';
+
+describe('libs/StringUtils.isEmptyString', () => {
+    it('basic tests', () => {
+        expect(StringUtils.isEmptyString('test')).toBe(false);
+        expect(StringUtils.isEmptyString('test test')).toBe(false);
+        expect(StringUtils.isEmptyString('test test test')).toBe(false);
+        expect(StringUtils.isEmptyString(' ')).toBe(true);
+    });
+    it('trim spaces', () => {
+        expect(StringUtils.isEmptyString(' test')).toBe(false);
+        expect(StringUtils.isEmptyString('test ')).toBe(false);
+        expect(StringUtils.isEmptyString(' test ')).toBe(false);
+    });
+    it('remove invisible characters', () => {
+        expect(StringUtils.isEmptyString('\u200B')).toBe(true);
+        expect(StringUtils.isEmptyString('\u200B')).toBe(true);
+        expect(StringUtils.isEmptyString('\u200B ')).toBe(true);
+        expect(StringUtils.isEmptyString('\u200B \u200B')).toBe(true);
+        expect(StringUtils.isEmptyString('\u200B \u200B ')).toBe(true);
+    });
+    it('remove invisible characters (Cc)', () => {
+        expect(StringUtils.isEmptyString('\u0000')).toBe(true);
+        expect(StringUtils.isEmptyString('\u0001')).toBe(true);
+        expect(StringUtils.isEmptyString('\u0009')).toBe(true);
+    });
+    it('remove invisible characters (Cf)', () => {
+        expect(StringUtils.isEmptyString('\u200E')).toBe(true);
+        expect(StringUtils.isEmptyString('\u200F')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2060')).toBe(true);
+    });
+    it('remove invisible characters (Cs)', () => {
+        expect(StringUtils.isEmptyString('\uD800')).toBe(true);
+        expect(StringUtils.isEmptyString('\uD801')).toBe(true);
+        expect(StringUtils.isEmptyString('\uD802')).toBe(true);
+    });
+    it('remove invisible characters (Co)', () => {
+        expect(StringUtils.isEmptyString('\uE000')).toBe(true);
+        expect(StringUtils.isEmptyString('\uE001')).toBe(true);
+        expect(StringUtils.isEmptyString('\uE002')).toBe(true);
+    });
+    it('remove invisible characters (Zl)', () => {
+        expect(StringUtils.isEmptyString('\u2028')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2029')).toBe(true);
+        expect(StringUtils.isEmptyString('\u202A')).toBe(true);
+    });
+    it('basic check emojis not removed', () => {
+        expect(StringUtils.isEmptyString('😀')).toBe(false);
+    });
+    it('all emojis not removed', () => {
+        _.keys(enEmojis).forEach((key) => {
+            expect(StringUtils.isEmptyString(key)).toBe(false);
+        });
+    });
+    it('remove invisible characters (editpad)', () => {
+        expect(StringUtils.isEmptyString('\u0020')).toBe(true);
+        expect(StringUtils.isEmptyString('\u00A0')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2000')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2001')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2002')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2003')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2004')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2005')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2006')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2007')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2008')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2009')).toBe(true);
+        expect(StringUtils.isEmptyString('\u200A')).toBe(true);
+        expect(StringUtils.isEmptyString('\u2028')).toBe(true);
+        expect(StringUtils.isEmptyString('\u205F')).toBe(true);
+        expect(StringUtils.isEmptyString('\u3000')).toBe(true);
+        expect(StringUtils.isEmptyString(' ')).toBe(true);
+    });
+    it('other tests', () => {
+        expect(StringUtils.isEmptyString('\u200D')).toBe(true);
+        expect(StringUtils.isEmptyString('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe(false);
+        expect(StringUtils.isEmptyString('\uD83C')).toBe(true);
+        expect(StringUtils.isEmptyString('\uDFF4')).toBe(true);
+        expect(StringUtils.isEmptyString('\uDB40')).toBe(true);
+        expect(StringUtils.isEmptyString('\uDC67')).toBe(true);
+        expect(StringUtils.isEmptyString('\uDC62')).toBe(true);
+        expect(StringUtils.isEmptyString('\uDC65')).toBe(true);
+        expect(StringUtils.isEmptyString('\uDC6E')).toBe(true);
+        expect(StringUtils.isEmptyString('\uDC67')).toBe(true);
+        expect(StringUtils.isEmptyString('\uDC7F')).toBe(true);
+
+        // A special test, an invisible character from other Unicode categories than format and control
+        expect(StringUtils.isEmptyString('\u3164')).toBe(true);
+    });
+});