remove invisible Unicode special characters from search keywords

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Release Notes for Craft CMS 4
 
+## Fixed a bug where all multi-byte characters were getting stripped out of search indexes. ([#16457](https://github.com/craftcms/cms/issues/16457))
+
 ## 4.14.0.2 - 2025-01-21
 
 - Fixed an error that occurred when creating a new Structure section. ([#16476](https://github.com/craftcms/cms/issues/16476))

src/helpers/FileHelper.php

@@ -254,13 +254,9 @@ public static function sanitizeFilename(string $filename, array $options = []):
         // Replace any control characters in the name with a space.
         $filename = preg_replace("/\\x{00a0}/iu", ' ', $filename);
 
+        // Remove invisible chars from the filename
         // https://github.com/craftcms/cms/issues/12741
-        // Remove soft hyphens (00ad), no break (0083),
-        // zero width space (200b), zero width non-joiner (200c), zero width joiner (200d),
-        // LTR character (200e), RTL character (200f),
-        // invisible times (2062), invisible comma (2063), invisible plus (2064),
-        // zero width non-break space (feff) in the filename
-        $filename = preg_replace('/\\x{00ad}|\\x{0083}|\\x{200b}|\\x{200c}|\\x{200d}|\\x{200e}|\\x{200f}|\\x{2062}|\\x{2063}|\\x{2064}|\\x{feff}/iu', '', $filename);
+        $filename = preg_replace(StringHelper::invisibleCharsRegex(), '', $filename);
 
         // Strip any characters not allowed.
         $filename = str_replace($disallowedChars, '', strip_tags($filename));

src/helpers/Search.php

@@ -66,9 +66,9 @@ public static function normalizeKeywords(array|string $str, array $ignore = [],
             }
         }
 
-        // Get rid of Unicode special characters
+        // Get rid of invisible Unicode special characters
         // (see https://github.com/craftcms/cms/issues/16430)
-        $str = preg_replace('/[\x{80}-\x{10FFFF}]/u', '', $str);
+        $str = preg_replace(StringHelper::invisibleCharsRegex(), '', $str);
 
         // Strip out new lines and superfluous spaces
         return trim(preg_replace(['/[\n\r]+/u', '/\s{2,}/u'], ' ', $str));

src/helpers/StringHelper.php

@@ -2039,4 +2039,34 @@ public static function indent(string $str, string $indent = '    '): string
     {
         return implode("\n", array_map(fn(string $line) => $indent . $line, static::lines($str)));
     }
+
+    /**
+     * Returns a regex pattern for invisible characters.
+     *
+     * @return string
+     * @since 4.14.1
+     */
+    public static function invisibleCharsRegex(): string
+    {
+        $invisibleCharCodes = [
+            '00ad', // soft hyphen
+            '0083', // no break
+            '200b', // zero width space
+            '200c', // zero width non-joiner
+            '200d', // zero width joiner
+            '200e', // LTR character
+            '200f', // RTL character
+            '2062', // invisible times
+            '2063', // invisible comma
+            '2064', // invisible plus
+            'feff', //zero width non-break space
+        ];
+
+        array_walk(
+            $invisibleCharCodes,
+            fn(&$charCode) => $charCode = sprintf('\\x{%s}', $charCode)
+        );
+
+        return sprintf('/%s/iu', implode('|', $invisibleCharCodes));
+    }
 }

tests/unit/helpers/SearchHelperTest.php

@@ -49,7 +49,7 @@ public function normalizeKeywordsDataProviders(): array
             ['', '&#11aa;'],
             ['test test', 'TEST TEST'],
             ['', ['♠', '♣', '♥', '♦']],
-            ['', ['♠', '♣', '♥', '♦'], [], false],
+            ['♠ ♣ ♥ ♦', ['♠', '♣', '♥', '♦'], [], false],
             ['test', 'test                       '],
             ['', 'test', ['test']],
             ['test', 'test👍'],
@@ -66,6 +66,9 @@ public function normalizeKeywordsDataProviders(): array
             ['this accord', 'this?D’accord!'],
             ['womens', "women's"],
             ['womens', 'women’s'],
+            ['toolbox', 'tool­box'],
+            ['toolbox', 'tool‍box'],
+            ['tool box', 'tool box'],
         ];
     }
 }