[1.x] Automatically transliterate logographic inputs for slug generation

RELEASE_NOTES.md

@@ -16,6 +16,7 @@ This serves two purposes:
 - All page types now support the `description` front matter field (used in page metadata) in https://github.com/hydephp/develop/pull/1884
 - Added a new `Filesystem::findFiles()` method to find files in a directory in https://github.com/hydephp/develop/pull/2064
 - Added `webp` to the list of default media extensions in https://github.com/hydephp/framework/pull/663
+- Added a new slug generation helper to improve internationalization support in https://github.com/hydephp/develop/pull/2070
 
 ### Changed
 - Changed the `Hyde` facade to use a `@mixin` annotation instead of single method annotations in https://github.com/hydephp/develop/pull/1919
@@ -26,6 +27,7 @@ This serves two purposes:
 - The `torchlight:install` command is now hidden from the command list as it's already installed in https://github.com/hydephp/develop/pull/1879
 - Updated the home page fallback link in the 404 template to lead to the site root in https://github.com/hydephp/develop/pull/1880 (fixes https://github.com/hydephp/develop/issues/1781)
 - Normalized remote URL checks so that protocol relative URLs `//` are consistently considered to be remote in all places in https://github.com/hydephp/develop/pull/1882 (fixes https://github.com/hydephp/develop/issues/1788)
+- Page slugs are now generated using our automatically internationalizing slug generator to transliterate input to ASCII in https://github.com/hydephp/develop/pull/2070
 - Replaced internal usages of glob functions with our improved file finder in https://github.com/hydephp/develop/pull/2064
 - Updated to HydeFront v3.4 in https://github.com/hydephp/develop/pull/1803
 - Realtime Compiler: Virtual routes are now managed through the service container in https://github.com/hydephp/develop/pull/1858
@@ -46,6 +48,7 @@ This serves two purposes:
 - Fixed heading permalinks button text showing in Google Search previews https://github.com/hydephp/develop/issues/1801 in https://github.com/hydephp/develop/pull/1803
 - Fixed routing issues with nested 404 pages where an index page does not exist https://github.com/hydephp/develop/issues/1781 in https://github.com/hydephp/develop/pull/1880
 - Fixed URL metadata for blog posts not using customized post output directories in https://github.com/hydephp/develop/pull/1889
+- Fixed lacking support for logographic slug generation https://github.com/hydephp/hyde/issues/269 in https://github.com/hydephp/develop/pull/2070 
 - Improved printed documentation views in https://github.com/hydephp/develop/pull/2005
 - Fixed "BuildService finding non-existent files to copy in Debian" https://github.com/hydephp/framework/issues/662 in https://github.com/hydephp/develop/pull/2064
 - Fixed "Undefined constant `Hyde\Foundation\Kernel\GLOB_BRACE`" https://github.com/hydephp/hyde/issues/270 in https://github.com/hydephp/develop/pull/2064

docs/_data/partials/hyde-pages-api/hyde-kernel-string-methods.md

@@ -1,7 +1,7 @@
 <section id="hyde-kernel-string-methods">
 
 <!-- Start generated docs for Hyde\Foundation\Concerns\ImplementsStringHelpers -->
-<!-- Generated by HydePHP DocGen script at 2023-03-11 11:17:34 in 0.07ms -->
+<!-- Generated by HydePHP DocGen script at 2024-12-22 09:14:25 in 0.07ms -->
 
 #### `makeTitle()`
 
@@ -11,6 +11,14 @@ No description provided.
 Hyde::makeTitle(string $value): string
 ```
 
+#### `makeSlug()`
+
+No description provided.
+
+```php
+Hyde::makeSlug(string $value): string
+```
+
 #### `normalizeNewlines()`
 
 No description provided.

docs/architecture-concepts/the-hydekernel.md

@@ -140,7 +140,7 @@ Hyde::routes(): Hyde\Foundation\Kernel\RouteCollection
 <section id="hyde-kernel-string-methods">
 
 <!-- Start generated docs for Hyde\Foundation\Concerns\ImplementsStringHelpers -->
-<!-- Generated by HydePHP DocGen script at 2023-03-11 11:17:34 in 0.07ms -->
+<!-- Generated by HydePHP DocGen script at 2024-12-22 09:14:25 in 0.07ms -->
 
 #### `makeTitle()`
 
@@ -150,6 +150,14 @@ No description provided.
 Hyde::makeTitle(string $value): string
 ```
 
+#### `makeSlug()`
+
+No description provided.
+
+```php
+Hyde::makeSlug(string $value): string
+```
+
 #### `normalizeNewlines()`
 
 No description provided.

packages/framework/src/Foundation/Concerns/ImplementsStringHelpers.php

@@ -38,6 +38,19 @@ public static function makeTitle(string $value): string
         ));
     }
 
+    public static function makeSlug(string $value): string
+    {
+        // Expand camelCase and PascalCase to separate words
+        $value = preg_replace('/([a-z])([A-Z])/', '$1 $2', $value);
+
+        // Transliterate international characters to ASCII
+        $value = Str::transliterate($value);
+
+        // Todo: In v2.0 we will use the following dictionary: ['@' => 'at', '&' => 'and']
+
+        return Str::slug($value);
+    }
+
     public static function normalizeNewlines(string $string): string
     {
         return str_replace("\r\n", "\n", $string);

packages/framework/src/Framework/Actions/CreatesNewMarkdownPostFile.php

@@ -6,9 +6,9 @@
 
 use Hyde\Framework\Exceptions\FileConflictException;
 use Hyde\Facades\Filesystem;
+use Hyde\Hyde;
 use Hyde\Pages\MarkdownPost;
 use Illuminate\Support\Carbon;
-use Illuminate\Support\Str;
 
 /**
  * Offloads logic for the make:post command.
@@ -48,7 +48,7 @@ public function __construct(string $title, ?string $description, ?string $catego
         $this->customContent = $customContent;
 
         $this->date = Carbon::make($date ?? Carbon::now())->format('Y-m-d H:i');
-        $this->identifier = Str::slug($title);
+        $this->identifier = Hyde::makeSlug($title);
     }
 
     /**

packages/framework/src/Framework/Actions/CreatesNewPageSourceFile.php

@@ -81,7 +81,7 @@ protected function fileName(string $title): string
         }
 
         // And return a slug made from just the title without the subdirectory
-        return Str::slug(basename($title));
+        return Hyde::makeSlug(basename($title));
     }
 
     protected function normalizeSubdirectory(string $title): string

packages/framework/src/Framework/Features/Navigation/DocumentationSidebar.php

@@ -11,7 +11,6 @@
 use Hyde\Support\Facades\Render;
 use Hyde\Support\Models\Route;
 use Illuminate\Support\Collection;
-use Illuminate\Support\Str;
 
 use function collect;
 
@@ -48,14 +47,15 @@ public function getGroups(): array
     public function getItemsInGroup(?string $group): Collection
     {
         return $this->items->filter(function (NavItem $item) use ($group): bool {
-            return ($item->getGroup() === $group) || ($item->getGroup() === Str::slug($group));
+            return ($item->getGroup() === $group) || ($item->getGroup() === Hyde::makeSlug($group));
         })->sortBy('navigation.priority')->values();
     }
 
     public function isGroupActive(string $group): bool
     {
-        return Str::slug(Render::getPage()->navigationMenuGroup()) === $group
-            || $this->isPageIndexPage() && $this->shouldIndexPageBeActive($group);
+        $normalized = Hyde::makeSlug(Render::getPage()->navigationMenuGroup() ?? 'other');
+
+        return ($normalized === $group) || ($this->isPageIndexPage() && $this->shouldIndexPageBeActive($group));
     }
 
     public function makeGroupTitle(string $group): string

packages/framework/src/Framework/Features/Navigation/NavItem.php

@@ -7,7 +7,6 @@
 use Hyde\Foundation\Facades\Routes;
 use Hyde\Hyde;
 use Hyde\Support\Models\Route;
-use Illuminate\Support\Str;
 use Stringable;
 
 /**
@@ -133,6 +132,6 @@ protected static function getRouteGroup(Route $route): ?string
 
     protected static function normalizeGroupKey(?string $group): ?string
     {
-        return $group ? Str::slug($group) : null;
+        return $group ? Hyde::makeSlug($group) : null;
     }
 }

packages/framework/tests/Feature/HydeKernelTest.php

@@ -32,6 +32,7 @@
  * @covers \Hyde\Hyde
  *
  * @see \Hyde\Framework\Testing\Unit\HydeHelperFacadeMakeTitleTest
+ * @see \Hyde\Framework\Testing\Unit\HydeHelperFacadeMakeSlugTest
  * @see \Hyde\Framework\Testing\Feature\HydeExtensionFeatureTest
  */
 class HydeKernelTest extends TestCase
@@ -108,6 +109,11 @@ public function testMakeTitleHelperReturnsTitleFromPageSlug()
         $this->assertSame('Foo Bar', Hyde::makeTitle('foo-bar'));
     }
 
+    public function testMakeSlugHelperReturnsSlugFromTitle()
+    {
+        $this->assertSame('foo-bar', Hyde::makeSlug('Foo Bar'));
+    }
+
     public function testNormalizeNewlinesReplacesCarriageReturnsWithUnixEndings()
     {
         $this->assertSame("foo\nbar\nbaz", Hyde::normalizeNewlines("foo\nbar\r\nbaz"));

packages/framework/tests/Feature/InternationalizationTest.php

@@ -0,0 +1,117 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Hyde\Framework\Testing\Feature;
+
+use Hyde\Facades\Filesystem;
+use Hyde\Framework\Actions\CreatesNewMarkdownPostFile;
+use Hyde\Framework\Actions\StaticPageBuilder;
+use Hyde\Hyde;
+use Hyde\Pages\MarkdownPost;
+use Hyde\Testing\TestCase;
+
+/**
+ * @coversNothing High level test to ensure the internationalization features are working.
+ */
+class InternationalizationTest extends TestCase
+{
+    /**
+     * @dataProvider internationalCharacterSetsProvider
+     */
+    public function testCanCreateBlogPostFilesWithInternationalCharacterSets(
+        string $title,
+        string $description,
+        string $expectedSlug,
+        string $expectedTitle
+    ) {
+        $creator = new CreatesNewMarkdownPostFile($title, $description, 'blog', 'default', '2024-12-22 10:45');
+        $path = $creator->save();
+
+        $this->assertSame("_posts/$expectedSlug.md", $path);
+        $this->assertSame($expectedSlug, $creator->getIdentifier());
+        $this->assertSame($creator->getIdentifier(), Hyde::makeSlug($title));
+        $this->assertFileExists($path);
+
+        $contents = file_get_contents($path);
+
+        if (str_contains($title, ' ')) {
+            $expectedTitle = "'$expectedTitle'";
+        }
+
+        if (str_contains($description, ' ')) {
+            $description = "'$description'";
+        }
+
+        $this->assertStringContainsString("title: $expectedTitle", $contents);
+        $this->assertSame(<<<EOF
+        ---
+        title: {$expectedTitle}
+        description: {$description}
+        category: blog
+        author: default
+        date: '2024-12-22 10:45'
+        ---
+
+        ## Write something awesome.
+
+        EOF, $contents);
+
+        Filesystem::unlink($path);
+    }
+
+    /**
+     * @dataProvider internationalCharacterSetsProvider
+     */
+    public function testCanCompileBlogPostFilesWithInternationalCharacterSets(
+        string $title,
+        string $description,
+        string $expectedSlug,
+        string $expectedTitle
+    ) {
+        $page = new MarkdownPost($expectedSlug, [
+            'title' => $title,
+            'description' => $description,
+            'category' => 'blog',
+            'author' => 'default',
+            'date' => '2024-12-22 10:45',
+        ]);
+
+        $path = StaticPageBuilder::handle($page);
+
+        $this->assertSame(Hyde::path("_site/posts/$expectedSlug.html"), $path);
+        $this->assertFileExists($path);
+
+        $contents = file_get_contents($path);
+
+        $this->assertStringContainsString("<title>HydePHP - $expectedTitle</title>", $contents);
+        $this->assertStringContainsString("<h1 itemprop=\"headline\" class=\"mb-4\">$expectedTitle</h1>", $contents);
+        $this->assertStringContainsString("<meta name=\"description\" content=\"$description\">", $contents);
+
+        Filesystem::unlink($path);
+    }
+
+    public static function internationalCharacterSetsProvider(): array
+    {
+        return [
+            'Chinese (Simplified)' => [
+                '你好世界',
+                '简短描述',
+                'ni-hao-shi-jie',
+                '你好世界',
+            ],
+            'Japanese' => [
+                'こんにちは世界',
+                '短い説明',
+                'konnichihashi-jie',
+                'こんにちは世界',
+            ],
+            'Korean' => [
+                '안녕하세요 세계',
+                '짧은 설명',
+                'annyeonghaseyo-segye',
+                '안녕하세요 세계',
+            ],
+        ];
+    }
+}