I forked your code - take a look

.gitignore

@@ -0,0 +1,6 @@
+/vendor/
+/.idea/
+/composer.lock
+/phpcpd.phar
+/phpcs.phar
+/data/cache/

LEGAL.md

@@ -0,0 +1,18 @@
+/** legal notice: https://github.com/nitotm/efficient-language-detector/blob/main/legal.md */
+
+/*
+Copyright 2019,2020,2021,2022,2023 Nito T.M.
+Author URL: https://github.com/nitotm
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/

Makefile

@@ -0,0 +1,42 @@
+default: test
+
+init: get-phpcpd composer-update
+
+validate: rector phpcpd phpmd phpstan psalm
+
+rector:
+	./vendor/bin/rector -n
+
+phpcpd:
+	./phpcpd.phar src
+
+phpmd:
+	./vendor/bin/phpmd src text codesize,controversial,design,unusedcode
+	./vendor/bin/phpmd src text cleancode
+
+phpstan:
+	php ./vendor/bin/phpstan
+
+psalm:
+	./vendor/bin/psalm
+
+get-phpcpd:
+	wget https://phar.phpunit.de/phpcpd.phar
+	chmod +x ./phpcpd.phar
+
+composer-update:
+	composer update
+
+fix: fix-rector
+
+fix-rector:
+	./vendor/bin/rector
+
+benchmark:
+	php benchmark.php
+
+test:
+	php tests.php
+
+example:
+	php index.php

benchmark.php

@@ -0,0 +1,91 @@
+<?php
+/** legal notice: https://github.com/nitotm/efficient-language-detector/blob/main/LEGAL.md */
+
+declare(strict_types = 1);
+
+namespace Nitotm\Eld;
+
+include __DIR__ . "/vendor/autoload.php";
+
+if (PHP_SAPI !== 'cli') {
+    echo "<pre>";
+}
+
+$path = __DIR__ . '/benchmarkdata/';
+$files = [
+    $path . 'tweets.txt',
+    $path . 'big-test.txt',
+    $path . 'sentences.txt',
+    $path . 'word-pairs.txt',
+    $path . 'single-words.txt',
+];
+
+/*
+Results for v1.0.0, PHP 7.4.4, ngrams-m.php
+
+tweets.txt - Correct ratio: 99.28% Time: 0.30713295936584
+big-test.txt - Correct ratio: 99.42% Time: 2.4928371906281
+sentences.txt - Correct ratio: 98.78% Time: 2.1568570137024
+word-pairs.txt - Correct ratio: 87.56% Time: 0.66023302078247
+single-words.txt - Correct ratio: 73.31% Time: 0.47791314125061
+
+Results eurosat7/efficient-language-detector:main@2023-06-18, PHP 8.2.7, ngrams-m.php
+
+tweets.txt - Correct ratio: 98.93% Duration: 0.29958343505859
+big-test.txt - Correct ratio: 92.05% Duration: 2.1553561687469
+sentences.txt - Correct ratio: 93.31% Duration: 1.957035779953
+word-pairs.txt - Correct ratio: 73.43% Duration: 1.0882313251495
+single-words.txt - Correct ratio: 55.94% Duration: 0.93544888496399
+
+If correct ratio is vastly inferior, try with 'ngrams-m.safe.php'
+*/
+$languageData = new LanguageData(
+    'ngrams-m.php' //  -> 'ngrams-m.safe.php'
+);
+$languageSubset = new LanguageSet(
+    $languageData,
+    usecache: true,
+);
+$languageDetector = new LanguageDetector(
+    languageData: $languageData,
+    languageSet: $languageSubset,
+    minByteLength: 0,
+    minNgrams: 0
+);
+
+$summary = [];
+$perlang = [];
+foreach ($files as $file) {
+    $duration = 0;
+    $correct = 0;
+    $failed = 0;
+    $fp = fopen($file, 'rb');
+    if ($fp === false) {
+        echo('cannot read from file ' . $file . PHP_EOL);
+        continue;
+    }
+    echo "- " . basename($file) . PHP_EOL;
+    while (($line = fgets($fp, 4096)) !== false) {
+        [$lang, $text] = explode("\t", $line);
+        $start = microtime(true);
+        $result = $languageDetector->detect($text);
+        $duration += microtime(true) - $start;
+        if ($result->isValid && $result->language === $lang) {
+            $correct++;
+            $type = "correct";
+        } else {
+            $failed++;
+            $type = $result->isValid ? "wrong" : "miss";
+        }
+        echo "- - " . $type . ": " . ($result->language ?? "none") . ", expected " . $lang . PHP_EOL;
+        $perlang[$lang][$type] = ($perlang[$lang][$type] ?? 0) + 1;
+    }
+    fclose($fp);
+    $total = $correct + $failed;
+    $summary[] = basename($file) . ' - Correct ratio: ' . round(($correct / $total) * 100, 2) . '% Duration: ' . $duration;
+}
+echo PHP_EOL . PHP_EOL . "SUMMARY" . PHP_EOL;
+ksort($perlang);
+/** @noinspection ForgottenDebugOutputInspection */
+dump($perlang);
+echo implode(PHP_EOL, $summary) . PHP_EOL;

composer.json

@@ -20,16 +20,31 @@
   ],
   "autoload": {
     "psr-4": {
-      "Nitotm\\Eld\\": "src"
+      "Nitotm\\Eld\\": "src/Eld"
     }
   },
   "autoload-dev": {
     "psr-4": {
-      "Nitotm\\Eld\\Tests\\": "tests"
+      "Nitotm\\EldTests\\": "src/EldTests"
     }
   },
   "require": {
-    "php": "^7.3 || ^8.0",
+    "php": "^8.2",
     "ext-mbstring": "*"
+  },
+  "require-dev": {
+    "roave/security-advisories": "dev-latest",
+    "rector/rector": "^0.16.0",
+    "phpstan/phpstan-strict-rules": "^1.5",
+    "phpstan/extension-installer": "^1.3",
+    "phpmd/phpmd": "^2.13",
+    "vimeo/psalm": "^5.11",
+    "symfony/var-dumper": "*",
+    "squizlabs/php_codesniffer": "*"
+  },
+  "config": {
+    "allow-plugins": {
+      "phpstan/extension-installer": true
+    }
   }
-}
+}

data/language_corrections.php

@@ -0,0 +1,70 @@
+<?php
+// utf-8
+
+/** legal notice: https://github.com/nitotm/efficient-language-detector/blob/main/LEGAL.md */
+
+return
+    [
+        // ISO 639-1 codes
+        // warning: order does matter!
+        'am' => 0.0661,
+        'ar' => 0.0237,
+        'az' => 0.0269,
+        'be' => 0.0227,
+        'bg' => 0.0234,
+        'bn' => 0.1373,
+        'ca' => 0.0246,
+        'cs' => 0.0242,
+        'da' => 0.0277,
+        'de' => 0.0275,
+        'el' => 0.0369,
+        'en' => 0.0378,
+        'es' => 0.0252,
+        'et' => 0.0253,
+        'eu' => 0.0369,
+        'fa' => 0.0213,
+        'fi' => 0.026,
+        'fr' => 0.0253,
+        'gu' => 0.1197,
+        'he' => 0.0402,
+        'hi' => 0.0578,
+        'hr' => 0.0201,
+        'hu' => 0.0208,
+        'hy' => 0.0439,
+        'is' => 0.032,
+        'it' => 0.0251,
+        'ja' => 0.0375,
+        'ka' => 0.1383,
+        'kn' => 0.1305,
+        'ko' => 0.0222,
+        'ku' => 0.0256,
+        'lo' => 0.3488,
+        'lt' => 0.0246,
+        'lv' => 0.0264,
+        'ml' => 0.1322,
+        'mr' => 0.0571,
+        'ms' => 0.0251,
+        'nl' => 0.0342,
+        'no' => 0.0266,
+        'or' => 0.1269,
+        'pa' => 0.1338,
+        'pl' => 0.0275,
+        'pt' => 0.0252,
+        'ro' => 0.0247,
+        'ru' => 0.0184,
+        'sk' => 0.024,
+        'sl' => 0.0253,
+        'sq' => 0.0353,
+        'sr' => 0.0234,
+        'sv' => 0.033,
+        'ta' => 0.1513,
+        'te' => 0.1547,
+        'th' => 0.0882,
+        'tl' => 0.0368,
+        'tr' => 0.0258,
+        'uk' => 0.0206,
+        'ur' => 0.0282,
+        'vi' => 0.0467,
+        'yo' => 0.0329,
+        'zh' => 0.0152,
+    ];