split-lang-benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Language Detection Package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "import langdetect\n",
    "import fast_langdetect\n",
    "from lingua import Language, LanguageDetectorBuilder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Text Split Package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wtpsplit import SaT, WtP\n",
    "sat = SaT(\"sat-1l-sm\")\n",
    "sat.half().to(\"cuda\")\n",
    "wtp = WtP(\"wtp-bert-mini\")\n",
    "import budoux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"你喜欢看アニメ吗\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 124.43it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['你', '喜欢看', 'アニメ', '吗']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wtp.split(text_or_texts=text, threshold=4e-5, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts_with_digit = [\n",
    "    \"你喜欢看アニメ吗？\",\n",
    "    \"衬衫的价格是9.15便士\",\n",
    "    \"衬衫的价格是233亿元\",\n",
    "    \"衬衫的价格是233亿元人民币\",\n",
    "]\n",
    "\n",
    "texts_zh_jp_ko_en = [\n",
    "    \"我是 VGroupChatBot，一个旨在支持多人通信的助手，通过可视化消息来帮助团队成员更好地交流。我可以帮助团队成员更好地整理和共享信息，特别是在讨论、会议和Brainstorming等情况下。你好我的名字是西野くまですmy name is bob很高兴认识你どうぞよろしくお願いいたします「こんにちは」是什么意思。\",\n",
    "    \"你好，我的名字是西野くまです。I am from Tokyo, 日本の首都。今天的天气非常好，sky is clear and sunny。おはようございます、皆さん！我们一起来学习吧。Learning languages can be fun and exciting。昨日はとても忙しかったので、今日は少しリラックスしたいです。Let's take a break and enjoy some coffee。中文、日本語、and English are three distinct languages, each with its own unique charm。希望我们能一起进步，一起成长。Let's keep studying and improving our language skills together. ありがとう！\",\n",
    "    \"你好，今日はどこへ行きますか？\",\n",
    "    \"你好今日はどこへ行きますか？\",\n",
    "    \"我的名字是田中さんです。\",\n",
    "    \"我喜欢吃寿司和拉面おいしいです。\",\n",
    "    \"今天の天気はとてもいいですね。\",\n",
    "    \"我在学习日本語少し難しいです。\",\n",
    "    \"日语真是おもしろい啊\",\n",
    "    \"你喜欢看アニメ吗？\",\n",
    "    \"我想去日本旅行、特に京都に行きたいです。\",\n",
    "    \"昨天見た映画はとても感動的でした。我朋友是日本人彼はとても優しいです。\",\n",
    "    \"我们一起去カラオケ吧、楽しそうです。\",\n",
    "    \"我的家在北京、でも、仕事で東京に住んでいます。\",\n",
    "    \"我在学做日本料理、日本料理を作るのを習っています。\",\n",
    "    \"你会说几种语言、何ヶ国語話せますか？\",\n",
    "    \"我昨天看了一本书、その本はとても面白かったです。\",\n",
    "    \"你最近好吗、最近どうですか？\",\n",
    "    \"我在学做日本料理와 한국 요리、日本料理を作るのを習っています。\",\n",
    "    \"你会说几种语言、何ヶ国語話せますか？몇 개 언어를 할 수 있어요？\",\n",
    "    \"我昨天看了一本书、その本はとても面白かったです。어제 책을 읽었는데, 정말 재미있었어요。\",\n",
    "    \"我们一起去逛街와 쇼핑、買い物に行きましょう。쇼핑하러 가요。\",\n",
    "    \"你最近好吗、最近どうですか？요즘 어떻게 지내요？\",\n",
    "]\n",
    "\n",
    "texts_zh_jp = [\n",
    "    \"你好今日はどこへ行きますか\",\n",
    "    \"我的名字是田中さんです\",\n",
    "    \"我喜欢吃寿司和拉面おいしいです\",\n",
    "    \"今天の天気はとてもいいですね\",\n",
    "    \"我在学习日本語少し難しいです\",\n",
    "    \"日语真是おもしろい啊\",\n",
    "    \"你喜欢看アニメ吗\",\n",
    "    \"我想去日本旅行特に京都に行きたいです\",\n",
    "    \"昨天見た映画はとても感動的でした\",\n",
    "    \"我朋友是日本人彼はとても優しいです\",\n",
    "    \"我们一起去カラオケ吧\",\n",
    "    \"我的家在北京でも仕事で東京に住んでいます\",\n",
    "    \"我的名字是西野くまです\",\n",
    "    \"我的名字是西野くまですよろしくお願いいたします\",\n",
    "    \"好吃美味しい上手い\",\n",
    "    \"我给你送的手紙\",\n",
    "    \"真是面白い\",\n",
    "    \"春の花香り\",\n",
    "    \"何ヶ国語話せますか\",\n",
    "    \"今晚どこに行きます\"\n",
    "]\n",
    "\n",
    "texts_de_fr_en = [\n",
    "    \"Ich liebe Paris, c'est une belle ville, and the food is amazing!\",\n",
    "    \"Berlin ist wunderbar, je veux y retourner, and explore more.\",\n",
    "    \"Bonjour, wie geht's dir today?\",\n",
    "    \"Die Musik hier ist fantastisch, la musique est superbe, and I enjoy it a lot.\",\n",
    "    \"Guten Morgen, je t'aime, have a great day!\",\n",
    "    \"Das Wetter ist heute schön, il fait beau aujourd'hui, and it's perfect for a walk.\",\n",
    "    \"Ich mag dieses Buch, ce livre est intéressant, and it has a great story.\",\n",
    "    \"Vielen Dank, merci beaucoup, for your help.\",\n",
    "    \"Wir reisen nach Deutschland, nous voyageons en Allemagne, and we are excited.\",\n",
    "    \"Ich bin müde, je suis fatigué, and I need some rest.\",\n",
    "    \"Ich liebe Paris c'est une belle ville and the food is amazing!\",\n",
    "    \"Berlin ist wunderbar je veux y retourner and explore more.\",\n",
    "    \"Bonjour wie geht's dir today?\",\n",
    "    \"Die Musik hier ist fantastisch la musique est superbe and I enjoy it a lot.\",\n",
    "    \"Guten Morgen je t'aime have a great day!\",\n",
    "    \"Das Wetter ist heute schön il fait beau aujourd'hui and it's perfect for a walk.\",\n",
    "    \"Ich mag dieses Buch ce livre est intéressant and it has a great story.\",\n",
    "    \"Vielen Dank merci beaucoup for your help.\",\n",
    "    \"Wir reisen nach Deutschland nous voyageons en Allemagne and we are excited.\",\n",
    "    \"Ich bin müde je suis fatigué and I need some rest.\",\n",
    "]\n",
    "\n",
    "texts = texts_zh_jp_ko_en + texts_de_fr_en + texts_with_digit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Rule-Based judge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "chinese_char_pattern = re.compile(r\"[\\u4e00-\\u9fff]\")\n",
    "hangul_pattern = re.compile(r\"[\\uac00-\\ud7af]\")\n",
    "hiragana_pattern = re.compile(r\"[\\u3040-\\u309f]\")\n",
    "katakana_pattern = re.compile(r\"[\\u30a0-\\u30ff]\")\n",
    "\n",
    "\n",
    "def contains_chinese_char(text: str):\n",
    "    return bool(chinese_char_pattern.search(text))\n",
    "\n",
    "\n",
    "def _contains_hiragana(text: str):\n",
    "    return bool(hiragana_pattern.search(text))\n",
    "\n",
    "\n",
    "def _contains_katakana(text: str):\n",
    "    return bool(katakana_pattern.search(text))\n",
    "\n",
    "\n",
    "def contains_ja(text):\n",
    "    if (\n",
    "        _contains_hiragana(text)\n",
    "        or _contains_katakana(text)\n",
    "    ):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try different split logic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jp_budoux_parser----------\n",
      "['你好', '今日は', 'どこへ', '行きますか']\n",
      "['我的名字是田中さんです']\n",
      "['我喜欢吃寿司和拉面', 'おいしいです']\n",
      "['今天の', '天気は', 'とても', 'いいですね']\n",
      "['我在学习日本語少し', '難しいです']\n",
      "['日语真是おもしろい', '啊']\n",
      "['你喜欢看アニメ吗']\n",
      "['我想去日本旅行特に', '京都に', '行きたいです']\n",
      "['昨天見た', '映画は', 'とても', '感動的でした']\n",
      "['我朋友是日本人', '彼は', 'とても', '優しいです']\n",
      "['我们一起去カラオケ吧']\n",
      "['我的家在北京でも', '仕事で', '東京に', '住んでいます']\n",
      "['我的名字是西野くまです']\n",
      "['我的名字是西野くまですよろしく', 'お願い', 'いたします']\n",
      "['好吃美味しい', '上手い']\n",
      "['我给你送的手紙']\n",
      "['真是面白い']\n",
      "['春の', '花香り']\n",
      "['何ヶ国語話せますか']\n",
      "['今晚どこに', '行きます']\n",
      "----------jp_budoux_parser\n",
      "[['你好', '今日は', 'どこへ', '行きますか'], ['我的名字是田中さんです'], ['我喜欢吃寿司和拉面', 'おいしいです'], ['今天の', '天気は', 'とても', 'いいですね'], ['我在学习日本語少し', '難しいです'], ['日语真是おもしろい', '啊'], ['你喜欢看アニメ吗'], ['我想去日本旅行特に', '京都に', '行きたいです'], ['昨天見た', '映画は', 'とても', '感動的でした'], ['我朋友是日本人', '彼は', 'とても', '優しいです'], ['我们一起去カラオケ吧'], ['我的家在北京でも', '仕事で', '東京に', '住んでいます'], ['我的名字是西野くまです'], ['我的名字是西野くまですよろしく', 'お願い', 'いたします'], ['好吃美味しい', '上手い'], ['我给你送的手紙'], ['真是面白い'], ['春の', '花香り'], ['何ヶ国語話せますか'], ['今晚どこに', '行きます']]\n",
      "----------jp_budoux_parser+zh_budoux_parser\n",
      "['你', '好', '今日', 'は', 'どこへ', '行きますか']\n",
      "['我', '的', '名字', '是', '田', '中', 'さんです']\n",
      "['我', '喜欢', '吃', '寿司', '和', '拉面', 'おいしいです']\n",
      "['今天', 'の', '天', '気は', 'とても', 'いいですね']\n",
      "['我', '在', '学习', '日本', '語少', 'し', '難しいです']\n",
      "['日', '语真', '是', 'おも', 'しろい', '啊']\n",
      "['你', '喜欢', '看', 'アニメ', '吗']\n",
      "['我', '想', '去', '日本', '旅行', '特に', '京', '都', 'に', '行きたいです']\n",
      "['昨天', '見た', '映', '画', 'は', 'とても', '感動', '的', 'でし', 'た']\n",
      "['我', '朋友', '是', '日本', '人', '彼は', 'とても', '優しいです']\n",
      "['我们', '一起', '去', 'カラオケ', '吧']\n",
      "['我', '的', '家', '在', '北京', 'でも', '仕事で', '東京', 'に', '住ん', 'でいます']\n",
      "['我', '的', '名字', '是', '西野', 'くまです']\n",
      "['我', '的', '名字', '是', '西野', 'くまですよろしく', 'お願い', 'いたします']\n",
      "['好', '吃', '美味', 'しい', '上', '手い']\n",
      "['我', '给', '你', '送', '的', '手紙']\n",
      "['真', '是', '面白い']\n",
      "['春の', '花香り']\n",
      "['何', 'ヶ国', '語話', 'せますか']\n",
      "['今晚どこに', '行きます']\n",
      "[['你', '好', '今日', 'は', 'どこへ', '行きますか'], ['我', '的', '名字', '是', '田', '中', 'さんです'], ['我', '喜欢', '吃', '寿司', '和', '拉面', 'おいしいです'], ['今天', 'の', '天', '気は', 'とても', 'いいですね'], ['我', '在', '学习', '日本', '語少', 'し', '難しいです'], ['日', '语真', '是', 'おも', 'しろい', '啊'], ['你', '喜欢', '看', 'アニメ', '吗'], ['我', '想', '去', '日本', '旅行', '特に', '京', '都', 'に', '行きたいです'], ['昨天', '見た', '映', '画', 'は', 'とても', '感動', '的', 'でし', 'た'], ['我', '朋友', '是', '日本', '人', '彼は', 'とても', '優しいです'], ['我们', '一起', '去', 'カラオケ', '吧'], ['我', '的', '家', '在', '北京', 'でも', '仕事で', '東京', 'に', '住ん', 'でいます'], ['我', '的', '名字', '是', '西野', 'くまです'], ['我', '的', '名字', '是', '西野', 'くまですよろしく', 'お願い', 'いたします'], ['好', '吃', '美味', 'しい', '上', '手い'], ['我', '给', '你', '送', '的', '手紙'], ['真', '是', '面白い'], ['春の', '花香り'], ['何', 'ヶ国', '語話', 'せますか'], ['今晚どこに', '行きます']]\n",
      "----------jp_budoux_parser+zh_budoux_parser+combine single to left\n",
      "['你好', '今日', 'はどこへ行きますか']\n",
      "['我的', '名字是田中', 'さんです']\n",
      "['我喜欢吃', '寿司和', '拉面', 'おいしいです']\n",
      "['今天', 'の', '天', '気はとてもいいですね']\n",
      "['我在', '学习', '日本', '語少', 'し難しいです']\n",
      "['日语真是', 'おもしろい', '啊']\n",
      "['你喜欢看', 'アニメ', '吗']\n",
      "['我想去', '日本', '旅行', '特に', '京都', 'に行きたいです']\n",
      "['昨天', '見た', '映画', 'はとても', '感動的', 'でした']\n",
      "['我朋友是', '日本人', '彼はとても優しいです']\n",
      "['我们', '一起去', 'カラオケ', '吧']\n",
      "['我的家在', '北京', 'でも仕事で', '東京', 'に住んでいます']\n",
      "['我的', '名字是', '西野', 'くまです']\n",
      "['我的', '名字是', '西野', 'くまですよろしくお願いいたします']\n",
      "['好吃', '美味', 'しい', '上', '手い']\n",
      "['我给你送的', '手紙']\n",
      "['真是', '面白い']\n",
      "['春の花香り']\n",
      "['何ヶ国', '語話', 'せますか']\n",
      "['今晚どこに行きます']\n",
      "[['你好', '今日', 'はどこへ行きますか'], ['我的', '名字是田中', 'さんです'], ['我喜欢吃', '寿司和', '拉面', 'おいしいです'], ['今天', 'の', '天', '気はとてもいいですね'], ['我在', '学习', '日本', '語少', 'し難しいです'], ['日语真是', 'おもしろい', '啊'], ['你喜欢看', 'アニメ', '吗'], ['我想去', '日本', '旅行', '特に', '京都', 'に行きたいです'], ['昨天', '見た', '映画', 'はとても', '感動的', 'でした'], ['我朋友是', '日本人', '彼はとても優しいです'], ['我们', '一起去', 'カラオケ', '吧'], ['我的家在', '北京', 'でも仕事で', '東京', 'に住んでいます'], ['我的', '名字是', '西野', 'くまです'], ['我的', '名字是', '西野', 'くまですよろしくお願いいたします'], ['好吃', '美味', 'しい', '上', '手い'], ['我给你送的', '手紙'], ['真是', '面白い'], ['春の花香り'], ['何ヶ国', '語話', 'せますか'], ['今晚どこに行きます']]\n"
     ]
    }
   ],
   "source": [
    "from typing import List\n",
    "\n",
    "\n",
    "zh_budoux_parser = budoux.load_default_simplified_chinese_parser()\n",
    "zh_tc_budoux_parser = budoux.load_default_traditional_chinese_parser()\n",
    "jp_budoux_parser = budoux.load_default_japanese_parser()\n",
    "\n",
    "# print(\"zh_budoux_parser----------\")\n",
    "# for text in texts_zh_jp:\n",
    "#     print(zh_budoux_parser.parse(text))\n",
    "    \n",
    "# print(\"zh_tc_budoux_parser----------\")\n",
    "# for text in texts_zh_jp:\n",
    "#     print(zh_tc_budoux_parser.parse(text))\n",
    "    \n",
    "print(\"jp_budoux_parser----------\")\n",
    "for text in texts_zh_jp:\n",
    "    print(jp_budoux_parser.parse(text))\n",
    "\n",
    "\n",
    "print(\"----------jp_budoux_parser\")\n",
    "\n",
    "splitted_texts_jp = []\n",
    "for text in texts_zh_jp:\n",
    "    jp_split_text = jp_budoux_parser.parse(text)\n",
    "    splitted_texts_jp.append(jp_split_text)\n",
    "print(splitted_texts_jp)\n",
    "print(\"----------jp_budoux_parser+zh_budoux_parser\")\n",
    "\n",
    "splitted_texts_zh_jp = []\n",
    "for substrings in splitted_texts_jp:\n",
    "    words = []\n",
    "    for substring in substrings:\n",
    "        words.extend(zh_budoux_parser.parse(substring))\n",
    "    print(words)\n",
    "    splitted_texts_zh_jp.append(words)\n",
    "print(splitted_texts_zh_jp)\n",
    "\n",
    "print(\"----------jp_budoux_parser+zh_budoux_parser+combine single to left\")\n",
    "pre_split_texts:List[List[str]] = []\n",
    "for words in splitted_texts_zh_jp:\n",
    "    new_words = [words[0]]\n",
    "    for sub_text in words[1:]:\n",
    "        is_left_ja = contains_ja(new_words[-1])\n",
    "        is_cur_ja = contains_ja(sub_text)\n",
    "        is_both_same_lang = is_left_ja == is_cur_ja\n",
    "        is_both_ja = is_left_ja == True and is_both_same_lang\n",
    "        is_both_zh = is_left_ja == False and is_both_same_lang\n",
    "        if is_both_ja: # both substring is katakana or hiragana, then concat\n",
    "            new_words[-1] += sub_text\n",
    "        elif is_both_zh and len(sub_text) == 1:\n",
    "            # NOTE: both substring is full Chinese character, and current one is only one character\n",
    "            # NOTE: 90% is because we first use ja_parser then zh_parser (from `budoux`)\n",
    "            # NOTE: Since kanji in Japanese usually not appear by them self, Single character is CAUSED BY zh_parser \n",
    "            # NOTE: So we let single character concat together, if both substring did not contain kana\n",
    "            new_words[-1] += sub_text\n",
    "        else:\n",
    "            new_words.append(sub_text)\n",
    "    \n",
    "    if len(new_words) >= 2 and len(new_words[0]) == 1:\n",
    "        new_words[1] = new_words[0] + new_words[1]\n",
    "        new_words = new_words[1:]\n",
    "    pre_split_texts.append(new_words)   \n",
    "    print(new_words)     \n",
    "print(pre_split_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lingua import Language, LanguageDetectorBuilder\n",
    "all_detector = (\n",
    "    LanguageDetectorBuilder.from_all_languages()\n",
    "    .with_preloaded_language_models()\n",
    "    .build()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zh:你好|ja:今日|ja:はどこへ行きますか|\n",
      "zh:我的|zh:名字是田中|ja:さんです|\n",
      "zh:我喜欢吃|ja:寿司和|zh:拉面|ja:おいしいです|\n",
      "zh:今天|ja:の|zh:天|ja:気はとてもいいですね|\n",
      "zh:我在|zh:学习|ja:日本|zh:語少|ja:し難しいです|\n",
      "zh:日语真是|ja:おもしろい|zh:啊|\n",
      "zh:你喜欢看|ja:アニメ|zh:吗|\n",
      "zh:我想去|ja:日本|ja:旅行|ja:特に|ja:京都|ja:に行きたいです|\n",
      "zh:昨天|ja:見た|ja:映画|ja:はとても|zh:感動的|ja:でした|\n",
      "zh:我朋友是|zh:日本人|ja:彼はとても優しいです|\n",
      "zh:我们|zh:一起去|ja:カラオケ|zh:吧|\n",
      "zh:我的家在|ja:北京|ja:でも仕事で|ja:東京|ja:に住んでいます|\n",
      "zh:我的|zh:名字是|ja:西野|ja:くまです|\n",
      "zh:我的|zh:名字是|ja:西野|ja:くまですよろしくお願いいたします|\n",
      "zh:好吃|zh:美味|ja:しい|ja:上|ja:手い|\n",
      "zh:我给你送的|ja:手紙|\n",
      "zh:真是|ja:面白い|\n",
      "ja:春の花香り|\n",
      "zh:何ヶ国|ja:語話|ja:せますか|\n",
      "ja:今晚どこに行きます|\n"
     ]
    }
   ],
   "source": [
    "from langdetect.lang_detect_exception import LangDetectException\n",
    "\n",
    "def lingua_lang_detect_all(text: str) -> str:\n",
    "    language: Language | None = all_detector.detect_language_of(text=text)\n",
    "    if language is None:\n",
    "        return \"x\"\n",
    "    return language.iso_code_639_1.name.lower()\n",
    "\n",
    "def fast_lang_detect(text: str) -> str:\n",
    "    result = str(fast_langdetect.detect(text, low_memory=False)[\"lang\"])\n",
    "    result = result.lower()\n",
    "    return result\n",
    "\n",
    "def lang_detect(text: str) -> str:\n",
    "    try:\n",
    "        result = str(langdetect.detect(text))\n",
    "        result = result.lower()\n",
    "        return result\n",
    "    except LangDetectException as e:\n",
    "        return \"zh\"\n",
    "    except Exception as e:\n",
    "        pass\n",
    "    return \"x\"\n",
    "\n",
    "\n",
    "for substrings in pre_split_texts:\n",
    "    for substring in substrings:\n",
    "        # lang = lingua_lang_detect_all(substring)\n",
    "        lang = fast_lang_detect(substring)\n",
    "        \n",
    "        print(f\"{lang}:{substring}\",end='|')\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Difference between split by `  ` (space) and by `wtpsplit`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\admin\\.conda\\envs\\melotts\\lib\\site-packages\\wtpsplit\\__init__.py:45: DeprecationWarning: You are using WtP, the old sentence segmentation model. It is highly encouraged to use SaT instead due to strongly improved performance and efficiency. See https://github.com/segment-any-text/wtpsplit for more info. To ignore this warning, set ignore_legacy_warning=True.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Ich ', 'bin müde ', 'je suis fatigué ', 'and ', 'I ', 'need some rest']\n",
      "\n",
      "0.8507270812988281\n",
      "['Ich', 'bin', 'müde', 'je', 'suis', 'fatigué', 'and', 'I', 'need', 'some', 'rest']\n",
      "Ich: ['de:1.0']\n",
      "\n",
      "bin: ['en:0.394521027803421', 'id:0.21292246878147125', 'tr:0.15927168726921082', 'ms:0.09677533060312271', 'eo:0.030221346765756607', 'jv:0.023466553539037704', 'sq:0.013604077510535717', 'sv:0.012493844144046307']\n",
      "\n",
      "müde: ['de:0.9626638293266296', 'tr:0.026752416044473648']\n",
      "\n",
      "je: ['sr:0.8350609540939331', 'fr:0.15938909351825714']\n",
      "\n",
      "suis: ['fr:0.9970543384552002']\n",
      "\n",
      "fatigué: ['fr:0.7745229601860046', 'es:0.14570455253124237', 'nl:0.03426196426153183', 'de:0.014598727226257324']\n",
      "\n",
      "and: ['en:0.9983668923377991']\n",
      "\n",
      "I: ['en:0.9979130625724792']\n",
      "\n",
      "need: ['en:0.9885499477386475']\n",
      "\n",
      "some: ['en:0.993599534034729']\n",
      "\n",
      "rest: ['en:0.46527764201164246', 'nl:0.05554379150271416', 'ja:0.039999693632125854', 'pl:0.033725958317518234', 'ar:0.028888104483485222', 'sv:0.028634123504161835', 'no:0.024481678381562233', 'id:0.022417806088924408', 'tr:0.02039233408868313', 'zh:0.014322302304208279']\n",
      "\n",
      "\n",
      "0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\admin\\.conda\\envs\\melotts\\lib\\site-packages\\sklearn\\base.py:376: InconsistentVersionWarning: Trying to unpickle estimator LogisticRegression from version 1.2.2 when using version 1.5.0. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from wtpsplit import WtP\n",
    "from time import time\n",
    "from datetime import datetime\n",
    "# text = \"Vielen Dank merci beaucoup for your help.\"\n",
    "text = \"Ich bin müde je suis fatigué and I need some rest\"\n",
    "# text = \"日语使用者应超过一亿三千万人\"\n",
    "# text = \"我是 VGroupChatBot，一个旨在支持多人通信的助手，通过可视化消息来帮助团队成员更好地交流。我可以帮助团队成员更好地整理和共享信息，特别是在讨论、会议和Brainstorming等情况下。你好我的名字是西野くまですmy name is bob很高兴认识你どうぞよろしくお願いいたします「こんにちは」是什么意思。我的名字是西野くまです。I am from Tokyo, 日本の首都。今天的天气非常好\"\n",
    "time1 = datetime.now().timestamp()\n",
    "wtp = WtP('wtp-bert-mini')\n",
    "substrings = wtp.split(text_or_texts=text, threshold=2e-3)\n",
    "print(substrings)\n",
    "for substring in substrings:\n",
    "    # lang = lingua_lang_detect_all(substring)\n",
    "    lang = lingua_lang_detect_all(substring)\n",
    "    # print(f\"{lang}:{substring}\",end='|')\n",
    "print()\n",
    "time2 = datetime.now().timestamp()\n",
    "\n",
    "print(time2 - time1)\n",
    "\n",
    "from split_lang import LangSplitter\n",
    "lang_splitter = LangSplitter()\n",
    "substrings = lang_splitter._parse_without_zh_ja(text=text)\n",
    "substrings = text.split(' ')\n",
    "# substrings = lang_splitter._parse_zh_ja(text=text)\n",
    "# substrings = lang_splitter._parse_without_zh_ja(text=text)\n",
    "print(substrings)\n",
    "for substring in substrings:\n",
    "    # lang = lingua_lang_detect_all(substring)\n",
    "    lang = fast_langdetect.detect_multilingual(substring, low_memory=False, k=10, threshold=0.01)\n",
    "    lang_list = [f\"{item['lang']}:{item['score']}\" for item in lang]\n",
    "    print(f\"{substring}: {lang_list}\",end='\\n\\n')\n",
    "time3 = datetime.now().timestamp()\n",
    "print()\n",
    "print(time3 - time2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.66e-05\n",
      "0.00126\n",
      "0.013174603174603174\n"
     ]
    }
   ],
   "source": [
    "from wordfreq import word_frequency\n",
    "ja_freq = word_frequency('bin ', 'en')\n",
    "zh_freq = word_frequency('bin ', 'de')\n",
    "print(ja_freq)\n",
    "print(zh_freq)\n",
    "print(ja_freq / zh_freq)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "melotts",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}