make_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "3c895d16-ec05-4dd2-8cfc-415e0993caae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import os\n",
    "import zipfile\n",
    "import json\n",
    "\n",
    "flist = []\n",
    "items = []\n",
    "\n",
    "\n",
    "for item in glob('*'):\n",
    "    if os.path.isdir(item):\n",
    "        for item1 in glob(os.path.join(item,  '*', '라벨링*/TL2/*')):\n",
    "            with open(item1, 'r') as f:\n",
    "                items.append(json.loads(f.read()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "739fb58a-8176-40bc-b33a-d0f61a74acb0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "make_dict = []\n",
    "\n",
    "for item1 in items:\n",
    "\n",
    "    for item in item1['data']:\n",
    "        temp_dict = {}\n",
    "        temp_dict['text'] = '### source: ' + item['ko'] +'\\n\\n### ' + 'target: ' +  item['en']+ '<|endoftext|>'\n",
    "        make_dict.append(temp_dict)\n",
    "with open('KO_TO_EN.json', 'w') as f:\n",
    "    for entry in make_dict:\n",
    "        json.dump(entry, f)\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "bfd45160-5a27-48ce-914c-116982429f95",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset json (/home/work/.cache/huggingface/datasets/json/default-69f8a4538f09fd63/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['### source: 원하시는 색상을 회신해 주시면 바로 제작 들어가겠습니다.\\n'\n",
      " '\\n'\n",
      " '### target: If you reply to the color you want, we will start making it '\n",
      " 'right away.<|endoftext|>',\n",
      " '### source: 형님 제일 웃긴 그림이 뭔지 알아요.\\n'\n",
      " '\\n'\n",
      " '### target: You know what the funniest picture is.<|endoftext|>',\n",
      " '### source: >속옷을?\\n\\n### target: >Underwear?<|endoftext|>',\n",
      " '### source: 그래도 가격이 꽤 비싸니까 많이 살게요.\\n'\n",
      " '\\n'\n",
      " '### target: I wont buy a lot though since the price is still quite '\n",
      " 'high.<|endoftext|>',\n",
      " '### source: AAA님, 제가 회의에서 화를 냈던 점 정말 사과드리고 싶습니다.\\n'\n",
      " '\\n'\n",
      " '### target: Dear AAA, I really want to apologize for my anger at the '\n",
      " 'meeting.<|endoftext|>']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text'],\n",
       "    num_rows: 1200000\n",
       "})"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from pprint import pprint\n",
    "dataset = load_dataset('json', data_files='KO_TO_EN.json', split='train')\n",
    "pprint(dataset[:5]['text'])\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "0e45f5ba-a9b0-465b-ad4e-87ad15b87c06",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset json (/home/work/.cache/huggingface/datasets/json/default-20ccf50f4f2c70de/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"### source: I'm glad to hear that, and I hope you do consider doing business \"\n",
      " 'with us.\\n'\n",
      " '\\n'\n",
      " '### target: 그 말을 들으니 기쁘고, 저희와 거래하는 것을 고려해 주셨으면 합니다.<|endoftext|>',\n",
      " \"### source: I'm definitely thinking about it, but I have some queries to ask \"\n",
      " 'you.\\n'\n",
      " '\\n'\n",
      " '### target: 확실히 생각하고 있습니다만, 몇 가지 여쭤보고 싶은 게 있어요.<|endoftext|>',\n",
      " \"### source: In today's world, one in every five families has either a cat, \"\n",
      " 'dog, or both.\\n'\n",
      " '\\n'\n",
      " '### target: 오늘날 세계 5가구 중 1가구는 고양이나 개 또는 둘 다를 키우고 있습니다.<|endoftext|>',\n",
      " \"### source: When you tell them, we'll take care of their child.\\n\"\n",
      " '\\n'\n",
      " '### target: 그들에게 말하면, 저희가 그 아이들을 돌볼 것입니다.<|endoftext|>',\n",
      " '### source: OK, how about for swimming?\\n'\n",
      " '\\n'\n",
      " '### target: 좋아요, 수영은 어떤가요?<|endoftext|>']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text'],\n",
       "    num_rows: 1200307\n",
       "})"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = load_dataset('json', data_files='EN_TO_KO.json', split='train')\n",
    "pprint(dataset[:5]['text'])\n",
    "dataset"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deepspeed",
   "language": "python",
   "name": "deepspeed"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}