tfrecord-512-gen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "compressed-paris",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-05-09T20:39:56.896502Z",
     "iopub.status.busy": "2021-05-09T20:39:56.895351Z",
     "iopub.status.idle": "2021-05-09T20:40:03.963816Z",
     "shell.execute_reply": "2021-05-09T20:40:03.963130Z"
    },
    "papermill": {
     "duration": 7.079469,
     "end_time": "2021-05-09T20:40:03.964006",
     "exception": false,
     "start_time": "2021-05-09T20:39:56.884537",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import PIL\n",
    "import gc\n",
    "from PIL import Image\n",
    "from sklearn .model_selection import StratifiedKFold, GroupKFold\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "metallic-hardware",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T20:40:03.986222Z",
     "iopub.status.busy": "2021-05-09T20:40:03.985505Z",
     "iopub.status.idle": "2021-05-09T20:40:03.987490Z",
     "shell.execute_reply": "2021-05-09T20:40:03.987928Z"
    },
    "papermill": {
     "duration": 0.017927,
     "end_time": "2021-05-09T20:40:03.988093",
     "exception": false,
     "start_time": "2021-05-09T20:40:03.970166",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def _bytes_feature(value):\n",
    "    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n",
    "    if isinstance(value, type(tf.constant(0))):\n",
    "        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n",
    "    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n",
    "\n",
    "def _float_feature(value):\n",
    "    \"\"\"Returns a float_list from a float / double.\"\"\"\n",
    "    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n",
    "\n",
    "def _list_float_feature(value):\n",
    "    \"\"\"Returns a float_list from a float / double.\"\"\"\n",
    "    return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n",
    "\n",
    "def _int64_feature(value):\n",
    "    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n",
    "    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n",
    "\n",
    "def _list_int64_feature(value):\n",
    "    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n",
    "    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "asian-faith",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T20:40:04.003162Z",
     "iopub.status.busy": "2021-05-09T20:40:04.001903Z",
     "iopub.status.idle": "2021-05-09T20:40:04.006165Z",
     "shell.execute_reply": "2021-05-09T20:40:04.005625Z"
    },
    "papermill": {
     "duration": 0.012514,
     "end_time": "2021-05-09T20:40:04.006314",
     "exception": false,
     "start_time": "2021-05-09T20:40:03.993800",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "RESIZE_WIDTH = 512\n",
    "RESIZE_HEIGHT = 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "familiar-fiction",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T20:40:04.027896Z",
     "iopub.status.busy": "2021-05-09T20:40:04.027254Z",
     "iopub.status.idle": "2021-05-09T20:48:11.609867Z",
     "shell.execute_reply": "2021-05-09T20:48:11.610643Z"
    },
    "papermill": {
     "duration": 487.599067,
     "end_time": "2021-05-09T20:48:11.611212",
     "exception": false,
     "start_time": "2021-05-09T20:40:04.012145",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34250 samples\n"
     ]
    }
   ],
   "source": [
    "df_train = pd.read_parquet(f'../input/shope-text-vectorizer-for-all-train/train.parquet', engine='pyarrow')\n",
    "\n",
    "samples = df_train\n",
    "n_samples = len(samples)\n",
    "print(f'{n_samples} samples')\n",
    "with tf.io.TFRecordWriter(f'/kaggle/working/train.tfrec') as writer:\n",
    "    for row in samples.itertuples():\n",
    "        image = f'../input/shopee-product-matching/train_images/{row.image}'\n",
    "        img = open(image, 'rb')\n",
    "        img_read = img.read()\n",
    "        image_decoded = tf.image.decode_jpeg(img_read, channels=3)\n",
    "        resized_img = tf.image.resize_with_pad(image_decoded,target_width=RESIZE_WIDTH,target_height=RESIZE_HEIGHT,method=tf.image.ResizeMethod.BILINEAR)\n",
    "        resized_img = tf.cast(resized_img,tf.uint8)\n",
    "        resized_img = tf.io.encode_jpeg(resized_img)\n",
    "\n",
    "        feature = {\n",
    "            'filename': _bytes_feature(tf.compat.as_bytes(row.image)),\n",
    "            'label': _int64_feature(row.label_group),\n",
    "            'title_vec': _list_float_feature(row.title_vec),\n",
    "            'phash_vec': _list_float_feature(row.phash_vec),\n",
    "            'image': _bytes_feature(resized_img),\n",
    "            'height' : _int64_feature(RESIZE_HEIGHT),\n",
    "            'width' : _int64_feature(RESIZE_WIDTH)\n",
    "        }\n",
    "\n",
    "        example = tf.train.Example(features=tf.train.Features(feature=feature))\n",
    "        writer.write(example.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "developmental-brooks",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T20:48:11.634213Z",
     "iopub.status.busy": "2021-05-09T20:48:11.633165Z",
     "iopub.status.idle": "2021-05-09T20:48:12.406938Z",
     "shell.execute_reply": "2021-05-09T20:48:12.405921Z"
    },
    "papermill": {
     "duration": 0.786783,
     "end_time": "2021-05-09T20:48:12.407163",
     "exception": false,
     "start_time": "2021-05-09T20:48:11.620380",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 2.3G\r\n",
      "drwxr-xr-x 2 root root 4.0K May  9 20:40 .\r\n",
      "drwxr-xr-x 6 root root 4.0K May  9 20:39 ..\r\n",
      "---------- 1 root root 7.1K May  9 20:48 __notebook__.ipynb\r\n",
      "-rw-r--r-- 1 root root 2.3G May  9 20:48 train.tfrec\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lah"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hundred-continuity",
   "metadata": {
    "papermill": {
     "duration": 0.008441,
     "end_time": "2021-05-09T20:48:12.424842",
     "exception": false,
     "start_time": "2021-05-09T20:48:12.416401",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 505.959029,
   "end_time": "2021-05-09T20:48:14.583020",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-05-09T20:39:48.623991",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}