data/create_dvm_dataset.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from os.path import join\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "from torchvision.io import read_image\n",
    "from matplotlib import pyplot as plt\n",
    "import torchvision\n",
    "import torch\n",
    "import random\n",
    "import numpy as np\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "\n",
    "pd.options.display.max_columns = 700\n",
    "\n",
    "BASE = ''\n",
    "TABLES = join(BASE, 'tables_V2.0')\n",
    "FEATURES = join(BASE, 'features')\n",
    "\n",
    "front_view_only = False\n",
    "\n",
    "from typing import List\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cluster import k_means, SpectralClustering\n",
    "import multiprocessing as mp\n",
    "\n",
    "ANALYSIS = join(BASE, 'analysis')\n",
    "\n",
    "def conf_matrix_from_matrices(mat_gt, mat_pred):\n",
    "  overlap_and = (mat_pred & mat_gt)\n",
    "  tp = overlap_and.sum()\n",
    "  fp = mat_pred.sum()-overlap_and.sum()\n",
    "  fn = mat_gt.sum()-overlap_and.sum()\n",
    "  tn = mat_gt.shape[0]**2-(tp+fp+fn)\n",
    "  return tp, fp, fn, tn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_or_save(obj, path, index=None, header=None):\n",
    "  if isinstance(obj, pd.DataFrame):\n",
    "    if index is None or header is None:\n",
    "      raise ValueError('Index and header must be specified for saving a dataframe')\n",
    "    if os.path.exists(path):\n",
    "      if not header:\n",
    "        saved_df = pd.read_csv(path,header=None)\n",
    "      else:\n",
    "        saved_df = pd.read_csv(path)\n",
    "      naked_df = saved_df.reset_index(drop=True)\n",
    "      naked_df.columns = range(naked_df.shape[1])\n",
    "      naked_obj = obj.reset_index(drop=not index)\n",
    "      naked_obj.columns = range(naked_obj.shape[1])\n",
    "      if naked_df.round(6).equals(naked_obj.round(6)):\n",
    "        return\n",
    "      else:\n",
    "        diff = (naked_df.round(6) == naked_obj.round(6))\n",
    "        diff[naked_df.isnull()] = naked_df.isnull() & naked_obj.isnull()\n",
    "        assert diff.all().all(), \"Dataframe is not the same as saved dataframe\"\n",
    "    else:\n",
    "      obj.to_csv(path, index=index, header=header)\n",
    "  else:\n",
    "    if os.path.exists(path):\n",
    "      saved_obj = torch.load(path)\n",
    "      if isinstance(obj, list):\n",
    "        for i in range(len(obj)):\n",
    "          check_array_equality(obj[i], saved_obj[i])\n",
    "      else:\n",
    "        check_array_equality(obj, saved_obj)\n",
    "    else:\n",
    "      print(f'Saving to {path}')\n",
    "      torch.save(obj, path)\n",
    "\n",
    "\n",
    "def check_array_equality(ob1, ob2):\n",
    "  if torch.is_tensor(ob1) or isinstance(ob1, np.ndarray):\n",
    "    assert (ob2 == ob1).all()\n",
    "  else:\n",
    "    assert ob2 == ob1"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Tabular Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_data = pd.read_csv(join(TABLES, 'Ad_table.csv'))\n",
    "ad_data.rename(columns={' Genmodel': 'Genmodel', ' Genmodel_ID': 'Genmodel_ID'}, inplace=True)\n",
    "\n",
    "basic_data = pd.read_csv(join(TABLES, 'Basic_table.csv'))\n",
    "\n",
    "image_data = pd.read_csv(join(TABLES, 'Image_table.csv'))\n",
    "image_data.rename(columns={' Image_ID': 'Image_ID', ' Image_name': 'Image_name', ' Predicted_viewpoint':'Predicted_viewpoint', ' Quality_check':'Quality_check'}, inplace=True)\n",
    "\n",
    "price_data = pd.read_csv(join(TABLES, 'Price_table.csv'))\n",
    "price_data.rename(columns={' Genmodel': 'Genmodel', ' Genmodel_ID': 'Genmodel_ID', ' Year': 'Year', ' Entry_price': 'Entry_price'}, inplace=True)\n",
    "\n",
    "sales_data = pd.read_csv(join(TABLES, 'Sales_table.csv'))\n",
    "sales_data.rename(columns={'Genmodel ': 'Genmodel', 'Genmodel_ID ': 'Genmodel_ID'}, inplace=True)\n",
    "\n",
    "trim_data = pd.read_csv(join(TABLES, 'Trim_table.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parser_adv_id(x):\n",
    "  split = x[\"Image_ID\"].split('$$')\n",
    "  return f\"{split[0]}$${split[1]}\"\n",
    "\n",
    "image_data[\"Adv_ID\"] = image_data.apply(lambda x: parser_adv_id(x), axis=1)\n",
    "if front_view_only:\n",
    "  image_data = image_data[(image_data[\"Quality_check\"]==\"P\")&(image_data[\"Predicted_viewpoint\"]==0)]\n",
    "image_data.drop_duplicates(subset=['Adv_ID'], inplace=True)\n",
    "image_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(ad_data))\n",
    "feature_df = ad_data.merge(price_data[['Genmodel_ID', 'Entry_price', 'Year']], left_on=['Genmodel_ID','Reg_year'], right_on=['Genmodel_ID','Year'])\n",
    "print(len(feature_df))\n",
    "feature_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = feature_df.merge(image_data[['Adv_ID', 'Image_name', 'Predicted_viewpoint']], left_on=['Adv_ID'], right_on=['Adv_ID'])\n",
    "assert data_df[\"Adv_ID\"].is_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_engine_size(x):\n",
    "  return float(x['Engin_size'][:-1])\n",
    "\n",
    "data_df.dropna(inplace=True)\n",
    "data_df['Engine_size'] = data_df.apply(lambda x: extract_engine_size(x), axis=1)\n",
    "data_df.drop(columns=['Engin_size'], inplace=True)\n",
    "data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_df = data_df.loc[:,'Adv_ID']\n",
    "image_name_df = data_df.loc[:,'Image_name']\n",
    "viewpoint_df = data_df.loc[:,'Predicted_viewpoint']\n",
    "\n",
    "continuous_df = data_df.loc[:,(\n",
    "  'Adv_year',\n",
    "  'Adv_month',\n",
    "  'Reg_year',\n",
    "  'Runned_Miles',\n",
    "  'Price',\n",
    "  'Seat_num',\n",
    "  'Door_num',\n",
    "  'Entry_price', \n",
    "  'Engine_size'\n",
    "  )]\n",
    "\n",
    "categorical_ids = ['Color',\n",
    "  'Bodytype',\n",
    "  'Gearbox',\n",
    "  'Fuel_type',\n",
    "  'Genmodel_ID']\n",
    "\n",
    "\n",
    "\n",
    "categorical_df = data_df.loc[:,categorical_ids]\n",
    "\n",
    "continuous_df['Runned_Miles'] = pd.to_numeric(continuous_df['Runned_Miles'], errors='coerce')\n",
    "continuous_df['Price'] = pd.to_numeric(continuous_df['Price'], errors='coerce')\n",
    "\n",
    "# normalize\n",
    "continuous_df=(continuous_df-continuous_df.mean())/continuous_df.std()\n",
    "\n",
    "categorical_df['Color'] = categorical_df['Color'].astype('category')\n",
    "categorical_df['Bodytype'] = categorical_df['Bodytype'].astype('category')\n",
    "categorical_df['Gearbox'] = categorical_df['Gearbox'].astype('category')\n",
    "categorical_df['Fuel_type'] = categorical_df['Fuel_type'].astype('category')\n",
    "categorical_df['Genmodel_ID'] = categorical_df['Genmodel_ID'].astype('category')\n",
    "\n",
    "cat_columns = categorical_df.select_dtypes(['category']).columns\n",
    "\n",
    "categorical_df[cat_columns] = categorical_df[cat_columns].apply(lambda x: x.cat.codes)\n",
    "\n",
    "data_df = pd.concat([id_df, continuous_df, categorical_df, image_name_df, viewpoint_df], axis=1)\n",
    "data_df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "minimum_population = 100\n",
    "values = (data_df.value_counts(subset=['Genmodel_ID'])>=minimum_population).values\n",
    "codes = (data_df.value_counts(subset=['Genmodel_ID'])>=minimum_population).index\n",
    "populated_codes = []\n",
    "for i, v in enumerate(values):\n",
    "  if v:\n",
    "    populated_codes.append(int(codes[i][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(populated_codes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = data_df[data_df['Genmodel_ID'].isin(populated_codes)]\n",
    "map = {}\n",
    "for i,l in enumerate(data_df['Genmodel_ID'].unique()):\n",
    "  map[l] = i\n",
    "data_df['Genmodel_ID'] = data_df['Genmodel_ID'].map(map)\n",
    "data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_indices = []\n",
    "for indx, row in data_df.iterrows():\n",
    "    im_name = row['Image_name']\n",
    "    split = im_name.split('$$')\n",
    "    path = join(BASE, 'resized_DVM', split[0], split[1], split[2], split[3], im_name)\n",
    "    if not os.path.exists(path):\n",
    "        bad_indices.append(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ids = list(data_df['Adv_ID'])\n",
    "addendum = '_all_views'\n",
    "non_feature_columns = ['Adv_ID', 'Image_name', 'Predicted_viewpoint', 'Genmodel_ID']\n",
    "if front_view_only:\n",
    "  train_set_ids, test_ids = train_test_split(_ids, test_size=0.1, random_state=2022)\n",
    "  train_ids, val_ids = train_test_split(train_set_ids, test_size=0.2, random_state=2022)\n",
    "  \n",
    "  bad_indices_train = torch.load(join(FEATURES, f'bad_indices_train{addendum}.pt'))\n",
    "  bad_indices_val = torch.load(join(FEATURES, f'bad_indices_val{addendum}.pt'))\n",
    "\n",
    "  print(f'Val length before {len(val_ids)}')\n",
    "  for _id in bad_indices_val:\n",
    "      val_ids.remove(_id)\n",
    "  print(f'Val length after {len(val_ids)}')\n",
    "\n",
    "  print(f'Train length before {len(train_ids)}')\n",
    "  for _id in bad_indices_train:\n",
    "      train_ids.remove(_id)\n",
    "  print(f'Train length after {len(train_ids)}')\n",
    "else:\n",
    "  addendum = '_all_views'\n",
    "  train_set_ids, test_ids = train_test_split(_ids, test_size=0.5, random_state=2022, stratify=data_df['Genmodel_ID'])\n",
    "  train_ids, val_ids = train_test_split(train_set_ids, test_size=0.2, random_state=2022, stratify=data_df[data_df['Adv_ID'].isin(train_set_ids)]['Genmodel_ID'])\n",
    "\n",
    "check_or_save(train_ids, join(FEATURES, f'train_ids{addendum}.pt'))\n",
    "check_or_save(val_ids, join(FEATURES, f'val_ids{addendum}.pt'))\n",
    "check_or_save(test_ids, join(FEATURES, f'test_ids{addendum}.pt'))\n",
    "\n",
    "train_df = data_df.set_index('Adv_ID').loc[train_ids]\n",
    "val_df = data_df.set_index('Adv_ID').loc[val_ids]\n",
    "test_df = data_df.set_index('Adv_ID').loc[test_ids]\n",
    "\n",
    "train_labels_all = list(train_df['Genmodel_ID'])\n",
    "val_labels_all = list(val_df['Genmodel_ID'])\n",
    "test_labels_all = list(test_df['Genmodel_ID'])\n",
    "\n",
    "check_or_save(train_labels_all, join(FEATURES,f'labels_model_all_train{addendum}.pt'))\n",
    "check_or_save(val_labels_all, join(FEATURES,f'labels_model_all_val{addendum}.pt'))\n",
    "check_or_save(test_labels_all, join(FEATURES,f'labels_model_all_test{addendum}.pt'))\n",
    "\n",
    "check_or_save(train_df.loc[:,~train_df.columns.isin(non_feature_columns)],join(FEATURES,f'dvm_features_train_noOH{addendum}.csv'), index=False, header=False)\n",
    "check_or_save(val_df.loc[:,~val_df.columns.isin(non_feature_columns)],join(FEATURES,f'dvm_features_val_noOH{addendum}.csv'), index=False, header=False)\n",
    "check_or_save(test_df.loc[:,~test_df.columns.isin(non_feature_columns)],join(FEATURES,f'dvm_features_test_noOH{addendum}.csv'), index=False, header=False)\n",
    "\n",
    "check_or_save(train_df, join(FEATURES,f'dvm_full_features_train_noOH{addendum}.csv'), index=True, header=True)\n",
    "check_or_save(val_df, join(FEATURES,f'dvm_full_features_val_noOH{addendum}.csv'), index=True, header=True)\n",
    "check_or_save(test_df, join(FEATURES,f'dvm_full_features_test_noOH{addendum}.csv'), index=True, header=True)\n",
    "\n",
    "lengths = [1 for i in range(len(continuous_df.columns))]\n",
    "\n",
    "if 'Genmodel_ID' in categorical_ids:\n",
    "  categorical_ids.remove('Genmodel_ID')\n",
    "max = list(data_df[categorical_ids].max(axis=0))\n",
    "max = [i+1 for i in max]\n",
    "lengths = lengths + max\n",
    "check_or_save(lengths, join(FEATURES, f'tabular_lengths{addendum}.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_paths(df):\n",
    "  paths = []\n",
    "  for indx, row in df.iterrows():\n",
    "      im_name = row['Image_name']\n",
    "      split = im_name.split('$$')\n",
    "      path = join(BASE, 'resized_DVM', split[0], split[1], split[2], split[3], im_name)\n",
    "      paths.append(path)\n",
    "  return paths\n",
    "\n",
    "# For big dataset need to save only paths to load live\n",
    "addendum = '_all_views'\n",
    "train_df = pd.read_csv(join(FEATURES,f'dvm_full_features_train_noOH{addendum}.csv'))\n",
    "val_df = pd.read_csv(join(FEATURES,f'dvm_full_features_val_noOH{addendum}.csv'))\n",
    "test_df = pd.read_csv(join(FEATURES,f'dvm_full_features_test_noOH{addendum}.csv'))\n",
    "\n",
    "for df, name in zip([train_df, val_df, test_df], ['train', 'val', 'test']):\n",
    "  paths = get_paths(df)\n",
    "  check_or_save(paths, join(FEATURES, f'{name}_paths{addendum}.pt'))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save Normalized Ims"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if front_view_only:\n",
    "  IMAGES = join(BASE, 'Confirmed_fronts')\n",
    "else:\n",
    "  IMAGES = join(BASE, 'resized_DVM')\n",
    "\n",
    "for df, t_split in zip([train_df, val_df, test_df], ['train', 'val', 'test']):\n",
    "  images = []\n",
    "  for i,row in df.iterrows():\n",
    "    image_name = row['Image_name']\n",
    "    split = image_name.split('$$')\n",
    "    \n",
    "    if front_view_only:\n",
    "      path = join(IMAGES,split[0],split[2],image_name)\n",
    "    else:\n",
    "      path = join(IMAGES,split[0],split[1],split[2],split[3],image_name)\n",
    "    images.append(read_image(path))\n",
    "  images_t = torch.stack(images).float()\n",
    "  images_t = images_t/255\n",
    "  check_or_save(images_t, join(FEATURES, f'{t_split}_images{addendum}.pt'))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Low Data Splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def low_data_split(df, nclasses):\n",
    "  critical_ids = df.groupby('Genmodel_ID', as_index=False).head(n=1)['Adv_ID']\n",
    "  other_ids = df.loc[~df['Adv_ID'].isin(critical_ids)]['Adv_ID'].values\n",
    "  to_fill_size = (int(len(df)*0.1)-len(critical_ids))\n",
    "  stratify = None\n",
    "  if to_fill_size >= nclasses:\n",
    "    stratify = df.set_index('Adv_ID').loc[other_ids]['Genmodel_ID']\n",
    "  if to_fill_size > 0:\n",
    "    _, low_data_ids = train_test_split(other_ids, test_size=to_fill_size, random_state=2023, stratify=stratify)\n",
    "  else:\n",
    "    low_data_ids = []\n",
    "  new_ids = np.concatenate([critical_ids,low_data_ids])\n",
    "  return new_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "addendum = '_all_views'\n",
    "data_str = 'images'\n",
    "location = \"\"\n",
    "non_feature_columns = ['Image_name', 'Genmodel_ID', 'Predicted_viewpoint', 'Adv_ID']\n",
    "nclasses = 151\n",
    "if addendum=='_all_views':\n",
    "  #data_str = 'paths'\n",
    "  #location = '_server'\n",
    "  nclasses = 286\n",
    "\n",
    "\n",
    "for k, prev_k in zip([0.1,0.01],['','_0.1']):\n",
    "  df = pd.read_csv(join(FEATURES,f'dvm_full_features_train_noOH{addendum}{prev_k}.csv'))\n",
    "  ids = torch.load(join(FEATURES, f'train_ids{addendum}{prev_k}.pt'))\n",
    "  ims = torch.load(join(FEATURES, f'train_{data_str}{addendum}{location}{prev_k}.pt'))\n",
    "  labels = torch.load(join(FEATURES, f'labels_model_all_train{addendum}{prev_k}.pt'))\n",
    "  low_data_ids = low_data_split(df, nclasses)\n",
    "  true_false_mask = [i in low_data_ids for i in ids]\n",
    "  ld = [id for id in ids if id in low_data_ids]\n",
    "  low_data_ids = ld\n",
    "  low_data_df = df.loc[true_false_mask]\n",
    "  if addendum=='_all_views' and not data_str=='images':\n",
    "    ims = np.array(ims)\n",
    "  else:  \n",
    "    ims = torch.tensor(ims)\n",
    "  low_data_ims = ims[true_false_mask]\n",
    "  low_data_labels = [labels[i] for i in range(len(ids)) if ids[i] in low_data_ids]\n",
    "\n",
    "  \n",
    "  check_or_save(low_data_df.loc[:,~low_data_df.columns.isin(non_feature_columns)], join(FEATURES,f'dvm_features_train_noOH{addendum}_{k}.csv'), index=False, header=False)\n",
    "  check_or_save(low_data_df, join(FEATURES,f'dvm_full_features_train_noOH{addendum}_{k}.csv'), index=False, header=True)\n",
    "  check_or_save(low_data_ims, join(FEATURES, f'train_{data_str}{addendum}{location}_{k}.pt'))\n",
    "  check_or_save(low_data_ids, join(FEATURES, f'train_ids{addendum}_{k}.pt'))\n",
    "  check_or_save(low_data_labels, join(FEATURES, f'labels_model_all_train{addendum}_{k}.pt'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "split = 'train'\n",
    "for k in [0.1, 0.01]:\n",
    "  low_data_ids = torch.load(join(FEATURES, f'{split}_ids{addendum}_{k}.pt'))\n",
    "  low_data_df = pd.read_csv(join(FEATURES,f'dvm_full_features_{split}_noOH{addendum}_{k}.csv'))\n",
    "  print(low_data_df.value_counts('Genmodel_ID'))\n",
    "  print(len(low_data_ids))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check Transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "import torch\n",
    "from os.path import join\n",
    "\n",
    "BASE = ''\n",
    "TABLES = join(BASE, 'tables_V2.0')\n",
    "FEATURES = join(BASE, 'features')\n",
    "\n",
    "train_images = torch.load(join(FEATURES, f'val_images.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_size=128\n",
    "\n",
    "transform = transforms.Compose([\n",
    "      transforms.RandomApply([transforms.ColorJitter(brightness=0.8, contrast=0.8, saturation=0.8)], p=0.8),\n",
    "      transforms.RandomGrayscale(p=0.2),\n",
    "      transforms.RandomApply([transforms.GaussianBlur(kernel_size=29, sigma=(0.1, 2.0))],p=0.5),\n",
    "      transforms.RandomResizedCrop(size=(img_size,img_size), scale=(0.2, 1.0), ratio=(0.75, 1.3333333333333333)),\n",
    "      transforms.RandomHorizontalFlip(p=0.5),\n",
    "      transforms.Resize(size=(img_size,img_size)),\n",
    "      transforms.Lambda(lambda x : x.float())\n",
    "    ])\n",
    "\n",
    "im = train_images[1]\n",
    "im_t = transform(im)\n",
    "_ = plt.imshow(im_t.permute(1,2,0))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# New Physical Features"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding missing values to physical table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill using other values\n",
    "physical_df_orig = pd.read_csv(join(FEATURES,'Ad_table (extra).csv'))\n",
    "physical_df_orig.rename(columns={' Genmodel_ID':'Genmodel_ID', ' Genmodel':'Genmodel'}, inplace=True)\n",
    "\n",
    "# Manual touches\n",
    "\n",
    "# Peugeot RCZ\n",
    "physical_df_orig.loc[physical_df_orig['Genmodel_ID'] == '69_36','Wheelbase']=2612\n",
    "# Ford Grand C-Max\n",
    "physical_df_orig.loc[physical_df_orig['Genmodel_ID'] == '29_20','Wheelbase']=2788 \n",
    "\n",
    "def fill_from_other_entry(row):\n",
    "    for attr in ['Wheelbase', 'Length', 'Width', 'Height']:\n",
    "        if pd.isna(row[attr]) or row[attr]==0:\n",
    "            other_rows = physical_df_orig.loc[physical_df_orig['Genmodel_ID']==row['Genmodel_ID']]\n",
    "            other_rows.dropna(subset=[attr], inplace=True)\n",
    "            other_rows.drop_duplicates(subset=[attr], inplace=True)\n",
    "            other_rows = other_rows[other_rows[attr]>0]\n",
    "            if len(other_rows)>0:\n",
    "                row[attr] = other_rows[attr].values[0]\n",
    "    return row\n",
    "\n",
    "physical_df_orig = physical_df_orig.apply(fill_from_other_entry, axis=1)\n",
    "\n",
    "physical_df_orig.to_csv(join(FEATURES,'Ad_table_physical_filled.csv'), index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add physical attributes to features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add jitter to physical dimensions so they aren't just labels\n",
    "def add_jitter(x, jitter=50):\n",
    "    return x + random.randint(-jitter, jitter)\n",
    "\n",
    "random.seed(2022)\n",
    "physical_df = pd.read_csv(join(FEATURES,'Ad_table_physical_filled.csv'))\n",
    "for attr in ['Wheelbase', 'Length', 'Width', 'Height']:\n",
    "    physical_df[attr] = physical_df[attr].apply(add_jitter)\n",
    "physical_df.to_csv(join(FEATURES,'Ad_table_physical_filled_jittered_50.csv'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ford ranger (29_30) has wrong height. Missing 1 in front... 805.0 instead of 1805.0\n",
    "# Mercedes Benz (59_29) wrong wheelbase, 5246.0 instead of 3106\n",
    "# Kia Rio (43_9) wrong wheelbase, 4065.0 instead of 2580\n",
    "# FIXED\n",
    "\n",
    "\n",
    "physical_df = pd.read_csv(join(FEATURES,'Ad_table_physical_filled_jittered_50.csv'))[['Adv_ID', 'Wheelbase','Height','Width','Length']]\n",
    "for v in ['_all_views']:\n",
    "    for split in ['train', 'val', 'test']:\n",
    "        features_df = pd.read_csv(join(FEATURES,f'dvm_full_features_{split}_noOH{v}.csv'))\n",
    "        merged_df = features_df.merge(physical_df, on='Adv_ID')\n",
    "        physical_only_df = merged_df[['Wheelbase','Height','Width','Length','Bodytype']]\n",
    "\n",
    "        for attr in ['Wheelbase','Height','Width','Length']:\n",
    "            assert merged_df[attr].isna().sum()==0\n",
    "            assert (merged_df[attr]==0).sum()==0\n",
    "\n",
    "        # normalize physical attributes\n",
    "        for attr in ['Wheelbase','Height','Width','Length']:\n",
    "            merged_df[attr] = (merged_df[attr]-merged_df[attr].mean())/merged_df[attr].std()\n",
    "            physical_only_df[attr] = (physical_only_df[attr]-physical_only_df[attr].mean())/physical_only_df[attr].std()\n",
    "\n",
    "        # Drop unwanted cols\n",
    "        non_feature_columns = ['Adv_ID', 'Image_name', 'Genmodel_ID']\n",
    "        if v == '_all_views':\n",
    "            non_feature_columns.append('Predicted_viewpoint')\n",
    "        merged_df = merged_df.drop(non_feature_columns, axis=1)\n",
    "\n",
    "        merged_df_cols = merged_df.columns.tolist()\n",
    "        rearranged_cols = merged_df_cols[-4:]+merged_df_cols[:-4]\n",
    "        merged_df = merged_df[rearranged_cols]\n",
    "        check_or_save(merged_df, join(FEATURES,f'dvm_features_{split}_noOH{v}_physical_jittered_50.csv'), index=False, header=False)\n",
    "        check_or_save(physical_only_df, join(FEATURES,f'dvm_features_{split}_noOH{v}_physical_only_jittered_50.csv'), index=False, header=False)\n",
    "    lengths = torch.load(join(FEATURES,f'tabular_lengths{v}.pt'))\n",
    "    new_lengths = [1,1,1,1]\n",
    "    lengths = new_lengths + lengths\n",
    "    check_or_save(lengths, join(FEATURES,f'tabular_lengths{v}_physical.pt'))\n",
    "    lengths = [1,1,1,1,13]\n",
    "    check_or_save(lengths, join(FEATURES,f'tabular_lengths{v}_physical_only.pt'))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add Labels to Featues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for v in ['_all_views']:\n",
    "    for split in ['train', 'val']:\n",
    "        labels = torch.load(join(FEATURES,f'labels_model_all_{split}{v}.pt'))\n",
    "        features = pd.read_csv(join(FEATURES,f'dvm_features_{split}_noOH{v}_physical_jittered_50.csv'), header=None)\n",
    "        features['label'] = labels\n",
    "        check_or_save(features, join(FEATURES,f'dvm_features_{split}_noOH{v}_physical_jittered_50_labeled.csv'), index=False, header=False)\n",
    "    lengths = torch.load(join(FEATURES,f'tabular_lengths{v}_physical.pt'))\n",
    "    lengths.append(max(labels)+1)\n",
    "    check_or_save(lengths, join(FEATURES,f'tabular_lengths{v}_physical_labeled.pt'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('selfsuper')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a21be7fe9607dfe0c8ee311f8a5f36f314167f49973cd8e355a42459a56bba0d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}