main_figs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80f6d28e-b64b-44af-bea9-b31d9ce9283b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.linear_model import LinearRegression\n",
    "import os\n",
    "from datetime import datetime\n",
    "from scipy.optimize import curve_fit\n",
    "\n",
    "from modelGPT.constants import (\n",
    "    ALL_FEATURES, GROUND_TRUTH_CSV, MODEL_NAME_COL, DATASET_COL, PRED_TARGET\n",
    ")\n",
    "\n",
    "sns.set_palette('colorblind')\n",
    "df = pd.read_csv(GROUND_TRUTH_CSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "885a755f-7819-44fc-915a-eeec18b2f172",
   "metadata": {},
   "outputs": [],
   "source": [
    "name_order = {'imagenet1k':'Imagenet','sun397':'SUN397', 'flowers':'flowers102', 'caltech101':'CalTech101',\n",
    "              'pets':'Oxford Pets','cifar100':'CIFAR100', \n",
    "          'cars':'Stanford Cars','stl10':'stl10','voc2007':'VOC2007','resisc45':'resisc45',\n",
    "              'eurosat':'euroSAT','fgvc_aircraft':'FGVC aircraft', 'gtsrb':'GTSRB',\n",
    "          'kitti_closest_vehicle_distance':'KITTI','country211':'country211', 'dmlab':'DMLab', 'svhn':'SVHN','fer2013':'FER2013','diabetic_retinopathy':'Retinopathy',\n",
    "          'clevr_closest_object_distance':'CLEVR-DISTANCE','clevr_count_all':'CLEVR-COUNT','mnist':'MNIST','dtd':'DTD', 'pcam':'Pcam', 'renderedsst2':'renderedsst2'\n",
    "             }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a520a507-6917-4be4-80dd-85abaa2b1702",
   "metadata": {},
   "outputs": [],
   "source": [
    "translation_dict = {\n",
    "    'coca_ViT-B-32 laion2b_s13b_b90k': 'CoCa-ViT-B/32, L2b-c',\n",
    "    'coca_ViT-B-32 mscoco_finetuned_laion2b_s13b_b90k': 'CoCa-ViT-B/32, L2b-c + coco',\n",
    "    'coca_ViT-L-14 laion2b_s13b_b90k': 'CoCa-ViT-L/14, L2b-c',\n",
    "    'coca_ViT-L-14 mscoco_finetuned_laion2b_s13b_b90k': 'CoCa-ViT-L/14, L2b-c + coco',\n",
    "    'convnext_base laion400m_s13b_b51k': 'ConvNEXT-B, L400m-c',\n",
    "    'convnext_base_w laion2b_s13b_b82k': 'ConvNEXT-BW, L2b-d',\n",
    "    'convnext_base_w laion2b_s13b_b82k_augreg': 'ConvNEXT-BW, L2b-e',\n",
    "    'convnext_base_w laion_aesthetic_s13b_b82k': 'ConvNEXT-BW, L2b-f',\n",
    "    'convnext_base_w_320 laion_aesthetic_s13b_b82k': 'ConvNEXT-BW-320, L2b-f',\n",
    "    'convnext_base_w_320 laion_aesthetic_s13b_b82k_augreg': 'ConvNEXT-BW-320, L2b-g',\n",
    "    'convnext_large_d laion2b_s26b_b102k_augreg': 'ConvNEXT-LD, L2b-h',\n",
    "    'convnext_large_d_320 laion2b_s29b_b131k_ft': 'ConvNEXT-LD-320, L2b-i',\n",
    "    'convnext_large_d_320 laion2b_s29b_b131k_ft_soup': 'ConvNEXT-LD-320, L2b-j',\n",
    "    'RN50 openai': 'RN50, WIT',\n",
    "    'RN101 openai': 'RN101, WIT',\n",
    "    'RN50x4 openai': 'RN50x4, WIT',\n",
    "    'RN50x16 openai': 'RN50x16, WIT',\n",
    "    'RN50x64 openai': 'RN50x64, WIT',\n",
    "    'ViT-B-32 openai': 'ViT-B/32, WIT',\n",
    "    'ViT-B-16 openai': 'ViT-B/16, WIT',\n",
    "    'ViT-L-14 openai': 'ViT-L/14, WIT',\n",
    "    'ViT-L-14-336 openai': 'ViT-L/14-336, WIT',\n",
    "    'ViT-B-16 laion400m_e32': 'ViT-B/16, L400m-a',\n",
    "    'ViT-B-16-plus-240 laion400m_e32': 'ViT-B/16-240, L400m-a',\n",
    "    'ViT-B-32 laion2b_e16': 'ViT-B/32, L2b-c',\n",
    "    'ViT-B-32 laion2b_s34b_b79k': 'ViT-B/32, L2b-b',\n",
    "    'ViT-B-32 laion400m_e31': 'ViT-B/32, L400m-b',\n",
    "    'ViT-B-32 laion400m_e32': 'ViT-B/32, L400m-a',\n",
    "    'ViT-B-32-quickgelu laion400m_e32': 'ViT-B/32, L400m-a2',\n",
    "    'ViT-H-14 laion2b_s32b_b79k': 'ViT-H/14, L2b-b',\n",
    "    'ViT-L-14 laion2b_s32b_b82k': 'ViT-L/14, L2b-b',\n",
    "    'ViT-L-14 laion400m_e31': 'ViT-L/14, L400m-b',\n",
    "    'ViT-L-14 laion400m_e32': 'ViT-L/14, L400m-a',\n",
    "    'ViT-g-14 laion2b_s12b_b42k': 'ViT-G/14, L2b-a',\n",
    "    'ViT-g-14 laion2b_s34b_b88k': 'ViT-G/14, L2b-b'\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "194c97cc-5e59-40ad-a501-4fb07aa84232",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "PRED_TARGET = 'acc1'\n",
    "datasets = df.dataset.unique()\n",
    "gt_df = pd.pivot_table(\n",
    "            df, values=PRED_TARGET, index=[MODEL_NAME_COL], \n",
    "            columns=DATASET_COL\n",
    ")\n",
    "\n",
    "model = LinearRegression()\n",
    "\n",
    "X = df[ALL_FEATURES]\n",
    "y = df[PRED_TARGET]\n",
    "model.fit(X, y)\n",
    "df['pred'] = model.predict(X)\n",
    "\n",
    "\n",
    "model_pred = pd.pivot_table(\n",
    "            df, values='pred', index=[MODEL_NAME_COL], \n",
    "            columns=DATASET_COL\n",
    ")\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots(1,1,figsize=(6.5, 4.3))\n",
    "\n",
    "\n",
    "pred = []\n",
    "gt = []\n",
    "\n",
    "for dataset in datasets:\n",
    "    ax.plot(model_pred[dataset], gt_df[dataset], '.')\n",
    "    pred.append(model_pred[dataset].values)\n",
    "    gt.append(gt_df[dataset].values)\n",
    "\n",
    "ax.plot([0,1], [0,1])\n",
    "ax.legend([name_order[d] for d in datasets], loc=[1.02,-0.01], labelspacing=0, fontsize =11)\n",
    "ax.set_xlim([0,1])\n",
    "ax.set_ylim([0,1])\n",
    "ax.set_xlabel('Predicted Accuracy', fontsize =12)\n",
    "ax.set_ylabel('Ground-Truth Accuracy', fontsize =12)\n",
    "plt.title(f'$R^2$ = {round(r2_score(np.concatenate(gt,0), np.concatenate(pred,0)),2)}')\n",
    "plt.grid(linestyle='dotted')\n",
    "plt.tight_layout(rect=[0, 0, 1, 1])\n",
    "plt.savefig('Pred-vs-GT.pdf', format='pdf', bbox_inches='tight', dpi=300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48e89a1b-b287-4bcf-83eb-e5e1e4f7e44f",
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_rename={\n",
    "    'intra_close':'Silhouette score', \n",
    "    'inter_close':'Class Dispersion', \n",
    "    'intraclass_sim':'Fisher criterion', \n",
    "    'superclass_metric':'Synonym Consistency'\n",
    "}\n",
    "\n",
    "models = []\n",
    "order_dict = {v: i for i, v in enumerate(name_order.keys())}\n",
    "def sort_key(v):\n",
    "    return [order_dict[vv] for vv in v]\n",
    "\n",
    "df_openai = df.loc[df.pretrained.isin(['openai'])]\n",
    "\n",
    "df_openai_RN = df_openai.loc[df_openai.model.isin(['RN50', 'RN50x4', 'RN50x16', 'RN50x64', 'RN101'])].groupby('model_fullname')\n",
    "df_openai_ViT = df_openai.loc[~df_openai.model.isin(['RN50', 'RN50x4', 'RN50x16', 'RN50x64', 'RN101'])].groupby('model_fullname')\n",
    "\n",
    "df_400m = df.loc[df.pretrained.isin([d for d in df.pretrained.unique() if 'laion400m' in d])].groupby('model_fullname')\n",
    "df_2b = df.loc[~df.pretrained.isin([d for d in df.pretrained.unique() if 'laion400m' in d or d=='openai'])].groupby('model_fullname')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "325ff315-5dd4-4c20-987b-96aa5db1ef07",
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_rename={\n",
    "    'intra_close':'Silhouette score', \n",
    "    'inter_close':'Class Dispersion', \n",
    "    'intraclass_sim':'Fisher criterion', \n",
    "    'superclass_metric':'Synonym Consistency'\n",
    "}\n",
    "\n",
    "models = []\n",
    "order_dict = {v: i for i, v in enumerate(name_order.keys())}\n",
    "def sort_key(v):\n",
    "    return [order_dict[vv] for vv in v]\n",
    "\n",
    "df_openai = df.loc[df.pretrained.isin(['openai'])]\n",
    "\n",
    "df_openai_RN = df_openai.loc[df_openai.model.isin(['RN50', 'RN50x4', 'RN50x16', 'RN50x64', 'RN101'])].groupby('model_fullname')\n",
    "df_openai_ViT = df_openai.loc[~df_openai.model.isin(['RN50', 'RN50x4', 'RN50x16', 'RN50x64', 'RN101'])].groupby('model_fullname')\n",
    "\n",
    "df_400m = df.loc[df.pretrained.isin([d for d in df.pretrained.unique() if 'laion400m' in d])].groupby('model_fullname')\n",
    "df_2b = df.loc[~df.pretrained.isin([d for d in df.pretrained.unique() if 'laion400m' in d or d=='openai'])]#.groupby('model_fullname')\n",
    "df_2b_convnext = df_2b.loc[df_2b.model.isin([d for d in df.model.unique() if 'convnext' in d])].groupby('model_fullname')\n",
    "df_2b_ViT = df_2b.loc[~df_2b.model.isin([d for d in df.model.unique() if 'convnext' in d])].groupby('model_fullname')\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots(1, 2, figsize=(10, 4))\n",
    "\n",
    "metric_rename={\n",
    "    'intra_close':'Silhouette', \n",
    "    'inter_close':'Class Disp.', \n",
    "    'intraclass_sim':'Fisher', \n",
    "    'superclass_metric':'Synonym'\n",
    "}\n",
    "\n",
    "colors = ['#006BA4', '#FF800E', '#ABABAB', '#595959', '#5F9ED1', '#C85200', '#898989', '#A2C8EC', '#FFBC79', '#CFCFCF']\n",
    "colors = ['k', '#006BA4', '#FF800E', '#898989', '#A2C8EC', '#FFBC79', '#CFCFCF']\n",
    "\n",
    "for i, metrics in enumerate([['intra_close', 'intraclass_sim'], ['inter_close', 'superclass_metric']]):\n",
    "    metric = metrics[0]\n",
    "    \n",
    "    tmp_openai_RN_1 = df_openai_RN[metric].mean().sort_values()\n",
    "    tmp_openai_ViT_1 = df_openai_ViT[metric].mean().sort_values()\n",
    "    tmp_400m_1 = df_400m[metric].mean().sort_values()\n",
    "    tmp_2b_1 = df_2b_convnext[metric].mean().sort_values()\n",
    "    tmp_2b_3 = df_2b_ViT[metric].mean().sort_values()\n",
    "\n",
    "    ax[i].plot([translation_dict[t] for t in tmp_openai_ViT_1.index], tmp_openai_ViT_1.values, '-', c=colors[0], label=metric_rename[metrics[0]])\n",
    "    ax[i].plot([], [], '--', c=colors[0], label=metric_rename[metrics[1]])    \n",
    "    ax[i].plot([translation_dict[t] for t in tmp_openai_RN_1.index], tmp_openai_RN_1.values, '-', c=colors[0], label='WIT', marker='s', markersize=4)\n",
    "    ax[i].plot([translation_dict[t] for t in tmp_400m_1.index], tmp_400m_1.values, '-', c=colors[1], label='L400m')\n",
    "    ax[i].plot([translation_dict[t] for t in tmp_2b_3.index], tmp_2b_3.values, '-', label='L2b',c=colors[2])\n",
    "    ax[i].plot([translation_dict[t] for t in tmp_2b_1.index], tmp_2b_1.values, '-',c=colors[2],  marker='x')\n",
    "\n",
    "    metric = metrics[1]\n",
    "    tmp_openai_RN_2 = df_openai_RN[metric].mean()[tmp_openai_RN_1.index]\n",
    "    tmp_openai_ViT_2 = df_openai_ViT[metric].mean()[tmp_openai_ViT_1.index]\n",
    "    tmp_400m_2 = df_400m[metric].mean()[tmp_400m_1.index]\n",
    "    tmp_2b_2 = df_2b_convnext[metric].mean()[tmp_2b_1.index]\n",
    "    tmp_2b_4 = df_2b_ViT[metric].mean()[tmp_2b_3.index]\n",
    "    \n",
    "    ax[i].plot([translation_dict[t] for t in tmp_openai_ViT_2.index], tmp_openai_ViT_2.values, '--', c=colors[0])\n",
    "    ax[i].plot([translation_dict[t] for t in tmp_openai_RN_2.index], tmp_openai_RN_2.values, '--', c=colors[0],  marker='s', markersize=4)\n",
    "    ax[i].plot([translation_dict[t] for t in tmp_400m_2.index], tmp_400m_2.values, '--',c=colors[1])\n",
    "    ax[i].plot([translation_dict[t] for t in tmp_2b_4.index], tmp_2b_4.values, '--',c=colors[2])\n",
    "    ax[i].plot([translation_dict[t] for t in tmp_2b_2.index], tmp_2b_2.values, '--',c=colors[2],  marker='x')\n",
    "\n",
    "    tmp = list(tmp_openai_ViT_1.index) +  list(tmp_openai_RN_1.index) + list(tmp_400m_1.index) +  list(tmp_2b_3.index)+  list(tmp_2b_1.index)\n",
    "    ax[i].set_xticklabels([translation_dict[t].split(',')[0]  for t in tmp], rotation=90)\n",
    "    ax[i].set_ylabel('Cosine Similarity')\n",
    "    leg = ax[i].legend(loc='lower right')\n",
    "    for l in leg.legendHandles[2:]:\n",
    "        l.set_linewidth(5)\n",
    "    \n",
    "ax[0].set_ylabel('Cosine Similarity $(\\\\leftarrow)$')\n",
    "ax[1].set_ylabel('Cosine Similarity $(\\\\rightarrow)$')\n",
    "ax[0].set_title('Inter-class Similarity')\n",
    "ax[1].set_title('Intra-class Similarity')\n",
    "ax[0].set_ylim([0.40, 0.95])\n",
    "ax[1].set_ylim([0.40, 0.95])\n",
    "plt.tight_layout(rect=[0, 0, 1, 1])\n",
    "plt.subplots_adjust(wspace=0.17)\n",
    "ax[0].axvspan(4, 8, ymin=0, ymax=1, facecolor='yellow', alpha=0.2)\n",
    "ax[0].axvspan(27, 34, ymin=0, ymax=1, facecolor='grey', alpha=0.15)\n",
    "ax[1].axvspan(4, 8, ymin=0, ymax=1, facecolor='yellow', alpha=0.2)\n",
    "ax[1].axvspan(27, 34, ymin=0, ymax=1, facecolor='grey', alpha=0.15)\n",
    "plt.savefig('ModelUnd.pdf', format='pdf', bbox_inches='tight', dpi=300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46c49bd4-1bf7-4495-84f8-17f8da2017a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "forecast_days = 365//2\n",
    "sns.set_context(\"notebook\", font_scale=1.1)\n",
    "\n",
    "df = pd.read_csv('LOVM/openai_num_models.csv')\n",
    "\n",
    "df['Date'] = pd.to_datetime(df['Date'])\n",
    "base_date = df['Date'].min()\n",
    "df['Time'] = (df['Date'] - base_date).dt.days\n",
    "def func(time, a, b, c):\n",
    "    return a * np.exp(b * time)+ c\n",
    "guess = [1, 0.01, 1 ]\n",
    "\n",
    "popt, pcov = curve_fit(func, df['Time'], df['Number of Pretrained Models'],  p0=guess)\n",
    "print(\"Optimized parameters [a, b] are : \", popt)\n",
    "\n",
    "df['exp_fit'] = func(df['Time'], popt[0], popt[1], popt[2])\n",
    "\n",
    "last_date = df['Date'].max()\n",
    "forecast_dates = pd.date_range(start=last_date, periods=forecast_days + 1)  # we start at the day after the last date\n",
    "forecast_time = (forecast_dates - base_date).days\n",
    "\n",
    "\n",
    "# Create a DataFrame\n",
    "df_forecast = pd.DataFrame({\n",
    "    'Date': pd.to_datetime(forecast_dates),\n",
    "    'Time': forecast_time\n",
    "})\n",
    "\n",
    "# Apply the model to forecast\n",
    "df_forecast['Forecast'] = func(df_forecast['Time'], popt[0], popt[1], popt[2])\n",
    "\n",
    "\n",
    "# Set the figure size for a square plot\n",
    "plt.figure(figsize=(5, 4))\n",
    "\n",
    "# Plot the data using seaborn\n",
    "sns.scatterplot(data=df, x=\"Date\", y=\"Number of Pretrained Models\", marker='s')\n",
    "sns.lineplot(data=df, x=\"Date\", y=\"exp_fit\", label = 'fit')\n",
    "sns.lineplot(data=df_forecast, x=\"Date\", y=\"Forecast\", dashes=(5, 2), label = 'forcast')\n",
    "\n",
    "# Rotate the x-axis labels for better readability\n",
    "plt.xticks(rotation=25)\n",
    "\n",
    "# Set the x and y-axis labels\n",
    "plt.xlabel(\"Date\", fontsize=14)\n",
    "plt.ylabel(\"Number of Pretrained Models\", fontsize=14)\n",
    "\n",
    "# Save the plot with DPI 300\n",
    "plt.savefig(\"pretrained_models_plot.png\", dpi=300)\n",
    "\n",
    "# Display the plot\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}