nb/example_inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "if os.getcwd().endswith(\"nb\"):\n",
    "    os.chdir(\"..\")  # should be bone-age repo"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-26T13:59:42.360078637Z",
     "start_time": "2023-07-26T13:59:41.616081962Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "%%bash\n",
    "CKPT_DIR=\"$BASE_DIR/output/best_models/\"  # path to checkpoint directory, TODO change if needed\n",
    "BASE_DIR=$(pwd)\n",
    "OUTPUT_DIR=\"$BASE_DIR/output/predictions_rsna/\"  # path in which all prediction .csv files will be saved\n",
    "\n",
    "l=$(cat <<EOF\n",
    "masked_effnet_supShal_highRes_fancy_aug.ckpt\n",
    "masked_effnet_super_shallow_fancy_aug.ckpt\n",
    "masked_effnet-b4_shallow_pretr_fancy_aug.ckpt\n",
    "EOF\n",
    ")\n",
    "mkdir -p $OUTPUT_DIR\n",
    "\n",
    "for c in $l\n",
    "do\n",
    "  echo \"predicting with model $c\"\n",
    "  python predict.py \\\n",
    "        --ckp_path=\"$CKPT_DIR/$c\" \\\n",
    "        --legacy_ckp \\\n",
    "        --gpus=1 \\\n",
    "        --precision=16 \\\n",
    "        --num_workers=8 \\\n",
    "        --batch_size=32 \\\n",
    "        --mask_crop_size=-1 \\\n",
    "        --mask_dirs=../data/masks/fscnn_cos \\\n",
    "        --img_dir=../data/annotated \\\n",
    "        --annotation_csv=data/rsna_test.csv \\\n",
    "        --output_path=\"$OUTPUT_DIR/${c%.ckpt}.csv\" \\\n",
    "\n",
    "done"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Here, the `annotation_csv` argument should point to the .csv file containing at least the `image_ID` (matching the file name), the subdirectory within the `img_dir` directory containing the files (might be left blank), and the `sex` (females encoded as 0. males at 1). To subset the used images, please refer to the splitting arguments in the source code.\n",
    "\n",
    "Make sure that:\n",
    "  * `mask_dirs` points to the directory containing the masks (usually the fscnn_cos directory, see hand-segmentation repo)\n",
    "  * `legacy_ckp` is selected for old (i.e. non-mtl) checkpoints (basically all relevant ones)\n",
    "  * `mask_crop_size` is set appropriately (i.e. -1 = no cropping for RSNA data, for external data 1.15 might be better to compensate for weird scanning)\n",
    "  * `img_dir` points to the directory containing the images (usually the 'annotated' directory, see the data-management repo)\n",
    "  * `gpus`, `batch_size`, and `num_workers` are set appropriately depending on the hardware"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "3.8705049999999996"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# needs to match $OUTPUT_DIR in the bash script above\n",
    "output_dir = \"output/predictions_rsna\"\n",
    "anno = pd.read_csv(\"data/rsna_test.csv\")\n",
    "\n",
    "# gather all predictions\n",
    "df = {c.replace(\".csv\", \"\"): pd.read_csv(f\"{output_dir}/{c}\")[[\"image_ID\", \"pred_bone_age\"]].rename(columns={\"pred_bone_age\": c.replace(\".csv\", \"\")}) for c in os.listdir(output_dir) if not \"ensemble\" in c}\n",
    "result = pd.concat([df[c].set_index('image_ID') for c in df], axis=1)\n",
    "\n",
    "# take mean of all predictions for ensemble\n",
    "result[\"ensemble_bone_age\"] = result.mean(axis=1)\n",
    "\n",
    "# add ground truth if available\n",
    "result = result.reset_index().merge(anno[[\"image_ID\", \"bone_age\"]].rename(columns={\"bone_age\": \"gt_bone_age\"}))\n",
    "\n",
    "# save to csv\n",
    "result.to_csv(f\"{output_dir}/ensemble.csv\", index=False)\n",
    "\n",
    "# sanity check MAD score\n",
    "np.linalg.norm(result[\"ensemble_bone_age\"] - result[\"gt_bone_age\"], 1) / len(result)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-07-26T14:00:22.549313575Z",
     "start_time": "2023-07-26T14:00:22.522814062Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}