solution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "66bd754c",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-06-02T14:47:57.394567Z",
     "iopub.status.busy": "2022-06-02T14:47:57.393807Z",
     "iopub.status.idle": "2022-06-02T14:55:50.018923Z",
     "shell.execute_reply": "2022-06-02T14:55:50.017580Z"
    },
    "papermill": {
     "duration": 472.634619,
     "end_time": "2022-06-02T14:55:50.021780",
     "exception": false,
     "start_time": "2022-06-02T14:47:57.387161",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing /kaggle/input/imc-2022-loftr-quadtreeattention/packages/torch-1.8.2cu102-cp37-cp37m-linux_x86_64.whl\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch==1.8.2cu102) (4.2.0)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torch==1.8.2cu102) (1.21.6)\r\n",
      "Installing collected packages: torch\r\n",
      "  Attempting uninstall: torch\r\n",
      "    Found existing installation: torch 1.11.0\r\n",
      "    Uninstalling torch-1.11.0:\r\n",
      "      Successfully uninstalled torch-1.11.0\r\n",
      "Successfully installed torch-1.8.2+cu102\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\r\n",
      "\u001b[0mProcessing /kaggle/input/imc-2022-loftr-quadtreeattention/packages/torchvision-0.9.2cu102-cp37-cp37m-linux_x86_64.whl\r\n",
      "Requirement already satisfied: pillow>=4.1.1 in /opt/conda/lib/python3.7/site-packages (from torchvision==0.9.2cu102) (9.1.0)\r\n",
      "Requirement already satisfied: torch==1.8.2 in /opt/conda/lib/python3.7/site-packages (from torchvision==0.9.2cu102) (1.8.2+cu102)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torchvision==0.9.2cu102) (1.21.6)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch==1.8.2->torchvision==0.9.2cu102) (4.2.0)\r\n",
      "Installing collected packages: torchvision\r\n",
      "  Attempting uninstall: torchvision\r\n",
      "    Found existing installation: torchvision 0.12.0\r\n",
      "    Uninstalling torchvision-0.12.0:\r\n",
      "      Successfully uninstalled torchvision-0.12.0\r\n",
      "Successfully installed torchvision-0.9.2+cu102\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\r\n",
      "\u001b[0mProcessing /kaggle/input/imc-2022-loftr-quadtreeattention/packages/kornia_moons-0.1.9-py3-none-any.whl\r\n",
      "Requirement already satisfied: opencv-python in /opt/conda/lib/python3.7/site-packages (from kornia-moons==0.1.9) (4.5.4.60)\r\n",
      "Requirement already satisfied: matplotlib in /opt/conda/lib/python3.7/site-packages (from kornia-moons==0.1.9) (3.5.2)\r\n",
      "Requirement already satisfied: kornia in /opt/conda/lib/python3.7/site-packages (from kornia-moons==0.1.9) (0.5.8)\r\n",
      "Requirement already satisfied: torch in /opt/conda/lib/python3.7/site-packages (from kornia-moons==0.1.9) (1.8.2+cu102)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch->kornia-moons==0.1.9) (4.2.0)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torch->kornia-moons==0.1.9) (1.21.6)\r\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->kornia-moons==0.1.9) (1.4.2)\r\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->kornia-moons==0.1.9) (21.3)\r\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->kornia-moons==0.1.9) (4.33.3)\r\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->kornia-moons==0.1.9) (9.1.0)\r\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.7/site-packages (from matplotlib->kornia-moons==0.1.9) (2.8.2)\r\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib->kornia-moons==0.1.9) (0.11.0)\r\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->kornia-moons==0.1.9) (3.0.9)\r\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib->kornia-moons==0.1.9) (1.16.0)\r\n",
      "Installing collected packages: kornia-moons\r\n",
      "Successfully installed kornia-moons-0.1.9\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\r\n",
      "\u001b[0mProcessing /kaggle/input/imc-2022-loftr-quadtreeattention/packages/loguru-0.6.0-py3-none-any.whl\r\n",
      "Installing collected packages: loguru\r\n",
      "Successfully installed loguru-0.6.0\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\r\n",
      "\u001b[0mProcessing /kaggle/input/imc-2022-loftr-quadtreeattention/packages/einops-0.4.1-py3-none-any.whl\r\n",
      "Installing collected packages: einops\r\n",
      "Successfully installed einops-0.4.1\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\r\n",
      "\u001b[0mProcessing /kaggle/input/imc-2022-loftr-quadtreeattention/packages/timm-0.5.4-py3-none-any.whl\r\n",
      "Requirement already satisfied: torch>=1.4 in /opt/conda/lib/python3.7/site-packages (from timm==0.5.4) (1.8.2+cu102)\r\n",
      "Requirement already satisfied: torchvision in /opt/conda/lib/python3.7/site-packages (from timm==0.5.4) (0.9.2+cu102)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch>=1.4->timm==0.5.4) (4.2.0)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torch>=1.4->timm==0.5.4) (1.21.6)\r\n",
      "Requirement already satisfied: pillow>=4.1.1 in /opt/conda/lib/python3.7/site-packages (from torchvision->timm==0.5.4) (9.1.0)\r\n",
      "Installing collected packages: timm\r\n",
      "Successfully installed timm-0.5.4\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\r\n",
      "\u001b[0mProcessing /kaggle/working/QuadTreeAttention\r\n",
      "  Preparing metadata (setup.py) ... \u001b[?25l-\b \bdone\r\n",
      "\u001b[?25hBuilding wheels for collected packages: quadtree-attention-package\r\n",
      "  Building wheel for quadtree-attention-package (setup.py) ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for quadtree-attention-package: filename=quadtree_attention_package-0.0.0-cp37-cp37m-linux_x86_64.whl size=4463247 sha256=e056524a942e9ec332d44eec559da87729c842423ab59b5a3a98d1bb02867d45\r\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-afk74wf5/wheels/4e/74/c3/37b49f8e7d7cffe8f4c96b3158452eca926bd72920a0460d4e\r\n",
      "Successfully built quadtree-attention-package\r\n",
      "Installing collected packages: quadtree-attention-package\r\n",
      "Successfully installed quadtree-attention-package-0.0.0\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('../input/imc-2022-loftr-quadtreeattention')\n",
    "sys.path.append('../input/imc-2022-loftr-quadtreeattention/QuadTreeAttention')\n",
    "sys.path.append(\"../input/super-glue-pretrained-network\")\n",
    "\n",
    "!pip install ../input/imc-2022-loftr-quadtreeattention/packages/torch-1.8.2cu102-cp37-cp37m-linux_x86_64.whl\n",
    "!pip install ../input/imc-2022-loftr-quadtreeattention/packages/torchvision-0.9.2cu102-cp37-cp37m-linux_x86_64.whl\n",
    "!pip install ../input/imc-2022-loftr-quadtreeattention/packages/kornia_moons-0.1.9-py3-none-any.whl\n",
    "!pip install ../input/imc-2022-loftr-quadtreeattention/packages/loguru-0.6.0-py3-none-any.whl\n",
    "!pip install ../input/imc-2022-loftr-quadtreeattention/packages/einops-0.4.1-py3-none-any.whl\n",
    "!pip install ../input/imc-2022-loftr-quadtreeattention/packages/timm-0.5.4-py3-none-any.whl\n",
    "\n",
    "!cp -r ../input/imc-2022-loftr-quadtreeattention/QuadTreeAttention/ ../working/ # input folder is read only\n",
    "!cd ../working/QuadTreeAttention && pip install ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a7e54b92",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-02T14:55:50.038643Z",
     "iopub.status.busy": "2022-06-02T14:55:50.037383Z",
     "iopub.status.idle": "2022-06-02T14:55:59.649869Z",
     "shell.execute_reply": "2022-06-02T14:55:59.648733Z"
    },
    "papermill": {
     "duration": 9.623591,
     "end_time": "2022-06-02T14:55:59.652629",
     "exception": false,
     "start_time": "2022-06-02T14:55:50.029038",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torchvision/transforms/functional_pil.py:207: DeprecationWarning: BILINEAR is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BILINEAR instead.\n",
      "  def resize(img, size, interpolation=Image.BILINEAR):\n",
      "/opt/conda/lib/python3.7/site-packages/torchvision/transforms/functional_pil.py:280: DeprecationWarning: BICUBIC is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BICUBIC instead.\n",
      "  def perspective(img, perspective_coeffs, interpolation=Image.BICUBIC, fill=None):\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:585: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  np.object,\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:627: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  np.object,\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:637: DeprecationWarning: `np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  np.bool,\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py:108: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  np.object:\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py:110: DeprecationWarning: `np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  np.bool:\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/autograph/impl/api.py:22: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses\n",
      "  import imp\n",
      "/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/numpy_ops/np_random.py:110: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  def randint(low, high=None, size=None, dtype=onp.int):  # pylint: disable=missing-function-docstring\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:23: DeprecationWarning: NEAREST is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.NEAREST or Dither.NONE instead.\n",
      "  'nearest': pil_image.NEAREST,\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:24: DeprecationWarning: BILINEAR is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BILINEAR instead.\n",
      "  'bilinear': pil_image.BILINEAR,\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:25: DeprecationWarning: BICUBIC is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BICUBIC instead.\n",
      "  'bicubic': pil_image.BICUBIC,\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:28: DeprecationWarning: HAMMING is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.HAMMING instead.\n",
      "  if hasattr(pil_image, 'HAMMING'):\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:29: DeprecationWarning: HAMMING is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.HAMMING instead.\n",
      "  _PIL_INTERPOLATION_METHODS['hamming'] = pil_image.HAMMING\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:30: DeprecationWarning: BOX is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BOX instead.\n",
      "  if hasattr(pil_image, 'BOX'):\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:31: DeprecationWarning: BOX is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BOX instead.\n",
      "  _PIL_INTERPOLATION_METHODS['box'] = pil_image.BOX\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:33: DeprecationWarning: LANCZOS is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.LANCZOS instead.\n",
      "  if hasattr(pil_image, 'LANCZOS'):\n",
      "/opt/conda/lib/python3.7/site-packages/keras_preprocessing/image/utils.py:34: DeprecationWarning: LANCZOS is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.LANCZOS instead.\n",
      "  _PIL_INTERPOLATION_METHODS['lanczos'] = pil_image.LANCZOS\n",
      "/opt/conda/lib/python3.7/site-packages/timm/data/auto_augment.py:39: DeprecationWarning: BILINEAR is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BILINEAR instead.\n",
      "  _RANDOM_INTERPOLATION = (Image.BILINEAR, Image.BICUBIC)\n",
      "/opt/conda/lib/python3.7/site-packages/timm/data/auto_augment.py:39: DeprecationWarning: BICUBIC is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BICUBIC instead.\n",
      "  _RANDOM_INTERPOLATION = (Image.BILINEAR, Image.BICUBIC)\n",
      "/opt/conda/lib/python3.7/site-packages/timm/data/transforms.py:39: DeprecationWarning: NEAREST is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.NEAREST or Dither.NONE instead.\n",
      "  Image.NEAREST: 'nearest',\n",
      "/opt/conda/lib/python3.7/site-packages/timm/data/transforms.py:40: DeprecationWarning: BILINEAR is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BILINEAR instead.\n",
      "  Image.BILINEAR: 'bilinear',\n",
      "/opt/conda/lib/python3.7/site-packages/timm/data/transforms.py:41: DeprecationWarning: BICUBIC is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BICUBIC instead.\n",
      "  Image.BICUBIC: 'bicubic',\n",
      "/opt/conda/lib/python3.7/site-packages/timm/data/transforms.py:42: DeprecationWarning: BOX is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BOX instead.\n",
      "  Image.BOX: 'box',\n",
      "/opt/conda/lib/python3.7/site-packages/timm/data/transforms.py:43: DeprecationWarning: HAMMING is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.HAMMING instead.\n",
      "  Image.HAMMING: 'hamming',\n",
      "/opt/conda/lib/python3.7/site-packages/timm/data/transforms.py:44: DeprecationWarning: LANCZOS is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.LANCZOS instead.\n",
      "  Image.LANCZOS: 'lanczos',\n"
     ]
    }
   ],
   "source": [
    "from kornia_moons.feature import *\n",
    "import kornia.feature as KF\n",
    "import kornia as K\n",
    "import numpy as np\n",
    "import pydegensac\n",
    "import torch\n",
    "import cv2\n",
    "import csv\n",
    "import gc\n",
    "\n",
    "from FeatureMatching.src.lightning.lightning_loftr import PL_LoFTR\n",
    "from FeatureMatching.src.utils.plotting import make_matching_figure\n",
    "from config.default import get_cfg_defaults\n",
    "\n",
    "from models.matching import Matching\n",
    "from models.utils import (compute_pose_error, compute_epipolar_error,\n",
    "                          estimate_pose, make_matching_plot,\n",
    "                          error_colormap, AverageTimer, pose_auc, read_image,\n",
    "                          rotate_intrinsics, rotate_pose_inplane,\n",
    "                          scale_intrinsics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7b375e10",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-02T14:55:59.670997Z",
     "iopub.status.busy": "2022-06-02T14:55:59.670004Z",
     "iopub.status.idle": "2022-06-02T14:56:05.190200Z",
     "shell.execute_reply": "2022-06-02T14:56:05.188997Z"
    },
    "papermill": {
     "duration": 5.532485,
     "end_time": "2022-06-02T14:56:05.193086",
     "exception": false,
     "start_time": "2022-06-02T14:55:59.660601",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-02 14:56:02.042 | INFO     | FeatureMatching.src.lightning.lightning_loftr:__init__:49 - Load '../input/imc-2022-loftr-quadtreeattention/checkpoints/quadtree_outdoor.ckpt' as pretrained checkpoint\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda')\n",
    "\n",
    "model_loftr = PL_LoFTR(get_cfg_defaults(), pretrained_ckpt='../input/imc-2022-loftr-quadtreeattention/checkpoints/quadtree_outdoor.ckpt')\n",
    "model_loftr = model_loftr.to(device).eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9ec6f897",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-02T14:56:05.211477Z",
     "iopub.status.busy": "2022-06-02T14:56:05.211043Z",
     "iopub.status.idle": "2022-06-02T14:56:05.973137Z",
     "shell.execute_reply": "2022-06-02T14:56:05.971907Z"
    },
    "papermill": {
     "duration": 0.774437,
     "end_time": "2022-06-02T14:56:05.975901",
     "exception": false,
     "start_time": "2022-06-02T14:56:05.201464",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded SuperPoint model\n",
      "Loaded SuperGlue model (\"outdoor\" weights)\n"
     ]
    }
   ],
   "source": [
    "resize = [-1, ]\n",
    "resize_float = True\n",
    "\n",
    "config = {\n",
    "    \"superpoint\": {\n",
    "        \"nms_radius\": 4,\n",
    "        \"keypoint_threshold\": 0.005,\n",
    "        \"max_keypoints\": 1024\n",
    "    },\n",
    "    \"superglue\": {\n",
    "        \"weights\": \"outdoor\",\n",
    "        \"sinkhorn_iterations\": 20,\n",
    "        \"match_threshold\": 0.2,\n",
    "    }\n",
    "}\n",
    "model_sg = Matching(config).eval().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f5b4d3da",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-02T14:56:05.993727Z",
     "iopub.status.busy": "2022-06-02T14:56:05.993270Z",
     "iopub.status.idle": "2022-06-02T14:56:06.009429Z",
     "shell.execute_reply": "2022-06-02T14:56:06.008293Z"
    },
    "papermill": {
     "duration": 0.027892,
     "end_time": "2022-06-02T14:56:06.011798",
     "exception": false,
     "start_time": "2022-06-02T14:56:05.983906",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DIM = (640, 1120)\n",
    "src = '../input/image-matching-challenge-2022'\n",
    "\n",
    "test_samples = []\n",
    "with open(f'{src}/test.csv') as f:\n",
    "    reader = csv.reader(f, delimiter=',')\n",
    "    for i, row in enumerate(reader):\n",
    "        # Skip header.\n",
    "        if i == 0:\n",
    "            continue\n",
    "        test_samples += [row]\n",
    "\n",
    "\n",
    "def FlattenMatrix(M, num_digits=8):    \n",
    "    return ' '.join([f'{v:.{num_digits}e}' for v in M.flatten()])\n",
    "\n",
    "\n",
    "def load_torch_image(fname, device):\n",
    "    img_raw = cv2.imread(fname)\n",
    "\n",
    "    scale_w = DIM[0] / img_raw.shape[1]\n",
    "    scale_h = DIM[1] / img_raw.shape[0]\n",
    "\n",
    "    img_rs = cv2.resize(img_raw, DIM)\n",
    "    img_rs = K.image_to_tensor(img_rs, False).float() / 255.\n",
    "    img_rs = K.color.bgr_to_rgb(img_rs)\n",
    "\n",
    "    img_raw = K.image_to_tensor(img_raw, False).float() / 255.\n",
    "    img_raw = K.color.bgr_to_rgb(img_raw)\n",
    "\n",
    "    return img_rs.to(device), img_raw.to(device), scale_w, scale_h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f8c4b08e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-02T14:56:06.029698Z",
     "iopub.status.busy": "2022-06-02T14:56:06.029311Z",
     "iopub.status.idle": "2022-06-02T14:56:21.278558Z",
     "shell.execute_reply": "2022-06-02T14:56:21.277378Z"
    },
    "papermill": {
     "duration": 15.261049,
     "end_time": "2022-06-02T14:56:21.281245",
     "exception": false,
     "start_time": "2022-06-02T14:56:06.020196",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "533 MATCHES!\n",
      "Running time:  5.419436454772949  s\n",
      "319 MATCHES!\n",
      "Running time:  2.6471190452575684  s\n",
      "49 MATCHES!\n",
      "Running time:  2.1064584255218506  s\n"
     ]
    }
   ],
   "source": [
    "plot = True\n",
    "F_dict = {}\n",
    "\n",
    "import time\n",
    "for i, row in enumerate(test_samples):\n",
    "    sample_id, batch_id, image_0_id, image_1_id = row\n",
    "    # Load the images.\n",
    "    st = time.time()\n",
    "\n",
    "    # _________________ loftr ___________________\n",
    "    \n",
    "    image_0, image_raw0, scale_w0, scale_h0 = load_torch_image(f'{src}/test_images/{batch_id}/{image_0_id}.png', device)\n",
    "    image_1, image_raw1, scale_w1, scale_h1 = load_torch_image(f'{src}/test_images/{batch_id}/{image_1_id}.png', device)\n",
    "    batch = {\"image0\": K.color.rgb_to_grayscale(image_0), \"image1\": K.color.rgb_to_grayscale(image_1)}\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        model_loftr.matcher(batch)\n",
    "    \n",
    "    mkpts0 = batch['mkpts0_f'].cpu().numpy()\n",
    "    mkpts1 = batch['mkpts1_f'].cpu().numpy()\n",
    "    \n",
    "    # rearrange original aspect ratio\n",
    "    mkpts0[:, 0] = mkpts0[:, 0] * (1/scale_w0)\n",
    "    mkpts0[:, 1] = mkpts0[:, 1] * (1/scale_h0)\n",
    "    mkpts0 = mkpts0.astype(np.int32)\n",
    "\n",
    "    mkpts1[:, 0] = mkpts1[:, 0] * (1/scale_w1)\n",
    "    mkpts1[:, 1] = mkpts1[:, 1] * (1/scale_h1)\n",
    "    mkpts1 = mkpts1.astype(np.int32)\n",
    "    \n",
    "    # _______________ superglue _________________\n",
    "    \n",
    "    image_1, inp_1, scales_1 = read_image(f'{src}/test_images/{batch_id}/{image_0_id}.png', device, resize, 0, resize_float)\n",
    "    image_2, inp_2, scales_2 = read_image(f'{src}/test_images/{batch_id}/{image_1_id}.png', device, resize, 0, resize_float)\n",
    "    \n",
    "    sg_pred = model_sg({\"image0\": inp_1, \"image1\": inp_2})\n",
    "    sg_pred = {k: v[0].detach().cpu().numpy() for k, v in sg_pred.items()}\n",
    "    sg_kpts1, sg_kpts2 = sg_pred[\"keypoints0\"], sg_pred[\"keypoints1\"]\n",
    "    sg_matches, sg_conf = sg_pred[\"matches0\"], sg_pred[\"matching_scores0\"]\n",
    "\n",
    "    sg_valid = sg_matches > -1\n",
    "    sg_mkpts0 = sg_kpts1[sg_valid]\n",
    "    sg_mkpts1 = sg_kpts2[sg_matches[sg_valid]]\n",
    "    sg_mconf = sg_conf[sg_valid]\n",
    "    \n",
    "    # rearrange original aspect ratio\n",
    "    # sg_mkpts0[:, 0] = sg_mkpts0[:, 0] * (1/scale_w0)\n",
    "    # sg_mkpts0[:, 1] = sg_mkpts0[:, 1] * (1/scale_h0)\n",
    "    # sg_mkpts0 = sg_mkpts0.astype(np.int32)\n",
    "\n",
    "    # sg_mkpts1[:, 0] = sg_mkpts1[:, 0] * (1/scale_w1)\n",
    "    # sg_mkpts1[:, 1] = sg_mkpts1[:, 1] * (1/scale_h1)\n",
    "    # sg_mkpts1 = sg_mkpts1.astype(np.int32)\n",
    "    \n",
    "    # __________________________________________\n",
    "\n",
    "    mkpts0 = np.append(mkpts0, sg_mkpts0, axis=0)\n",
    "    mkpts1 = np.append(mkpts1, sg_mkpts1, axis=0)\n",
    "    \n",
    "    if len(mkpts0) > 7:\n",
    "        F, inliers = cv2.findFundamentalMat(mkpts0, mkpts1, cv2.USAC_MAGSAC, 0.2, 0.9999, 100000)\n",
    "        inliers = inliers > 0\n",
    "        print(f'{inliers.sum()} MATCHES!')\n",
    "        assert F.shape == (3, 3), 'Malformed F?'\n",
    "        F_dict[sample_id] = F\n",
    "    else:\n",
    "        print('Less than 7 points')\n",
    "        F_dict[sample_id] = np.zeros((3, 3))\n",
    "        continue\n",
    "    gc.collect()\n",
    "    nd = time.time()    \n",
    "    if (i < 3) and plot:\n",
    "        print(\"Running time: \", nd - st, \" s\")\n",
    "        draw_LAF_matches(\n",
    "        KF.laf_from_center_scale_ori(torch.from_numpy(mkpts0).view(1,-1, 2),\n",
    "                                    torch.ones(mkpts0.shape[0]).view(1,-1, 1, 1),\n",
    "                                    torch.ones(mkpts0.shape[0]).view(1,-1, 1)),\n",
    "\n",
    "        KF.laf_from_center_scale_ori(torch.from_numpy(mkpts1).view(1,-1, 2),\n",
    "                                    torch.ones(mkpts1.shape[0]).view(1,-1, 1, 1),\n",
    "                                    torch.ones(mkpts1.shape[0]).view(1,-1, 1)),\n",
    "        torch.arange(mkpts0.shape[0]).view(-1,1).repeat(1,2),\n",
    "        K.tensor_to_image(image_raw0),\n",
    "        K.tensor_to_image(image_raw1),\n",
    "        inliers,\n",
    "        draw_dict={'inlier_color': (0.2, 1, 0.2),\n",
    "                   'tentative_color': None, \n",
    "                   'feature_color': (0.2, 0.5, 1), 'vertical': False})\n",
    "    \n",
    "with open('submission.csv', 'w') as f:\n",
    "    f.write('sample_id,fundamental_matrix\\n')\n",
    "    for sample_id, F in F_dict.items():\n",
    "        f.write(f'{sample_id},{FlattenMatrix(F)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27e69d2e",
   "metadata": {
    "papermill": {
     "duration": 0.007673,
     "end_time": "2022-06-02T14:56:21.297251",
     "exception": false,
     "start_time": "2022-06-02T14:56:21.289578",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 518.086008,
   "end_time": "2022-06-02T14:56:24.773246",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-06-02T14:47:46.687238",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}