Product_Recognition_on_Store_Shelves.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Product Recognition on Store Shelves\n",
    "\n",
    "- Baraghini Nicholas\n",
    "- Marini Luca"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import of the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step A - Multiple Product Detection:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load first scene image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load Scene\n",
    "scene_img = cv2.imread('./scenes/e1.png', cv2.COLOR_BGR2RGB)\n",
    "\n",
    "#Plot Scene\n",
    "plt.figure(figsize=(20, 10)) \n",
    "plt.imshow(cv2.cvtColor(scene_img, cv2.COLOR_BGR2RGB));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load first model image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load Model\n",
    "model_img = cv2.imread('./models/0.jpg', cv2.COLOR_BGR2RGB)\n",
    "\n",
    "#Plot Model\n",
    "plt.figure() \n",
    "plt.imshow(cv2.cvtColor(model_img, cv2.COLOR_BGR2RGB));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sift = cv2.xfeatures2d.SIFT_create()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_img = cv2.cvtColor(model_img, cv2.COLOR_BGR2GRAY)\n",
    "kp_model = sift.detect(model_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of keypoints \n",
    "print(len(kp_model))\n",
    "\n",
    "# Location of the keypoint\n",
    "print(kp_model[0].pt)\n",
    "\n",
    "# Scale\n",
    "print(kp_model[0].size)\n",
    "\n",
    "# Rotation\n",
    "print(kp_model[0].angle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_img_visualization = cv2.drawKeypoints(model_img, kp_model, None, flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)\n",
    "plt.imshow(cv2.cvtColor(model_img_visualization, cv2.COLOR_BGR2RGB))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scene_img = cv2.cvtColor(scene_img, cv2.COLOR_BGR2GRAY)\n",
    "kp_scene = sift.detect(scene_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scene_img_visualization = cv2.drawKeypoints(scene_img,kp_scene,None,flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)\n",
    "plt.imshow(cv2.cvtColor(scene_img_visualization, cv2.COLOR_BGR2RGB))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Plot Scene\n",
    "#plt.figure(figsize=(20, 10)) \n",
    "#plt.imshow(cv2.cvtColor(scene,cv2.COLOR_BGR2RGB))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kp_model, des_model = sift.compute(model_img, kp_model)\n",
    "kp_scene, des_scene = sift.compute(scene_img, kp_scene)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Number of model descriptors: \", len(des_model))\n",
    "print(\"Dimensionality of a SIFT descriptor: \", des_model[0].shape)\n",
    "print(\"Type of the descriptor: \", des_model[0].dtype)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining a function which find matches between keypoints belonging to the model, with keypoints belonging to the scene, and through a threshold collects all the worth considered matches in a list, which is then returned by the function itself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Matching(Model_Descriptors, Scene_Descriptors, Treshold = 0.45, k=2):\n",
    "    FLANN_INDEX_KDTREE = 1\n",
    "\n",
    "        # Defining parameters for algorithm \n",
    "    index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)\n",
    "\n",
    "        # Defining search params.\n",
    "        # checks=50 specifies the number of times the trees in the index should be recursively traversed.\n",
    "        # Higher values gives better precision, but also takes more time\n",
    "    search_params = dict(checks = 50)\n",
    "\n",
    "        # Initializing matcher\n",
    "    flann = cv2.FlannBasedMatcher(index_params, search_params)\n",
    "        # Matching and finding the 2 closest elements for each query descriptor.\n",
    "    matches = flann.knnMatch(Model_Descriptors, Scene_Descriptors, k)\n",
    "        #defining an array containing all the matches that results to be considered \"good\" matches applying a certain treshold \n",
    "    good = []\n",
    "    for m,n in matches:\n",
    "        if m.distance < Treshold * n.distance: #  if m.distance/n.distance < Threshold:\n",
    "            good.append(m)\n",
    "            \n",
    "    return good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "good = Matching(des_model, des_scene)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checking if we found enough matching\n",
    "MIN_MATCH_COUNT = 30\n",
    "print(len(good))\n",
    "if len(good)>MIN_MATCH_COUNT:\n",
    "    \n",
    "    # building the corrspondences arrays of good matches\n",
    "    src_pts = np.float32([ kp_model[m.queryIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "    dst_pts = np.float32([ kp_scene[m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "    \n",
    "    # Using RANSAC to estimate a robust homography. \n",
    "    # It returns the homography M and a mask for the discarded points\n",
    "    M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)\n",
    "    \n",
    "    # Mask of discarded point used in visualization\n",
    "    matchesMask = mask.ravel().tolist()\n",
    "    \n",
    "    # Corners of the query image\n",
    "    h,w = model_img.shape[:2]\n",
    "    pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2) # myc: these are the 4 corners of the query image\n",
    "    \n",
    "    # Projecting the corners into the train image\n",
    "    dst = cv2.perspectiveTransform(pts,M) # myc: you project the corners by using the homography. M is the homography\n",
    "\n",
    "    # Drawing the bounding box\n",
    "    scene_img = cv2.polylines(scene_img, [np.int32(dst)],True, (0, 255, 0), 3, cv2.LINE_AA)\n",
    "    \n",
    "else:\n",
    "    print( \"Not enough matches are found - {}/{}\".format(len(good), MIN_MATCH_COUNT) )\n",
    "    matchesMask = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drawing the matches\n",
    "draw_params = dict(matchColor = (0,255,0), # draw matches in green color\n",
    "                   singlePointColor = None, # not draw keypoints only matching lines\n",
    "                   matchesMask = matchesMask, # draw only inliers\n",
    "                   flags = 2) # not draw keypoints only lines\n",
    "img3 = cv2.drawMatches(model_img, kp_model, scene_img, kp_scene, good, None, **draw_params)\n",
    "plt.imshow(cv2.cvtColor(img3, cv2.COLOR_BGR2RGB))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(cv2.cvtColor(scene_img, cv2.COLOR_BGR2RGB))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creation of some dictionaries that contain all the keypoint, descriptors and images that we will need during stepA, so that they are loaded once and at the beginning, to save later computational cost."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_OF_MODELS = 27\n",
    "\n",
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "CENTER_INDEX     = 3\n",
    "V_INDEX          = 4\n",
    "TRESHOLD_INDEX = 5\n",
    "\n",
    "# Dictionary that contains the image, all keypoints and descriptors for each model images\n",
    "model_images_features = {}\n",
    "mean_of_model_intensities_r_g_b = {}\n",
    "\n",
    "for i in range(NUM_OF_MODELS):\n",
    "    model_img = cv2.imread('./models/{}.jpg'.format(i), cv2.COLOR_BGR2GRAY)#cv2.COLOR_BGR2RGB)\n",
    "    kp_model = sift.detect(model_img)\n",
    "    kp_model, des_model = sift.compute(model_img, kp_model)\n",
    "    model_images_features[i] = [model_img, kp_model, des_model]\n",
    "    \n",
    "    b,g,r = cv2.split(model_images_features[i][IMAGE_INDEX])\n",
    "    # save the mean of the intensities (divided per channel) for every model image\n",
    "    mean_of_model_intensities_r_g_b[i] = [np.mean(r), np.mean(g), np.mean(b)]\n",
    "    # print(mean_of_model_intensities_r_g_b[i][0], mean_of_model_intensities_r_g_b[i][1], mean_of_model_intensities_r_g_b[i][2])\n",
    "    # print(model_images_features[i][IMAGE_INDEX][0].shape)\n",
    "    # print(mean_of_model_intensities[i])\n",
    "\n",
    "    \n",
    "    # plt.imshow(cv2.cvtColor(model_images_features[str(i)][0], cv2.COLOR_BGR2RGB))\n",
    "    # plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenes_to_test = [1, 2, 3, 4, 5]\n",
    "\n",
    "# Dictionary that contains the image, all keypoints and descriptors for each easy scene image\n",
    "e_scenes_images_features = {}\n",
    "\n",
    "for i in scenes_to_test:\n",
    "    scene_img = cv2.imread('./scenes/e{}.png'.format(i), cv2.COLOR_BGR2RGB)\n",
    "    kp_scene = sift.detect(scene_img)\n",
    "    kp_scene, des_scene = sift.compute(scene_img, kp_scene)\n",
    "    e_scenes_images_features[i] = [scene_img, kp_scene, des_scene]\n",
    "    # scene_img_visualization = cv2.drawKeypoints(scene_img,kp_e_scemes_images['1'],None,flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)\n",
    "    # plt.imshow(cv2.cvtColor(e_scenes_images_features[str(i)][0], cv2.COLOR_BGR2RGB))\n",
    "    # plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function that will output the position in the image reference system of each instance "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_result(e_results, idx_of_scene):\n",
    "    for i in models_to_test:\n",
    "        print('Product {} - {} instance/s found:'.format(i, e_results[idx_of_scene][i]['count']))\n",
    "        n = 0\n",
    "        if e_results[idx_of_scene][i].get('width', None):\n",
    "            n += 1\n",
    "            print('\\tInstance {} position: {}, width: {}px, height: {}px'.format(n, e_results[idx_of_scene][i]['pos'], e_results[idx_of_scene][i]['width'], e_results[idx_of_scene][i]['height']))\n",
    "    print('_' * 80 + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distance_2_points(A, B):\n",
    "    return math.sqrt( np.power(A[0] - B[0], 2) +  np.power(A[1] - B[1], 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that plots the final merged bounding boxes\n",
    "def plot_final_bounding_boxes(img, final_corners_of_bounding_boxes, difficulty_scenes_images_features, j, thickness=20):\n",
    "    print('Final corners of bounding boxes:')\n",
    "    print(final_corners_of_bounding_boxes)\n",
    "    \n",
    "    if img is None:\n",
    "        img = np.copy(difficulty_scenes_images_features[j][IMAGE_INDEX])\n",
    "    print(img.shape)\n",
    "\n",
    "    \n",
    "    # print final bounding boxes on an image\n",
    "    for top_left_corner, bottom_right_corner in final_corners_of_bounding_boxes:\n",
    "        scene_img_with_FINAL_bounding_boxes = cv2.rectangle(img, top_left_corner, bottom_right_corner, (0,255,0), thickness)\n",
    "\n",
    "    \n",
    "    \n",
    "    # plot image with final bounding boxes (with bounding boxes that do not overlap each other)\n",
    "    if(len(final_corners_of_bounding_boxes) > 0):\n",
    "        print('  - Found {} instances of model {} in scene {}'.format(len(final_corners_of_bounding_boxes),i,j))\n",
    "        # plotting the bounding box\n",
    "        plt.imshow(cv2.cvtColor(scene_img_with_FINAL_bounding_boxes, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "        \n",
    "        return scene_img_with_FINAL_bounding_boxes\n",
    "    else:\n",
    "        print('  - Model {} NOT found in the scene {}'.format(i,j))\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that solves exceeding dimensions of bounding_boxes in scene\n",
    "def solve_exceeding_dimensions_of_bounding_boxes_in_scene(final_corners_of_bounding_boxes, difficulty_scenes_images_features):\n",
    "    final_corners_of_bounding_boxes_without_exceeding_dimensions = []\n",
    "    # adjust bounding boxes that go out the dimensions of the scene image\n",
    "    scene_height = difficulty_scenes_images_features[j][IMAGE_INDEX].shape[0]\n",
    "    scene_width = difficulty_scenes_images_features[j][IMAGE_INDEX].shape[1]\n",
    "\n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes):\n",
    "        fin_top_left_corner = list(fin_top_left_corner)\n",
    "        fin_bottom_right_corner = list(fin_bottom_right_corner)\n",
    "        # top left corner that goes out on the left of the scene image\n",
    "        if fin_top_left_corner[0] < 0:\n",
    "            fin_top_left_corner[0] = 0\n",
    "        # bottom right corner that goes out on the left of the scene image\n",
    "        if fin_bottom_right_corner[0] < 0:\n",
    "            fin_bottom_right_corner[0] = 0\n",
    "        # top left corner that goes out on the right of the scene image\n",
    "        if fin_top_left_corner[0] > scene_width:\n",
    "            fin_top_left_corner[0] = scene_width\n",
    "        # bottom right corner that goes out on the right of the scene image\n",
    "        if fin_bottom_right_corner[0] > scene_width:\n",
    "            fin_bottom_right_corner[0] = scene_width\n",
    "        # top left corner that goes out on the top of the scene image\n",
    "        if fin_top_left_corner[1] < 0:\n",
    "            fin_top_left_corner[1] = 0\n",
    "        # bottom right corner that goes out on the top of the scene image\n",
    "        if fin_bottom_right_corner[1] < 0:\n",
    "            fin_bottom_right_corner[1] = 0  \n",
    "        # top left corner that goes out on the bottom of the scene image\n",
    "        if fin_top_left_corner[1] > scene_height:\n",
    "            fin_top_left_corner[1] = scene_height\n",
    "        # bottom right corner that goes out on the bottom of the scene image\n",
    "        if fin_bottom_right_corner[1] > scene_height:\n",
    "            fin_bottom_right_corner[1] = scene_height\n",
    "\n",
    "        fin_top_left_corner = tuple(fin_top_left_corner)\n",
    "        fin_bottom_right_corner = tuple(fin_bottom_right_corner)\n",
    "        final_corners_of_bounding_boxes_without_exceeding_dimensions.append([ (int(fin_top_left_corner[0]), \n",
    "                                                                               int(fin_top_left_corner[1])), \n",
    "                                                                             (int(fin_bottom_right_corner[0]), \n",
    "                                                                              int(fin_bottom_right_corner[1]))])\n",
    "    return final_corners_of_bounding_boxes_without_exceeding_dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve_color_problem(final_corners_of_bounding_boxes_without_exceeding_dimensions, difficulty_scenes_images_features, j):\n",
    "    final_corners_of_bounding_boxes_after_color_problem = []\n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes_without_exceeding_dimensions):\n",
    "    \n",
    "        # print('fin top left: ', fin_top_left_corner)\n",
    "        # print('fin bottom right: ', fin_bottom_right_corner)\n",
    "        # print(m_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "        #                                                           fin_top_left_corner[0]:fin_bottom_right_corner[0]].shape)\n",
    "\n",
    "        # r, g , b intensities of current bounding box found in scene \n",
    "        b_b_box,g_b_box,r_b_box = cv2.split(\n",
    "            difficulty_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                   fin_top_left_corner[0]:fin_bottom_right_corner[0]])\n",
    "        \n",
    "        # r, g , b mean intensities of current bounding box found in scene\n",
    "        mean_r_b_box = np.mean(r_b_box)\n",
    "        mean_g_b_box = np.mean(g_b_box)\n",
    "        mean_b_b_box = np.mean(b_b_box)\n",
    "\n",
    "        # compute the difference of intensity by using the 3 channels r,g,b of bounding box and model image\n",
    "        # print('mean_r_b_box: ', mean_r_b_box)\n",
    "        # print('mean_g_b_box: ', mean_g_b_box)\n",
    "        # print('mean_b_b_box: ', mean_b_b_box)\n",
    "\n",
    "        diff_r = np.absolute(mean_of_model_intensities_r_g_b[i][0] - mean_r_b_box)\n",
    "        diff_g = np.absolute(mean_of_model_intensities_r_g_b[i][1] - mean_g_b_box)\n",
    "        diff_b = np.absolute(mean_of_model_intensities_r_g_b[i][2] - mean_b_b_box)\n",
    "\n",
    "        print('diff_r: ', diff_r)\n",
    "        print('diff_g: ', diff_g)\n",
    "        print('diff_b: ', diff_b)\n",
    "\n",
    "        # I consider a bounding box as good if and only if the difference of the intiensities from the model,\n",
    "        # in all 3 channels, are below a certain treshold\n",
    "        if diff_r <= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES and diff_g <= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES and \\\n",
    "        diff_b <= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES:\n",
    "            final_corners_of_bounding_boxes_after_color_problem.append([ (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                                                        (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))])\n",
    "            \n",
    "    return final_corners_of_bounding_boxes_after_color_problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that split an image into (n_bins_heigth x n_bins_width) bins and returns a dictonary\n",
    "# that contains the means of the 3 color channels of the (n_bins_heigth x n_bins_width) bins\n",
    "def split_image_into_N_x_M_bins_with_intensity_means(image, n_bins_width = 3, n_bins_heigth = 4):\n",
    "    img_N_x_M_bins = {}\n",
    "    \n",
    "    img2 = np.copy(image)\n",
    "    \n",
    "    img_width = image.shape[1]\n",
    "    img_height = image.shape[0]\n",
    "    \n",
    "    step_width = int(img_width / n_bins_width)\n",
    "    step_height = int(img_height / n_bins_heigth)\n",
    "    \n",
    "    r = 0\n",
    "    c = 0\n",
    "    \n",
    "    print('img_height: ', img_height)\n",
    "    print('step_height: ', step_height)\n",
    "    if img_height != 0 and step_height != 0 and img_width != 0 and step_width != 0:\n",
    "        for row in np.arange(0, img_height, step_height):\n",
    "            c = 0\n",
    "            cv2.line(img2,(0, row),(img_width, row),(0,0,0),3) \n",
    "            for col in np.arange(0, img_width, step_width):\n",
    "                # print('row {} col {}'.format(r,c))\n",
    "                if row + 2 * step_height > img_height and col + 2 * step_width > img_width:\n",
    "                    partial_r_channel, partial_g_channel, partial_b_channel = cv2.split(image[row:, col:])\n",
    "\n",
    "                elif row + 2 * step_height > img_height:\n",
    "                    partial_r_channel, partial_g_channel, partial_b_channel = cv2.split(image[row:, col : col + step_width])\n",
    "\n",
    "                elif col + 2 * step_width > img_width:\n",
    "                    partial_r_channel, partial_g_channel, partial_b_channel = cv2.split(image[row : row + step_height, col :])\n",
    "                else:\n",
    "                    partial_r_channel, partial_g_channel, partial_b_channel = cv2.split(\n",
    "                        image[row : row + step_height, col : col + step_width])\n",
    "\n",
    "                cv2.line(img2,(col, 0),(col, img_height),(0,0,0),3) \n",
    "                if r < n_bins_heigth and c < n_bins_width:\n",
    "                    # save means of the 3 channels (r,g,b) of each bin\n",
    "                    img_N_x_M_bins[r, c] = (np.mean(partial_r_channel), np.mean(partial_g_channel), np.mean(partial_b_channel))\n",
    "                    # IF YOU WANT TO VISUALIZE EACH BIN UNCOMMENT THE FOLLOWING LINES:\n",
    "                    # plot each bin in blue channel color\n",
    "                    # plt.imshow(cv2.cvtColor(partial_b_channel, cv2.COLOR_BGR2RGB))\n",
    "                    # plt.show()\n",
    "\n",
    "                    # salva le medie e non le immagini dei bins\n",
    "\n",
    "                c += 1\n",
    "            r += 1\n",
    "        plt.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()  \n",
    "    return img_N_x_M_bins\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve_color_problem_with_N_x_M_bins(final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                    difficulty_scenes_images_features, i, j, N=3, M=4, \n",
    "                                    COLOR_DIFF_IN_SINGLE_CHANNEL_TRES=50):\n",
    "    final_corners_of_bounding_boxes_after_color_problem = []\n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes_without_exceeding_dimensions):\n",
    "    \n",
    "        means_bins_model_img = split_image_into_N_x_M_bins_with_intensity_means(\n",
    "            model_images_features[i][IMAGE_INDEX], \n",
    "            N, M)\n",
    "        \n",
    "        print(fin_top_left_corner[1], fin_top_left_corner[0], fin_bottom_right_corner[1], fin_bottom_right_corner[0])\n",
    "        \n",
    "        means_bins_scene_img = split_image_into_N_x_M_bins_with_intensity_means(\n",
    "            difficulty_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                   fin_top_left_corner[0]:fin_bottom_right_corner[0]], \n",
    "            N, M)\n",
    "    \n",
    "        if not means_bins_scene_img or not means_bins_scene_img:\n",
    "            return []\n",
    "    \n",
    "        good = True\n",
    "        \n",
    "        # cycle the model dictionary and get the diff of the means by getting the keys of the scene dictionary\n",
    "        for k, v in means_bins_model_img.items():\n",
    "            means_k_scene = means_bins_scene_img.get(k)\n",
    "            # I consider a bounding box as good if and only if all the difference of the intiensities from the \n",
    "            # bins of the model, in all 3 channels, are below a certain treshold\n",
    "            diff_r = np.absolute(v[0] - means_k_scene[0])\n",
    "            diff_g = np.absolute(v[1] - means_k_scene[1])\n",
    "            diff_b = np.absolute(v[2] - means_k_scene[2])\n",
    "            \n",
    "            print('Bin ({}, {}):'.format(k[0], k[1]))\n",
    "            print('  - diff_r: ', diff_r)\n",
    "            print('  - diff_g: ', diff_g)\n",
    "            print('  - diff_b: ', diff_b)\n",
    "            \n",
    "            # PRINT ALL THE DIFFERENCES\n",
    "            # print('diff_r: ', diff_r)\n",
    "            # print('diff_g: ', diff_g)\n",
    "            # print('diff_b: ', diff_b)\n",
    "            \n",
    "            if diff_r >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES or diff_g >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES or \\\n",
    "                diff_b >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES:\n",
    "                print('___________________NO GOOD___________________')\n",
    "                good = False\n",
    "            \n",
    "        if good:\n",
    "            final_corners_of_bounding_boxes_after_color_problem.append([ (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                                                        (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))])\n",
    "            \n",
    "    \n",
    "    return final_corners_of_bounding_boxes_after_color_problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dictionary that will contain the output to be printed\n",
    "e_results = {}    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "\n",
    "models_to_test = [0, 1, 11, 19, 24, 26, 25] # [0,11,25]\n",
    "scenes_to_test = [1,2,3,4,5] # [2,3,4]\n",
    "MATCHING_TRESHOLD = 0.45\n",
    "\n",
    "MIN_NUM_OF_MATCHES = 15\n",
    "\n",
    "COLOR_DIFF_IN_SINGLE_CHANNEL_TRES = 77\n",
    "\n",
    "N_BINS_ON_WIDTH = 3\n",
    "\n",
    "M_BINS_ON_HEIGHT = 4\n",
    "\n",
    "final_scene_images_with_bb = {}\n",
    "\n",
    "for j in scenes_to_test:\n",
    "    final_scene_images_with_bb[j] = np.copy(e_scenes_images_features[j][IMAGE_INDEX])\n",
    "    e_results[j] = {}\n",
    "    for i in models_to_test:\n",
    "        print('_' * 80 + '\\n')\n",
    "        \n",
    "        e_results[j][i] = {}\n",
    "        e_results[j][i]['count'] = 0\n",
    "        \n",
    "        good = Matching(model_images_features[i][DESCRIPTOR_INDEX], e_scenes_images_features[j][DESCRIPTOR_INDEX], \n",
    "                        Treshold = MATCHING_TRESHOLD, k=2)#flann.knnMatch(model_images_features[i][DESCRIPTOR_INDEX], e_scenes_images_features[j][DESCRIPTOR_INDEX], k=2)\n",
    "        # Checking if we found enough matching\n",
    "        MIN_MATCH_COUNT = MIN_NUM_OF_MATCHES\n",
    "        \n",
    "        # If enough matches => the model is in the scene image\n",
    "        if len(good) >= MIN_MATCH_COUNT:\n",
    "            \n",
    "            \n",
    "            \n",
    "            # building the corrspondences arrays of good matches\n",
    "            src_pts = np.float32([ model_images_features[i][KEYPOINT_INDEX][m.queryIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "            dst_pts = np.float32([ e_scenes_images_features[j][KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "            \n",
    "            # Using RANSAC to estimate a robust homography. \n",
    "            # It returns the homography M and a mask for the discarded points\n",
    "            M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)\n",
    "\n",
    "            # Mask of discarded point used in visualization\n",
    "            matchesMask = mask.ravel().tolist()\n",
    "\n",
    "            # Corners of the query image\n",
    "            h,w = model_images_features[i][IMAGE_INDEX].shape[:2]\n",
    "            pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2) # myc: these are the 4 corners of the query image\n",
    "            \n",
    "            # Projecting the corners into the train image\n",
    "            dst = cv2.perspectiveTransform(pts,M) # myc: you project the corners by using the homography. M is the homography          \n",
    "            \n",
    "            # Drawing the bounding box\n",
    "            #scene_img = cv2.polylines(e_scenes_images_features[j][IMAGE_INDEX], [np.int32(dst)],\n",
    "                                      #True, (0, 255, 0), 3, cv2.LINE_AA)\n",
    "            #e_results[j][i]['pos']\n",
    "            top_left_bound_box_corner = dst[0][0]\n",
    "            bottom_left_bound_box_corner = dst[1][0]\n",
    "            top_right_bound_box_corner = dst[3][0]\n",
    "            bottom_right_bound_box_corner = dst[2][0]\n",
    "            # print(top_left_bound_box_corner)\n",
    "            \n",
    "            \n",
    "            # compute width and height of the current bounding box\n",
    "            width_of_bounding_box = int(distance_2_points(top_left_bound_box_corner, top_right_bound_box_corner))\n",
    "            height_of_bounding_box = int(distance_2_points(top_left_bound_box_corner, bottom_left_bound_box_corner))\n",
    "            \n",
    "            # I consider the current bounding box as good only if it has a shape of a rectangle, \n",
    "            # with an height > of the width, as the cereal boxes\n",
    "            if width_of_bounding_box < height_of_bounding_box:\n",
    "            \n",
    "                final_corners_of_bounding_boxes = []\n",
    "                final_corners_of_bounding_boxes.append([top_left_bound_box_corner, bottom_right_bound_box_corner])\n",
    "                final_corners_of_bounding_boxes_without_exceeding_dimensions = solve_exceeding_dimensions_of_bounding_boxes_in_scene(final_corners_of_bounding_boxes, e_scenes_images_features)\n",
    "\n",
    "                \n",
    "                # solve color problem with bins\n",
    "                \n",
    "                final_corners_of_bounding_boxes_after_color_problem = solve_color_problem_with_N_x_M_bins(\n",
    "                    final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                    e_scenes_images_features, i, j, N=N_BINS_ON_WIDTH, M=M_BINS_ON_HEIGHT, \n",
    "                    COLOR_DIFF_IN_SINGLE_CHANNEL_TRES=COLOR_DIFF_IN_SINGLE_CHANNEL_TRES)\n",
    "                \n",
    "\n",
    "                \n",
    "                # solve color problem without bins (as an unique image)\n",
    "                #final_corners_of_bounding_boxes_after_color_problem = solve_color_problem(\n",
    "                #    final_corners_of_bounding_boxes_without_exceeding_dimensions, e_scenes_images_features, j)\n",
    "\n",
    "                if len(final_corners_of_bounding_boxes_after_color_problem) > 0:\n",
    "                    for [fin_top_left_corner, fin_bottom_right_corner] in final_corners_of_bounding_boxes_after_color_problem:\n",
    "                        e_results[j][i]['count'] += 1\n",
    "\n",
    "                        # Save width and height measures of the bounding box\n",
    "                        e_results[j][i]['width'] = width_of_bounding_box\n",
    "                        e_results[j][i]['height'] = height_of_bounding_box\n",
    "\n",
    "                        # Save the position of the bounding box\n",
    "                        e_results[j][i]['pos'] = (int(fin_top_left_corner[0]), int(fin_top_left_corner[1]))\n",
    "\n",
    "\n",
    "                print('COLOR CORRECTION DONE:')\n",
    "                img_bb = plot_final_bounding_boxes(final_scene_images_with_bb[j], final_corners_of_bounding_boxes_after_color_problem, \n",
    "                                          e_scenes_images_features, j)\n",
    "                if img_bb is not None:\n",
    "                    final_scene_images_with_bb[j] = img_bb\n",
    "            \n",
    "        else:\n",
    "            print( \"Not enough matches are found - {}/{}\".format(len(good), MIN_MATCH_COUNT) )\n",
    "            matchesMask = None\n",
    "\n",
    "        # Drawing the matches\n",
    "        draw_params = dict(matchColor = (0,255,0), # draw matches in green color\n",
    "                           singlePointColor = None, # not draw keypoints only matching lines\n",
    "                           matchesMask = matchesMask, # draw only inliers\n",
    "                           flags = 2) # not draw keypoints only lines\n",
    "        img3 = cv2.drawMatches(model_images_features[i][IMAGE_INDEX], \n",
    "                               model_images_features[i][KEYPOINT_INDEX], \n",
    "                               e_scenes_images_features[j][IMAGE_INDEX], \n",
    "                               e_scenes_images_features[j][KEYPOINT_INDEX],\n",
    "                               good, \n",
    "                               None,  \n",
    "                               **draw_params)\n",
    "        plt.imshow(cv2.cvtColor(img3, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that plots the final merged bounding boxes\n",
    "def plot_dark_area_bounding_boxes(img, final_corners_of_bounding_boxes, difficulty_scenes_images_features, j):\n",
    "    print('Final corners of bounding boxes:')\n",
    "    print(final_corners_of_bounding_boxes)\n",
    "    \n",
    "    \n",
    "    # print final bounding boxes on an image\n",
    "    for top_left_corner, bottom_right_corner in final_corners_of_bounding_boxes:\n",
    "        scene_img_with_FINAL_bounding_boxes = cv2.rectangle(img, top_left_corner, bottom_right_corner, (0,0,0), -1)\n",
    "\n",
    "    \n",
    "    \n",
    "    # plot image with final bounding boxes (with bounding boxes that do not overlap each other)\n",
    "    if(len(final_corners_of_bounding_boxes) > 0):\n",
    "        print('  - Found {} instances of model {} in scene {}'.format(len(final_corners_of_bounding_boxes),i,j))\n",
    "        # plotting the bounding box\n",
    "        plt.imshow(cv2.cvtColor(scene_img_with_FINAL_bounding_boxes, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "        \n",
    "        return scene_img_with_FINAL_bounding_boxes\n",
    "    else:\n",
    "        print('  - Model {} NOT found in the scene {}'.format(i,j))\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "\n",
    "models_to_test = [0,1,11, 19, 24, 26, 25] #[1] #, 1, 11, 19, 24, 26, 25] # [0,11,25]\n",
    "scenes_to_test = [1,2,3,4,5] # [2,3,4]\n",
    "MATCHING_TRESHOLD = 0.45\n",
    "\n",
    "MIN_NUM_OF_MATCHES = 18\n",
    "\n",
    "COLOR_DIFF_IN_SINGLE_CHANNEL_TRES = 79\n",
    "\n",
    "N_BINS_ON_WIDTH = 3\n",
    "\n",
    "M_BINS_ON_HEIGHT = 4\n",
    "\n",
    "final_scene_images_with_bb = {}\n",
    "\n",
    "scene_single_model_features = {}\n",
    "\n",
    "for j in scenes_to_test:\n",
    "    final_scene_images_with_bb[j] = np.copy(e_scenes_images_features[j][IMAGE_INDEX])\n",
    "    e_results[j] = {}\n",
    "    for i in models_to_test:\n",
    "        scene_single_model = np.copy(e_scenes_images_features[j][IMAGE_INDEX])\n",
    "        print('_' * 80 + '\\n')\n",
    "        print('Finding model {} in scene {}'.format(i,j))\n",
    "        \n",
    "        e_results[j][i] = {}\n",
    "        e_results[j][i]['count'] = 0\n",
    "        \n",
    "         # Checking if we found enough matching\n",
    "        MIN_MATCH_COUNT = MIN_NUM_OF_MATCHES\n",
    "        \n",
    "        finding = True\n",
    "        \n",
    "        # If enough matches => the model is in the scene image\n",
    "        while finding:\n",
    "            \n",
    "            kp_scene = sift.detect(scene_single_model)\n",
    "            kp_scene, des_scene = sift.compute(scene_single_model, kp_scene)\n",
    "            scene_single_model_features[j] = [scene_single_model, kp_scene, des_scene]\n",
    "            \n",
    "            good = Matching(model_images_features[i][DESCRIPTOR_INDEX], scene_single_model_features[j][DESCRIPTOR_INDEX], \n",
    "                            Treshold = MATCHING_TRESHOLD, k=2)#flann.knnMatch(model_images_features[i][DESCRIPTOR_INDEX], e_scenes_images_features[j][DESCRIPTOR_INDEX], k=2)\n",
    "            print(len(good))\n",
    "            \n",
    "            if len(good) < MIN_MATCH_COUNT:\n",
    "                finding = False\n",
    "                \n",
    "            \n",
    "            if len(good) >= MIN_MATCH_COUNT:\n",
    "                \n",
    "                \n",
    "\n",
    "                # building the corrspondences arrays of good matches\n",
    "                src_pts = np.float32([ model_images_features[i][KEYPOINT_INDEX][m.queryIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "                dst_pts = np.float32([ scene_single_model_features[j][KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "\n",
    "                # Using RANSAC to estimate a robust homography. \n",
    "                # It returns the homography M and a mask for the discarded points\n",
    "                M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)\n",
    "\n",
    "                # Mask of discarded point used in visualization\n",
    "                matchesMask = mask.ravel().tolist()\n",
    "\n",
    "                # Corners of the query image\n",
    "                h,w = model_images_features[i][IMAGE_INDEX].shape[:2]\n",
    "                pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2) # myc: these are the 4 corners of the query image\n",
    "\n",
    "                # Projecting the corners into the train image\n",
    "                dst = cv2.perspectiveTransform(pts,M) # myc: you project the corners by using the homography. M is the homography          \n",
    "\n",
    "                # Drawing the bounding box\n",
    "                #scene_img = cv2.polylines(e_scenes_images_features[j][IMAGE_INDEX], [np.int32(dst)],\n",
    "                                          #True, (0, 255, 0), 3, cv2.LINE_AA)\n",
    "                #e_results[j][i]['pos']\n",
    "                top_left_bound_box_corner = dst[0][0]\n",
    "                bottom_left_bound_box_corner = dst[1][0]\n",
    "                top_right_bound_box_corner = dst[3][0]\n",
    "                bottom_right_bound_box_corner = dst[2][0]\n",
    "                # print(top_left_bound_box_corner)\n",
    "\n",
    "\n",
    "                # compute width and height of the current bounding box\n",
    "                width_of_bounding_box = int(distance_2_points(top_left_bound_box_corner, top_right_bound_box_corner))\n",
    "                height_of_bounding_box = int(distance_2_points(top_left_bound_box_corner, bottom_left_bound_box_corner))\n",
    "\n",
    "                # I consider the current bounding box as good only if it has a shape of a rectangle, \n",
    "                # with an height > of the width, as the cereal boxes\n",
    "                if width_of_bounding_box < height_of_bounding_box:\n",
    "\n",
    "                    final_corners_of_bounding_boxes = []\n",
    "                    final_corners_of_bounding_boxes.append([top_left_bound_box_corner, bottom_right_bound_box_corner])\n",
    "                    final_corners_of_bounding_boxes_without_exceeding_dimensions = solve_exceeding_dimensions_of_bounding_boxes_in_scene(final_corners_of_bounding_boxes, e_scenes_images_features)\n",
    "\n",
    "\n",
    "                    # solve color problem with bins\n",
    "\n",
    "                    final_corners_of_bounding_boxes_after_color_problem = solve_color_problem_with_N_x_M_bins(\n",
    "                        final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                        e_scenes_images_features, i, j, N=N_BINS_ON_WIDTH, M=M_BINS_ON_HEIGHT, \n",
    "                        COLOR_DIFF_IN_SINGLE_CHANNEL_TRES=COLOR_DIFF_IN_SINGLE_CHANNEL_TRES)\n",
    "\n",
    "\n",
    "\n",
    "                    # solve color problem without bins (as an unique image)\n",
    "                    #final_corners_of_bounding_boxes_after_color_problem = solve_color_problem(\n",
    "                    #    final_corners_of_bounding_boxes_without_exceeding_dimensions, e_scenes_images_features, j)\n",
    "\n",
    "                    if len(final_corners_of_bounding_boxes_after_color_problem) > 0:\n",
    "                        finding = False\n",
    "                        for [fin_top_left_corner, fin_bottom_right_corner] in final_corners_of_bounding_boxes_after_color_problem:\n",
    "                            e_results[j][i]['count'] += 1\n",
    "\n",
    "                            # Save width and height measures of the bounding box\n",
    "                            e_results[j][i]['width'] = width_of_bounding_box\n",
    "                            e_results[j][i]['height'] = height_of_bounding_box\n",
    "\n",
    "                            # Save the position of the bounding box\n",
    "                            e_results[j][i]['pos'] = (int(fin_top_left_corner[0]), int(fin_top_left_corner[1]))\n",
    "\n",
    "                    else:\n",
    "                        print('NO CORNERS AFTER COLOR CORRECTION')\n",
    "                        for [fin_top_left_corner, fin_bottom_right_corner] in final_corners_of_bounding_boxes_without_exceeding_dimensions:\n",
    "                            scene_single_model = plot_dark_area_bounding_boxes(scene_single_model, \n",
    "                                                                               final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                                                               e_scenes_images_features, j)\n",
    "                        #plt.imshow(cv2.cvtColor(scene_single_model, cv2.COLOR_BGR2RGB))\n",
    "                        #plt.show()\n",
    "\n",
    "                    print('COLOR CORRECTION DONE:')\n",
    "                    img_bb = plot_final_bounding_boxes(final_scene_images_with_bb[j], final_corners_of_bounding_boxes_after_color_problem, \n",
    "                                              e_scenes_images_features, j)\n",
    "                    if img_bb is not None:\n",
    "                        final_scene_images_with_bb[j] = img_bb\n",
    "\n",
    "            else:\n",
    "                print( \"Not enough matches are found - {}/{}\".format(len(good), MIN_MATCH_COUNT) )\n",
    "                matchesMask = None\n",
    "\n",
    "            # Drawing the matches\n",
    "            draw_params = dict(matchColor = (0,255,0), # draw matches in green color\n",
    "                               singlePointColor = None, # not draw keypoints only matching lines\n",
    "                               matchesMask = matchesMask, # draw only inliers\n",
    "                               flags = 2) # not draw keypoints only lines\n",
    "            img3 = cv2.drawMatches(model_images_features[i][IMAGE_INDEX], \n",
    "                                   model_images_features[i][KEYPOINT_INDEX], \n",
    "                                   e_scenes_images_features[j][IMAGE_INDEX], \n",
    "                                   e_scenes_images_features[j][KEYPOINT_INDEX],\n",
    "                                   good, \n",
    "                                   None,  \n",
    "                                   **draw_params)\n",
    "            #plt.imshow(cv2.cvtColor(img3, cv2.COLOR_BGR2RGB))\n",
    "            #plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final output of StepA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in scenes_to_test:\n",
    "    plt.imshow(cv2.cvtColor(final_scene_images_with_bb[j], cv2.COLOR_BGR2RGB))\n",
    "    plt.show()\n",
    "    print_result(e_results, j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Step B - Multiple Object Detection\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Computation of the center belonging to the model image.\n",
    " Knowing the model image has a rectangular shape, the center coordinates of the cereal box can be taken by halfing the vertical and horizontal dimensions of the model image; So definig a reference system with origin the top-left corner of the model image considering the vertical direction represented by the x-axis and the horizontal one represented by the y-axis then the center G of each model will be found in the position (height/2, width/2).\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Definition of the function which find the model center and stores it in the dictionary \"model_images_features\"\n",
    "def InstanceCenter(model_img,model_images_features, i, CENTER_INDEX=3):\n",
    "    height, width, channels = model_img.shape\n",
    "    v = int(height/2) # vertical coordinate of the image center\n",
    "    h = int(width/2)  # horizontal position of the image center\n",
    "    G = np.array([h, v]) # Defnition of the position center of the model\n",
    "    if len( model_images_features[i])<=CENTER_INDEX:\n",
    "        model_images_features[i].append(G) # Updating the model feature dictionary appending the information of the center\n",
    "    else :\n",
    "        #in the case G is already present in the model feature dictionary then update the G alreary there\n",
    "        model_images_features[i][CENTER_INDEX] = G"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Definition of the dictionary containing the informations of the scenes selected for this section:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenes_to_test = [1, 2, 3, 4, 5]\n",
    "\n",
    "# Dictionary that contains the image, all keypoints and descriptors for each easy scene image\n",
    "m_scenes_images_features = {}\n",
    "\n",
    "for i in scenes_to_test:\n",
    "    scene_img = cv2.imread('./scenes/m{}.png'.format(i), cv2.COLOR_BGR2RGB)\n",
    "    kp_scene = sift.detect(scene_img)\n",
    "    kp_scene, des_scene = sift.compute(scene_img, kp_scene)\n",
    "    m_scenes_images_features[i] = [scene_img, kp_scene, des_scene]\n",
    "    # scene_img_visualization = cv2.drawKeypoints(scene_img,kp_e_scemes_images['1'],None,flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)\n",
    "    # plt.imshow(cv2.cvtColor(e_scenes_images_features[str(i)][0], cv2.COLOR_BGR2RGB))\n",
    "    # plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining a function able to print out images that show up the location of the center and the position of each keypoints  :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function defined to print the image where it is specified the location of the keypoints and the center of the image \n",
    "def Print_Center_Keypoint(model_img, model_images_features, GoodMatches,Img_scale=1000):\n",
    "    #Print the G point in a resized version of each model image\n",
    "    #creating a resized image of the model image in order to print the results in an homogeneous way\n",
    "            #getting image dimensions \n",
    "    height, width, channels = model_img.shape\n",
    "            #defining the new dimensions of the image\n",
    "    dim_ratio = width/height\n",
    "    new_height = Img_scale\n",
    "    new_width  = int(new_height*dim_ratio)\n",
    "    new_dim = (new_width, new_height)\n",
    "            #image resizing\n",
    "    resized_img = cv2.resize(model_img, new_dim, interpolation = cv2.INTER_AREA) \n",
    "            #computing the ratio between the vertical and the horizontal dimensions resizing process\n",
    "    height_ratio = new_height/height\n",
    "    width_ratio = new_width/width\n",
    "            #computing the G in the image resized\n",
    "    g_x = int(model_images_features[i][CENTER_INDEX][0]*width_ratio)\n",
    "    g_y = int(model_images_features[i][CENTER_INDEX][1]*height_ratio)\n",
    "\n",
    "            #computing the Kp points in the image resized\n",
    "    for m in GoodMatches:\n",
    "                # getting the coordinates of the m-th keypoint in the i-th model\n",
    "        Kp_x = int(model_images_features[i][KEYPOINT_INDEX][m.queryIdx].pt[0])\n",
    "        Kp_y = int(model_images_features[i][KEYPOINT_INDEX][m.queryIdx].pt[1])\n",
    "        kp_x = int(Kp_x*width_ratio)\n",
    "        kp_y = int(Kp_y*height_ratio)\n",
    "                #Draw a Line \n",
    "        cv2.line(resized_img,(g_x,g_y),(kp_x,kp_y),(0,0,0),10) \n",
    "                #Drawing a dot in the position of Kp\n",
    "        cv2.circle(resized_img,(kp_x, kp_y), 10, (0,255,0),20)\n",
    "            \n",
    "        #Drawing a dot in the position of G\n",
    "    cv2.circle(resized_img,(g_x, g_y), 10, (255,55,236),25)\n",
    "        #Plotting the resized image\n",
    "    plt.imshow(cv2.cvtColor(resized_img, cv2.COLOR_BGR2RGB))\n",
    "    plt.show()    \n",
    "    print(\"Number of Key point found : \",len(GoodMatches))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Computing now the vectors starting from the keypoitns of the model and pointing the center G of the image then we would be able to identify the distance and the directions of each keypoint position with respect to G."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function create or update the model_images_features dictionary which contains all the relevant features of the \n",
    "# each model image, with a list containing all the voting vectors computed for each good match between the keypoints of \n",
    "# the model and the scene image\n",
    "\n",
    "\n",
    "def VotingVectors(Good_Matches,model_images_features, i):\n",
    "    # getting the coordinates of the center of the i-th model, G\n",
    "    G_x = model_images_features[i][CENTER_INDEX][0]\n",
    "    G_y = model_images_features[i][CENTER_INDEX][1]\n",
    "        \n",
    "    #Initializing the vector V \n",
    "    V = []\n",
    "    \n",
    "    for m in Good_Matches:\n",
    "        # getting the coordinates of the m-th keypoint in the i-th model\n",
    "        Kp_x = int(model_images_features[i][KEYPOINT_INDEX][m.queryIdx].pt[0])\n",
    "        Kp_y = int(model_images_features[i][KEYPOINT_INDEX][m.queryIdx].pt[1])\n",
    "            \n",
    "        #defining the point V = Kp[m,i] - G[i] \n",
    "        vx = Kp_x - G_x\n",
    "        vy = Kp_y - G_y\n",
    "            \n",
    "        #computing the slope of the direction aligning the center point G with the m-th keypoint\n",
    "        #in the case V = (0,0) it means basically that the point G[i] and Kp[m,i] are coincident and so the slope will not exist;\n",
    "        # Or anothe case where the slope does not exist is if the the G[i] and the Kp[m,i] are aligned in a vertical line;\n",
    "        #In such a cases the slope variable can be considered as an array containing two informations, the fisrt information is the \n",
    "        #actual value of the slope, while the second information encloses if the slope exists or not (1,0 respectively)\n",
    "        if (np.abs(vx) + np.abs(vy)) == 0 :\n",
    "            slope = [0, 0]\n",
    "        elif vy == 0 :\n",
    "            slope = [1, 0]\n",
    "        else:\n",
    "            slope = [vx/vy, 1]\n",
    "            \n",
    "        #creating an array containing the V coordinates and the slope of the line \n",
    "        v = [vx, vy, slope]\n",
    "        V.append(v)\n",
    "\n",
    "    if len( model_images_features[i]) <= V_INDEX :\n",
    "        # Updating the model feature dictionary appending the information of the vector V\n",
    "        model_images_features[i].append(V) \n",
    "    else: \n",
    "        #in the case V is already present in the model feature dictionary then update the V alreary there\n",
    "        model_images_features[i][V_INDEX] = V\n",
    "\n",
    "    return V    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "CENTER_INDEX     = 3\n",
    "V_INDEX          = 4\n",
    "\n",
    "for j in scenes_to_test:\n",
    "    for i in range(NUM_OF_MODELS):\n",
    "        \n",
    "        model_img = cv2.imread('./models/{}.jpg'.format(i), cv2.COLOR_BGR2RGB)\n",
    "        \n",
    "        #Find or update the center of the model image information collected in the model features dictionary\n",
    "        InstanceCenter(model_img,model_images_features, i, CENTER_INDEX=3) \n",
    "        \n",
    "        #Finding matches between the keypoints of the scene and the keypoints found in the model\n",
    "        good = Matching(model_images_features[i][DESCRIPTOR_INDEX], m_scenes_images_features[j][DESCRIPTOR_INDEX])\n",
    "                \n",
    "        # Creating or Updating the model feature dictionary with the informations regarding the voting vectors\n",
    "        V = VotingVectors(good,model_images_features, i)\n",
    "        \n",
    "        # Print the model images\n",
    "        Print_Center_Keypoint(model_img, model_images_features, good,Img_scale=1000)\n",
    "\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Building now an accumulator array in order to estimate the position of the center of each instance of the model in the scene :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function which is supposed to take a scene image and, provided the dimension of a cell of the accumulator array, \n",
    "#it sections graphically the image in order to visualize the accumulator array referred to the scene\n",
    "\n",
    "def plot_Img_sectioned(img, k1, k2):\n",
    "    H,W = img.shape[:2]\n",
    "    for w in range(0,W,k1):\n",
    "        #Draw a Vertical Line \n",
    "        cv2.line(img,(w,0),(w,H),(0,0,0),3) \n",
    "    for h in range(0,H,k2):\n",
    "        #Draw an Horizontal Line \n",
    "        cv2.line(img,(0,h),(W,h),(0,0,0),3) \n",
    "        \n",
    "    #Plotting the tabled image\n",
    "    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n",
    "    plt.show()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def KeyPoints_Comparison(models_images_features,scenes_images_features, Good_Matches):\n",
    "    model_kps_size = [] #list containing the size of each matched keypoint belonging to the model\n",
    "    scene_kps_size = [] #list containing the size of each matched keypoint belonging to the scene\n",
    "    ratio_of_sizes = [] #list containing the ratio between the sizes of the same keypoint found in the model and in the scene, representing the change in dimensions\n",
    "    \n",
    "    model_kps_angle = [] #list containing the angle of each matched keypoint belonging to the model\n",
    "    scene_kps_angle = [] #list containing the angle of each matched keypoint belonging to the scene\n",
    "    relative_angles = [] #list containing the angle between the model and the scene, representing rotation occuring between model and scene instance\n",
    "    \n",
    "    for o in Good_Matches:\n",
    "             #Defining two vectors respectively containing the size of the keypoints belonging to the model and \n",
    "             #to the scene\n",
    "        M_kp_size = np.float32(models_images_features[i][KEYPOINT_INDEX][o.queryIdx].size)\n",
    "        S_kp_size = np.float32(scenes_images_features[j][KEYPOINT_INDEX][o.trainIdx].size)\n",
    "        model_kps_size.append(M_kp_size)\n",
    "        scene_kps_size.append(S_kp_size)\n",
    "             #Defining two vectors respectively containing the angle of the keypoints belonging to the model and \n",
    "             #to the scene        \n",
    "        M_kp_angle = np.float32(models_images_features[i][KEYPOINT_INDEX][o.queryIdx].angle)\n",
    "        S_kp_angle = np.float32(scenes_images_features[j][KEYPOINT_INDEX][o.trainIdx].angle)\n",
    "        model_kps_angle.append(M_kp_angle)\n",
    "        scene_kps_angle.append(S_kp_angle)\n",
    "\n",
    "             #defining the change in dimensions between the keypoints of the model compared to the same ones found \n",
    "             #in the scene side\n",
    "        ratio_of_sizes.append(M_kp_size/S_kp_size)\n",
    "        \n",
    "             #defining the change in dimensions between the keypoints of the model compared to the same ones found \n",
    "             #in the scene side\n",
    "        relative_angles.append(M_kp_angle-S_kp_angle)       \n",
    "\n",
    "    #computation of the mean scale factor\n",
    "    if len(ratio_of_sizes) : \n",
    "        Mean_Scale_Factor = (sum(ratio_of_sizes))/len(ratio_of_sizes)\n",
    "    else: \n",
    "        # Mean_Scale_Factor = 0\n",
    "        Mean_Scale_Factor = 3.5\n",
    "    #computation of the mean relative angle\n",
    "    if len(relative_angles) :\n",
    "        Mean_Relative_Angle = (sum(relative_angles))/len(relative_angles)\n",
    "    else: \n",
    "        Mean_Relative_Angle = 0\n",
    "        \n",
    "        \n",
    "    return Mean_Scale_Factor, Mean_Relative_Angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Accumulator_Array(ACC_ARRAY_CELL_DIMENSION_1, ACC_ARRAY_CELL_DIMENSION_2, Scene_img, Model_img, \n",
    "                      models_images_features, scenes_images_features, Good_Matches, V, i, j, KEYPOINT_INDEX = 1):\n",
    "    #Accumulator dimensions\n",
    "    Acc_dim = (int(scenes_images_features[j][0].shape[0] / ACC_ARRAY_CELL_DIMENSION_2), \n",
    "               int(scenes_images_features[j][0].shape[1] / ACC_ARRAY_CELL_DIMENSION_1))\n",
    "    Accumulator_Array_Points = {}\n",
    "    #Accumulator array as a matrix of zeroes\n",
    "    Accumulator_Array = np.zeros(Acc_dim)\n",
    "    #print(Accumulator_Array.shape)\n",
    "    #extracting all the keypoints of the scene resulting in good matches\n",
    "    scene_pts = np.float32([scenes_images_features[j][KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "    \n",
    "    #computing the mean scale factor and the mean relative angle occuring between the model and its instances on the scene\n",
    "    r , alpha = KeyPoints_Comparison(models_images_features, scenes_images_features, Good_Matches)\n",
    "    \n",
    "    #defining the list wich will contain all the G estimation\n",
    "    G_scene = []\n",
    "    for l in range(len(Good_Matches)):\n",
    "        # print((models_images_features[i][KEYPOINT_INDEX][Good_Matches[l].queryIdx].pt))\n",
    "        #rescaling of the voting vectors\n",
    "        Vx_scene = V[l][0] / r\n",
    "        Vy_scene = V[l][1] / r\n",
    "        #computing the estimate position of the center pointed by the l-th voting vector rescaled starting from the l-th keypoint\n",
    "        Gx_scene = scene_pts[l][0][0] - Vx_scene\n",
    "        Gy_scene = scene_pts[l][0][1] - Vy_scene\n",
    "\n",
    "        #collecting the Center estimation in a list \n",
    "        G_scene.append([Gx_scene, Gy_scene])\n",
    "        \n",
    "        \n",
    "        GS_x = int(Gx_scene/(ACC_ARRAY_CELL_DIMENSION_1))\n",
    "        GS_y = int(Gy_scene/(ACC_ARRAY_CELL_DIMENSION_2))\n",
    "        # print([Gx_scene, Gy_scene])\n",
    "        if GS_x in range(Acc_dim[1]):\n",
    "            if GS_y in range(Acc_dim[0]):\n",
    "                # print([GS_y, GS_x])\n",
    "                Accumulator_Array[GS_y,GS_x] += 1\n",
    "\n",
    "                # save the scene points that fall into the current cell of the accumulator array, \n",
    "                if not (GS_y,GS_x, 'S') in Accumulator_Array_Points:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'S')] = []\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'S')].append((Gx_scene, Gy_scene))\n",
    "                else:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'S')].append((Gx_scene, Gy_scene))\n",
    "                '''\n",
    "                # save the model points corresponding to the scene point that fell into the current\n",
    "                # cell of the accumulator array, so to be able to compute then the homography\n",
    "                if not (GS_y,GS_x, 'M') in Accumulator_Array_Points:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'M')] = []\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'M')].append(np.float32(models_images_features[i][KEYPOINT_INDEX][Good_Matches[l].queryIdx].pt))\n",
    "                else:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'M')].append(np.float32(models_images_features[i][KEYPOINT_INDEX][Good_Matches[l].queryIdx].pt))\n",
    "\n",
    "                # save also the list of indexes of good matches to use to be able to draw the bounding box later\n",
    "                if not (GS_y,GS_x, 'L') in Accumulator_Array_Points:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'L')] = []\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'L')].append(l)\n",
    "                else:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'L')].append(l)\n",
    "                '''\n",
    "        \n",
    "    #print(Accumulator_Array)\n",
    "    \n",
    "    return Accumulator_Array, G_scene, Accumulator_Array_Points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function that estimates the centers in the scene of the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def center_estimation(models_images_features, scenes_images_features, Good_Matches, V, j):\n",
    "    #extracting all the keypoints of the scene resulting in good matches\n",
    "    scene_pts = np.float32([scenes_images_features[j][KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "    \n",
    "    #computing the mean scale factor and the mean relative angle occuring between the model and its instances on the scene\n",
    "    r , alpha = KeyPoints_Comparison(models_images_features, scenes_images_features, Good_Matches)\n",
    "    \n",
    "    #defining the list wich will contain all the G estimation\n",
    "    G_scene = []\n",
    "    for l in range(len(Good_Matches)):\n",
    "        # print((models_images_features[i][KEYPOINT_INDEX][Good_Matches[l].queryIdx].pt))\n",
    "        #rescaling of the voting vectors\n",
    "        Vx_scene = V[l][0] / r\n",
    "        Vy_scene = V[l][1] / r\n",
    "        #computing the estimate position of the center pointed by the l-th voting vector rescaled starting from the l-th keypoint\n",
    "        Gx_scene = scene_pts[l][0][0] - Vx_scene\n",
    "        Gy_scene = scene_pts[l][0][1] - Vy_scene\n",
    "\n",
    "        #collecting the Center estimation in a list \n",
    "        G_scene.append([Gx_scene, Gy_scene])\n",
    "    \n",
    "    return G_scene, r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that merges adjacent bounding boxes that overlap between each other       \n",
    "def merge_overlapping_bounding_boxes(corners_of_bounding_boxes, DISTANCE_BOUNDING_BOXES_TRESHOLD=200):\n",
    "    # top left and bottom right corners of the final bounding boxes\n",
    "    final_corners_of_bounding_boxes = []\n",
    "    \n",
    "    for index_1, [top_left_corner, bottom_right_corner] in enumerate(corners_of_bounding_boxes):\n",
    "            # add first couple of top left and bottom right corners of the final bounding boxes\n",
    "            if len(final_corners_of_bounding_boxes) == 0:\n",
    "                final_corners_of_bounding_boxes.append([ (int(top_left_corner[0]), int(top_left_corner[1])), \n",
    "                                                            (int(bottom_right_corner[0]), int(bottom_right_corner[1]))])\n",
    "            \n",
    "            \n",
    "            for index_2, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes):\n",
    "                # if a corner is already in the final corners then I don't add it\n",
    "                if top_left_corner == fin_top_left_corner and bottom_right_corner == fin_bottom_right_corner:\n",
    "                    break\n",
    "                # if a corner is near (below DISTANCE_BOUNDING_BOXES_TRESHOLD) a final corner, then I mean the two into a\n",
    "                # single one\n",
    "                if ( distance_2_points(top_left_corner, fin_top_left_corner) < DISTANCE_BOUNDING_BOXES_TRESHOLD and \n",
    "                distance_2_points(fin_bottom_right_corner, bottom_right_corner) < DISTANCE_BOUNDING_BOXES_TRESHOLD ) :\n",
    "                    \n",
    "                    sum_top_left = tuple(map(operator.add, top_left_corner, fin_top_left_corner))\n",
    "                    sum_bottom_right = tuple(map(operator.add, bottom_right_corner ,fin_bottom_right_corner))\n",
    "                    mean_top_left = (sum_top_left[0]/2, sum_top_left[1]/2)\n",
    "                    mean_bottom_right = (sum_bottom_right[0]/2, sum_bottom_right[1]/2)\n",
    "                    print('mean_top_left', mean_top_left)\n",
    "                    final_corners_of_bounding_boxes[index_2] = [ (int(mean_top_left[0]), int(mean_top_left[1])), \n",
    "                                                            (int(mean_bottom_right[0]), int(mean_bottom_right[1]))]\n",
    "        \n",
    "                    break\n",
    "                # if my corner is'n near any final corner then I add it to the final corners\n",
    "                if index_2 == len(final_corners_of_bounding_boxes) - 1:\n",
    "                    final_corners_of_bounding_boxes.append([ (int(top_left_corner[0]), int(top_left_corner[1])), \n",
    "                                                            (int(bottom_right_corner[0]), int(bottom_right_corner[1]))])\n",
    "    return final_corners_of_bounding_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PROVA PRIMA COLORE E POI ;ERGE\n",
    "\n",
    "import operator\n",
    "\n",
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "\n",
    "models_to_test = [0, 1, 11, 19, 24, 25, 26]\n",
    "scenes_to_test = [1,2,3,4,5]\n",
    "\n",
    "m_results = {}\n",
    "\n",
    "m_final_scene_images_with_bb = {}\n",
    "\n",
    "# Parameters to tune\n",
    "\n",
    "#dimension of a single cell of the accumulator array\n",
    "ACC_ARRAY_CELL_DIMENSION_1 = 120\n",
    "ACC_ARRAY_CELL_DIMENSION_2 = 120\n",
    "# minimum number of votes to consider G as a valid point\n",
    "MIN_VOTES = 1\n",
    "#treshold passed to the Matching function\n",
    "MATCHING_TRESHOLD = 0.45\n",
    "# distance to merge two bounding boxes\n",
    "DISTANCE_BOUNDING_BOXES_TRESHOLD = 200\n",
    "\n",
    "\n",
    "COLOR_DIFF_IN_SINGLE_CHANNEL_TRES = 82\n",
    "\n",
    "N_BINS_ON_WIDTH = 3\n",
    "\n",
    "M_BINS_ON_HEIGHT = 4\n",
    "\n",
    "\n",
    "#print(m_scenes_images_features[1][0].shape[:2])\n",
    "#plt.imshow(cv2.cvtColor(m_scenes_images_features[1][0], cv2.COLOR_BGR2RGB))\n",
    "#plt.show()\n",
    "\n",
    "for j in scenes_to_test:\n",
    "    m_final_scene_images_with_bb[j] = np.copy(m_scenes_images_features[j][IMAGE_INDEX])\n",
    "    m_results[j] = {}\n",
    "    for i in models_to_test:\n",
    "        scene_img = cv2.imread('./scenes/m{}.png'.format(j), cv2.COLOR_BGR2RGB)\n",
    "        model_img = cv2.imread('./models/{}.jpg'.format(i), cv2.COLOR_BGR2RGB)\n",
    "\n",
    "        print('_' * 80 + '\\n')\n",
    "        print('Finding model {} in scene {}'.format(i,j))\n",
    "\n",
    "        m_results[j][i] = {}\n",
    "        m_results[j][i]['count'] = 0\n",
    "        \n",
    "        # Special move\n",
    "        for special_move in [1,2]:\n",
    "            #Finding matches between the keypoints of the scene and the keypoints found in the model\n",
    "            good = Matching(model_images_features[i][DESCRIPTOR_INDEX], m_scenes_images_features[j][DESCRIPTOR_INDEX], \n",
    "                            Treshold=MATCHING_TRESHOLD)\n",
    "\n",
    "        \n",
    "        V = VotingVectors(good,model_images_features, i)\n",
    "\n",
    "        ACC, G_scene, Accumulator_Array_Points = Accumulator_Array(ACC_ARRAY_CELL_DIMENSION_1,\n",
    "                                                                      ACC_ARRAY_CELL_DIMENSION_2,\n",
    "                                                                      scene_img, \n",
    "                                                                      model_img,\n",
    "                                                                      model_images_features, \n",
    "                                                                      m_scenes_images_features, \n",
    "                                                                      good, \n",
    "                                                                      V, \n",
    "                                                                      i, j)\n",
    "        for g in G_scene :\n",
    "            #Drawing a dot in the position of G\n",
    "            #print(g)\n",
    "            #if np.max(ACC)>=20:\n",
    "            cv2.circle(scene_img,(int(g[0]), int(g[1])), 10, (255,55,236),25)\n",
    "\n",
    "        # List of indexes of highlighted cells (list of cells that have a num votes >= MIN_VOTES)\n",
    "        highlighted_cells_of_current_model_in_current_scene = []\n",
    "\n",
    "        print('List of highlighted cells:')\n",
    "        # print(ACC.shape[0], ACC.shape[1])\n",
    "        for t in range(ACC.shape[0]):\n",
    "            for w in range(ACC.shape[1]):\n",
    "                #print(t,w)\n",
    "                # If a cell of the accumulator array has more than MIN_VOTES than it means that the model has been found\n",
    "                if ACC[t,w] >= MIN_VOTES:\n",
    "\n",
    "\n",
    "                    print('  - Num votes: {} in cell {} '.format(ACC[t,w], (t,w)))\n",
    "                    # print('Max in Accumulator Array : ', np.max(ACC))\n",
    "                    highlighted_cells_of_current_model_in_current_scene.append([t, w])\n",
    "                    #print(Accumulator_Array_Points[(t, w, 'S')])\n",
    "\n",
    "\n",
    "        # print(highlighted_cells_of_current_model_in_current_scene)\n",
    "                \n",
    "        #V = VotingVectors(good, model_images_features, i)\n",
    "        # estimated centers of the bounding boxes\n",
    "        G_scenes, r = center_estimation(model_images_features, m_scenes_images_features, good, V, j)\n",
    "\n",
    "        print('r: ', r)\n",
    "        # getting the scaled height and width of the model in the scene\n",
    "        model_height_in_the_scene = model_images_features[i][IMAGE_INDEX].shape[0] / r                                               \n",
    "        model_width_in_the_scene = model_images_features[i][IMAGE_INDEX].shape[1] / r\n",
    "\n",
    "        m_scene_img_copy = np.copy(m_scenes_images_features[j][IMAGE_INDEX])\n",
    "\n",
    "        corners_of_bounding_boxes = []\n",
    "\n",
    "        for c in highlighted_cells_of_current_model_in_current_scene :\n",
    "            # Compute G (the center of the cereal box in the scene) as the mean of points that fall \n",
    "            # into the higlighted cell\n",
    "            G_mean = np.mean(Accumulator_Array_Points[(c[0], c[1], 'S')], axis=0)\n",
    "            cv2.circle(scene_img,(int(G_mean[0]), int(G_mean[1])), 20, (0,0,0),25)\n",
    "\n",
    "            cv2.rectangle(scene_img,(c[1]*ACC_ARRAY_CELL_DIMENSION_1,c[0]*ACC_ARRAY_CELL_DIMENSION_2),\n",
    "                          ((c[1]+1)*ACC_ARRAY_CELL_DIMENSION_1,(c[0]+1)*ACC_ARRAY_CELL_DIMENSION_2),(0,255,0),10)\n",
    "\n",
    "\n",
    "\n",
    "            top_left_corner_of_bounding_box = ( int(int(G_mean[0]) - (model_width_in_the_scene / 2) ), \n",
    "                                            int(int(G_mean[1]) - (model_height_in_the_scene / 2) ) )\n",
    "            bottom_right_corner_of_buonding_box = ( int(int(G_mean[0]) + (model_width_in_the_scene / 2) ), \n",
    "                                            int(int(G_mean[1]) + (model_height_in_the_scene / 2) ) )\n",
    "\n",
    "\n",
    "            top_right_corner_of_bounding_box = (bottom_right_corner_of_buonding_box[0], top_left_corner_of_bounding_box[1])\n",
    "\n",
    "            corners_of_bounding_boxes.append([top_left_corner_of_bounding_box, bottom_right_corner_of_buonding_box]) \n",
    "\n",
    "            scene_img_with_bounding_boxes = cv2.rectangle(m_scene_img_copy,\n",
    "                                          top_left_corner_of_bounding_box,\n",
    "                                          bottom_right_corner_of_buonding_box,\n",
    "                                          (0,255,0), 10)\n",
    "\n",
    "        \n",
    "\n",
    "        # solve exceeding dimensions of bounding boxes in the scene\n",
    "        final_corners_of_bounding_boxes_without_exceeding_dimensions = solve_exceeding_dimensions_of_bounding_boxes_in_scene(\n",
    "            corners_of_bounding_boxes, m_scenes_images_features)\n",
    "\n",
    "        # solve color problem\n",
    "        #final_corners_of_bounding_boxes_after_color_problem = solve_color_problem(final_corners_of_bounding_boxes_without_exceeding_dimensions, m_scenes_images_features, j)\n",
    "\n",
    "        # solve color problem with bins\n",
    "\n",
    "        final_corners_of_bounding_boxes_after_color_problem = solve_color_problem_with_N_x_M_bins(\n",
    "            final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "            m_scenes_images_features, i, j, N=N_BINS_ON_WIDTH, M=M_BINS_ON_HEIGHT, \n",
    "            COLOR_DIFF_IN_SINGLE_CHANNEL_TRES=COLOR_DIFF_IN_SINGLE_CHANNEL_TRES)\n",
    "        \n",
    "        # merge overlapping bounding boxes\n",
    "        final_corners_of_bounding_boxes = merge_overlapping_bounding_boxes(final_corners_of_bounding_boxes_after_color_problem, \n",
    "                                                                           DISTANCE_BOUNDING_BOXES_TRESHOLD)\n",
    "\n",
    "\n",
    "        print('Model {}'.format(i))\n",
    "        #plotting the model image\n",
    "        plt.imshow(cv2.cvtColor(model_img, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "        #plotting the scene image\n",
    "        plt.imshow(cv2.cvtColor(scene_img, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "\n",
    "        # plotting every cell of the accumulator array on the scene image\n",
    "        scene_img_sctioned = np.copy(scene_img)\n",
    "        plot_Img_sectioned(scene_img_sctioned, ACC_ARRAY_CELL_DIMENSION_1, ACC_ARRAY_CELL_DIMENSION_2)\n",
    "        print(ACC[ACC != 0])\n",
    "\n",
    "        # if I don't find any cell that has the min num of votes, then I don't print the bounding box\n",
    "        if(len(highlighted_cells_of_current_model_in_current_scene) > 0):\n",
    "            print('  - Found {} instances of model {} in scene {}'.format(len(highlighted_cells_of_current_model_in_current_scene),i,j))\n",
    "            # plotting the bounding box\n",
    "            plt.imshow(cv2.cvtColor(scene_img_with_bounding_boxes, cv2.COLOR_BGR2RGB))\n",
    "            plt.show()\n",
    "        else:\n",
    "            print('  - Model {} NOT found in the scene {}'.format(i,j))\n",
    "\n",
    "        # plot image with final bounding boxes (with bounding boxes that do not overlap each other)\n",
    "        plot_final_bounding_boxes(None, final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                  m_scenes_images_features, j)\n",
    "\n",
    "        print('AFTER COLOR CORRECTION:')\n",
    "        # plot image with final bounding boxes, after color problem\n",
    "        m_img_bb = plot_final_bounding_boxes(m_final_scene_images_with_bb[j], \n",
    "                                             final_corners_of_bounding_boxes, \n",
    "                                             m_scenes_images_features, j)\n",
    "\n",
    "        if m_img_bb is not None:\n",
    "            m_final_scene_images_with_bb[j] = m_img_bb\n",
    "\n",
    "        m_results[j][i]['count'] += len(final_corners_of_bounding_boxes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in scenes_to_test:\n",
    "    plt.imshow(cv2.cvtColor(m_final_scene_images_with_bb[j], cv2.COLOR_BGR2RGB))\n",
    "    plt.show()\n",
    "    print_result(m_results, j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve_color_problem_with_difference_N_x_M_bins(final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                                   difficulty_scenes_images_features, \n",
    "                                                   i, j, N=3, M=4, \n",
    "                                                   COLOR_DIFF_IN_SINGLE_CHANNEL_TRES = 50, \n",
    "                                                   MAX_NUM_OF_NO_GOOD_CELLS = 5):\n",
    "    final_corners_of_bounding_boxes_after_color_problem = []\n",
    "    \n",
    "    model_img = np.copy(model_images_features[i][IMAGE_INDEX])\n",
    "    \n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes_without_exceeding_dimensions):\n",
    "        \n",
    "        # model probably found in the scene\n",
    "        model_found_in_the_scene = np.copy(difficulty_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                   fin_top_left_corner[0]:fin_bottom_right_corner[0]])\n",
    "        # define new dimension that the model image should now have, \n",
    "        # which is the dimension of the probably found model in the scene \n",
    "        new_dim = (model_found_in_the_scene.shape[1], model_found_in_the_scene.shape[0])\n",
    "        \n",
    "        # model image resizing\n",
    "        resized_model_img = cv2.resize(model_img, new_dim, interpolation = cv2.INTER_AREA)\n",
    "        \n",
    "        # ValueError: operands could not be broadcast together with shapes (63,85,3) (85,63,3) \n",
    "\n",
    "        resized_model_img = resized_model_img.reshape(model_found_in_the_scene.shape[0], model_found_in_the_scene.shape[1],\n",
    "                                                     model_found_in_the_scene.shape[2])\n",
    "        \n",
    "        # perform difference of intensities between the scaled model image and the probably found model in the scene image \n",
    "        diff_image = resized_model_img -  model_found_in_the_scene\n",
    "        \n",
    "        # plot the resized model image\n",
    "        plt.imshow(cv2.cvtColor(resized_model_img, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "        \n",
    "        # plot the probably model found in the scene\n",
    "        plt.imshow(cv2.cvtColor(model_found_in_the_scene, cv2.COLOR_BGR2RGB))\n",
    "        plt.show() \n",
    "        \n",
    "        \n",
    "        \n",
    "        # divide the image containing the difference of intensities into N x M bins\n",
    "        means_bins_diff_img = split_image_into_N_x_M_bins_with_intensity_means(diff_image, \n",
    "                                                                               n_bins_width = N, \n",
    "                                                                               n_bins_heigth = M)\n",
    "        \n",
    "        if not means_bins_diff_img:\n",
    "            return []\n",
    "    \n",
    "        good = True\n",
    "        num_of_No_good = 0\n",
    "        \n",
    "        # cycle the model dictionary and get the means by getting the keys of the scene dictionary\n",
    "        for k, v in means_bins_diff_img.items():\n",
    "            means_k_diff = means_bins_diff_img.get(k)\n",
    "            \n",
    "            # PRINT ALL THE DIFFERENCES\n",
    "            print('Bin ({}, {}):'.format(k[0], k[1]))\n",
    "            print('  - diff_r: ', means_k_diff[0])\n",
    "            print('  - diff_g: ', means_k_diff[1])\n",
    "            print('  - diff_b: ', means_k_diff[2])\n",
    "            \n",
    "            if means_k_diff[0] >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES or \\\n",
    "                means_k_diff[1] >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES or \\\n",
    "                means_k_diff[2] >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES:\n",
    "                print('___________________NO GOOD___________________')\n",
    "                num_of_No_good += 1\n",
    "               \n",
    "                good = False\n",
    "        print('Num of No Good Bins: {}'.format(num_of_No_good))\n",
    "            \n",
    "        if num_of_No_good <= MAX_NUM_OF_NO_GOOD_CELLS: #good:\n",
    "            final_corners_of_bounding_boxes_after_color_problem.append(\n",
    "                [ (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                 (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))])\n",
    "            \n",
    "    \n",
    "    return final_corners_of_bounding_boxes_after_color_problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def KeyPoints_Comparison(models_images_features,scenes_images_features, Good_Matches):\n",
    "    model_kps_size = [] #list containing the size of each matched keypoint belonging to the model\n",
    "    scene_kps_size = [] #list containing the size of each matched keypoint belonging to the scene\n",
    "    ratio_of_sizes = [] #list containing the ratio between the sizes of the same keypoint found in the model and in the scene, representing the change in dimensions\n",
    "    \n",
    "    model_kps_angle = [] #list containing the angle of each matched keypoint belonging to the model\n",
    "    scene_kps_angle = [] #list containing the angle of each matched keypoint belonging to the scene\n",
    "    relative_angles = [] #list containing the angle between the model and the scene, representing rotation occuring between model and scene instance\n",
    "    \n",
    "    for o in Good_Matches:\n",
    "             #Defining two vectors respectively containing the size of the keypoints belonging to the model and \n",
    "             #to the scene\n",
    "        M_kp_size = np.float32(models_images_features[i][KEYPOINT_INDEX][o.queryIdx].size)\n",
    "        S_kp_size = np.float32(scenes_images_features[j][KEYPOINT_INDEX][o.trainIdx].size)\n",
    "        model_kps_size.append(M_kp_size)\n",
    "        scene_kps_size.append(S_kp_size)\n",
    "             #Defining two vectors respectively containing the angle of the keypoints belonging to the model and \n",
    "             #to the scene        \n",
    "        M_kp_angle = np.float32(models_images_features[i][KEYPOINT_INDEX][o.queryIdx].angle)\n",
    "        S_kp_angle = np.float32(scenes_images_features[j][KEYPOINT_INDEX][o.trainIdx].angle)\n",
    "        model_kps_angle.append(M_kp_angle)\n",
    "        scene_kps_angle.append(S_kp_angle)\n",
    "\n",
    "             #defining the change in dimensions between the keypoints of the model compared to the same ones found \n",
    "             #in the scene side\n",
    "        ratio_of_sizes.append(M_kp_size/S_kp_size)\n",
    "        \n",
    "             #defining the change in dimensions between the keypoints of the model compared to the same ones found \n",
    "             #in the scene side\n",
    "        relative_angles.append(M_kp_angle-S_kp_angle)       \n",
    "\n",
    "    #computation of the mean scale factor\n",
    "    if len(ratio_of_sizes) : \n",
    "        Mean_Scale_Factor = (sum(ratio_of_sizes))/len(ratio_of_sizes)\n",
    "    else: \n",
    "        # Mean_Scale_Factor = 0\n",
    "        Mean_Scale_Factor = 17.8#3.5#20#15#17.7\n",
    "    #computation of the mean relative angle\n",
    "    if len(relative_angles) :\n",
    "        Mean_Relative_Angle = (sum(relative_angles))/len(relative_angles)\n",
    "    else: \n",
    "        Mean_Relative_Angle = 0\n",
    "        \n",
    "        \n",
    "    return Mean_Scale_Factor, Mean_Relative_Angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dictionary for hard scene features\n",
    "h_scenes_images_features = {}\n",
    "scenes_to_test = [1,2,3,4,5,6]\n",
    "\n",
    "\n",
    "# add features for each hard scene image\n",
    "for i in scenes_to_test:\n",
    "    scene_img = cv2.imread('./scenes/h{}.jpg'.format(i), cv2.COLOR_BGR2GRAY) #cv2.COLOR_BGR2RGB)\n",
    "    kp_scene = sift.detect(scene_img)\n",
    "    kp_scene, des_scene = sift.compute(scene_img, kp_scene)\n",
    "    h_scenes_images_features[i] = [scene_img, kp_scene, des_scene]\n",
    "\n",
    "# indexes of the dictionary\n",
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "CENTER_INDEX     = 3\n",
    "V_INDEX          = 4\n",
    "TRESHOLD_INDEX = 5\n",
    "\n",
    "# models to test in stepC\n",
    "models_to_test = list(range(1, 27))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Divide a scene image into shelves images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show(img):\n",
    "    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB));\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_horizontal_lines_of_image(scene_img_index, difficulty_scenes_images_features):\n",
    "    # Load the image\n",
    "    src = np.copy(difficulty_scenes_images_features[scene_img_index][IMAGE_INDEX])\n",
    "    plt.imshow(cv2.cvtColor(src, cv2.COLOR_BGR2RGB));\n",
    "    plt.show()\n",
    "    # [gray]\n",
    "    # Transform source image to gray if it is not already\n",
    "\n",
    "\n",
    "    if len(src.shape) != 2:\n",
    "        gray = cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)\n",
    "    else:\n",
    "        gray = src\n",
    "    \n",
    "    # Show gray image\n",
    "    #show(gray)\n",
    "\n",
    "    # [bin]\n",
    "    # Apply adaptiveThreshold at the bitwise_not of gray, notice the ~ symbol\n",
    "    gray = cv2.bitwise_not(gray)\n",
    "    bw = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, \\\n",
    "                                cv2.THRESH_BINARY, 15, -2)\n",
    "    # Show binary image\n",
    "    #show(bw)\n",
    "\n",
    "    # Create the images that will use to extract the horizontal and vertical lines\n",
    "    horizontal = np.copy(bw)\n",
    "    # [horiz]\n",
    "    # Specify size on horizontal axis\n",
    "    cols = horizontal.shape[1]\n",
    "    horizontal_size = cols // 20\n",
    "\n",
    "    # Create structure element for extracting horizontal lines through morphology operations\n",
    "    horizontalStructure = cv2.getStructuringElement(cv2.MORPH_RECT, (horizontal_size, 2))\n",
    "\n",
    "    # Apply morphology operations\n",
    "    horizontal = cv2.erode(horizontal, horizontalStructure)\n",
    "    horizontal = cv2.erode(horizontal, horizontalStructure)\n",
    "\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "\n",
    "\n",
    "    # Show extracted horizontal lines\n",
    "    show(horizontal)\n",
    "    return horizontal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_height_coord_of_shelves_in_scene_img(height_coord_of_shelves_in_all_scenes, horizontal_features_img):\n",
    "    \n",
    "    horiz_height = horizontal_features_img.shape[0]\n",
    "    horiz_width = horizontal_features_img.shape[1]\n",
    "\n",
    "    # image with horizontal features in which I will show the shelves in red\n",
    "    horizontal_with_shelves_lines = np.copy(cv2.cvtColor(horizontal_features_img, cv2.COLOR_GRAY2RGB))\n",
    "    # scene image in which I will show the shelves in red\n",
    "    scene_img_with_shelves_lines = np.copy(h_scenes_images_features[j][IMAGE_INDEX])\n",
    "\n",
    "    height_coord_of_shelves_in_all_scenes.setdefault(j, []).append(0)\n",
    "\n",
    "    # cycle the pixels on the height of the image, with width = WIDTH_MEASURE \n",
    "    for height_coord in range(horiz_height):\n",
    "        # I checked that in the horizontal image there are intensities which are only = 0 or = 255\n",
    "        # if there is a white pixel in the along width = 100, and if the previous found pixel was black\n",
    "        # then record the position of a shelf\n",
    "        if( horizontal_features_img[height_coord, WIDTH_MEASURE] == 255 \n",
    "           and horizontal_features_img[height_coord - 1, WIDTH_MEASURE] == 0 ):\n",
    "            print(height_coord, WIDTH_MEASURE)\n",
    "            # add the height coord of the found shelf in the list of the current scene, which is contained in the dictionary\n",
    "            height_coord_of_shelves_in_all_scenes.setdefault(j, []).append(height_coord)\n",
    "            horizontal_with_shelves_lines = cv2.line(horizontal_with_shelves_lines, (0, height_coord), \n",
    "                                                     (horiz_width, height_coord), (0,0,255), 3)\n",
    "            \n",
    "            scene_img_with_shelves_lines = cv2.line(scene_img_with_shelves_lines, (0, height_coord), \n",
    "                                                     (horiz_width, height_coord), (0,0,255), 3)\n",
    "\n",
    "    show(horizontal_with_shelves_lines)\n",
    "    show(scene_img_with_shelves_lines)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_shelves_images_from_scene_img(scene_img_index, scenes_shelves_features,\n",
    "                                      height_coord_of_shelves_in_all_scenes, difficulty_scenes_images_features):\n",
    "    \n",
    "    MIN_HEIGHT_THAT_A_SHELF_SHOULD_HAVE = 20\n",
    "    \n",
    "    height_of_scene_img, width_of_scene_img = difficulty_scenes_images_features[scene_img_index][IMAGE_INDEX].shape[:2]\n",
    "    \n",
    "    for i, height_coord in enumerate(height_coord_of_shelves_in_all_scenes[scene_img_index]):        \n",
    "        if i == 0:\n",
    "            scenes_shelves_features[scene_img_index] = {}\n",
    "            \n",
    "        # If Itake the last shelf, then I have to take the portion of image starting from its height coord to\n",
    "        # the height of the image\n",
    "        if i == len(height_coord_of_shelves_in_all_scenes[scene_img_index]) - 1:\n",
    "            # I consider a shelf good only if its height is >= of a min treshold\n",
    "            if height_of_scene_img - height_coord >= MIN_HEIGHT_THAT_A_SHELF_SHOULD_HAVE:\n",
    "                scenes_shelves_features[scene_img_index][i] = \\\n",
    "                    [ difficulty_scenes_images_features[scene_img_index][IMAGE_INDEX][height_coord:height_of_scene_img]\\\n",
    "                    [0:width_of_scene_img] ]\n",
    "        # else I take the portion of the image from its height coord to the next height coord\n",
    "        else:\n",
    "            next_height_coord = height_coord_of_shelves_in_all_scenes[scene_img_index][i+1]\n",
    "\n",
    "            if next_height_coord - height_coord >= MIN_HEIGHT_THAT_A_SHELF_SHOULD_HAVE:\n",
    "                scenes_shelves_features[scene_img_index][i] = \\\n",
    "                    [ difficulty_scenes_images_features[scene_img_index][IMAGE_INDEX][height_coord:next_height_coord]\\\n",
    "                    [0:width_of_scene_img] ]\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "WIDTH_MEASURE = 100\n",
    "\n",
    "# dictionary that will contain the height coordinate of every shelf in each scene image\n",
    "height_coord_of_shelves_in_all_scenes = {}\n",
    "scenes_to_test = [1,2,3,4,5]\n",
    "\n",
    "scenes_shelves_features = {}\n",
    "\n",
    "\n",
    "for j in scenes_to_test:\n",
    "    print( ('_' * 40) + 'scene {}'.format(j) + ('_' * 40) + '\\n')\n",
    "    \n",
    "    horizontal_features_img = get_horizontal_lines_of_image(j, h_scenes_images_features)\n",
    "    \n",
    "    set_height_coord_of_shelves_in_scene_img(height_coord_of_shelves_in_all_scenes, horizontal_features_img)\n",
    "    \n",
    "    get_shelves_images_from_scene_img(j, scenes_shelves_features,\n",
    "                                      height_coord_of_shelves_in_all_scenes, h_scenes_images_features)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for scene_index, img_shelves_of_scene in scenes_shelves_features.items():\n",
    "    print(('_' * 40) + 'scene {}'.format(scene_index) + ('_' * 40))\n",
    "    for shelf_index, shelf_features in img_shelves_of_scene.items():\n",
    "        #doubled_shelf_image = cv2.resize(shelf_features[IMAGE_INDEX], (shelf_features[IMAGE_INDEX].shape[1] * 2, \n",
    "        #                                         shelf_features[IMAGE_INDEX].shape[0] * 2), \n",
    "        #           interpolation = cv2.INTER_AREA)\n",
    "        # print(shelf_features[IMAGE_INDEX].shape)\n",
    "        # save the image of a single shelf in the structure\n",
    "        print('shelf image:')\n",
    "        show(shelf_features[IMAGE_INDEX])\n",
    "        #print('doubled shelf image')\n",
    "        #show(doubled_shelf_image)\n",
    "        # compute keypoints and descriptors of the single shelf and save them in the structure too\n",
    "        kp_shelf = sift.detect(shelf_features[IMAGE_INDEX])\n",
    "        kp_shelf, des_shelf = sift.compute(shelf_features[IMAGE_INDEX], kp_shelf)\n",
    "        img_shelves_of_scene[shelf_index] = [shelf_features[IMAGE_INDEX], kp_shelf, des_shelf]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def KeyPoints_Comparison(models_images_features, scenes_images_features, Good_Matches, shelf=False):\n",
    "    model_kps_size = [] #list containing the size of each matched keypoint belonging to the model\n",
    "    scene_kps_size = [] #list containing the size of each matched keypoint belonging to the scene\n",
    "    ratio_of_sizes = [] #list containing the ratio between the sizes of the same keypoint found in the model and in the scene, representing the change in dimensions\n",
    "    \n",
    "    model_kps_angle = [] #list containing the angle of each matched keypoint belonging to the model\n",
    "    scene_kps_angle = [] #list containing the angle of each matched keypoint belonging to the scene\n",
    "    relative_angles = [] #list containing the angle between the model and the scene, representing rotation occuring between model and scene instance\n",
    "    \n",
    "    for o in Good_Matches:\n",
    "             #Defining two vectors respectively containing the size of the keypoints belonging to the model and \n",
    "             #to the scene\n",
    "        M_kp_size = np.float32(models_images_features[i][KEYPOINT_INDEX][o.queryIdx].size)\n",
    "        if shelf:\n",
    "            S_kp_size = np.float32(scenes_images_features[KEYPOINT_INDEX][o.trainIdx].size)\n",
    "        else:\n",
    "            S_kp_size = np.float32(scenes_images_features[j][KEYPOINT_INDEX][o.trainIdx].size)\n",
    "        model_kps_size.append(M_kp_size)\n",
    "        scene_kps_size.append(S_kp_size)\n",
    "             #Defining two vectors respectively containing the angle of the keypoints belonging to the model and \n",
    "             #to the scene        \n",
    "        M_kp_angle = np.float32(models_images_features[i][KEYPOINT_INDEX][o.queryIdx].angle)\n",
    "        if shelf:\n",
    "        \tS_kp_angle = np.float32(scenes_images_features[KEYPOINT_INDEX][o.trainIdx].angle)\n",
    "        else:\n",
    "            S_kp_angle = np.float32(scenes_images_features[j][KEYPOINT_INDEX][o.trainIdx].angle)\n",
    "        model_kps_angle.append(M_kp_angle)\n",
    "        scene_kps_angle.append(S_kp_angle)\n",
    "\n",
    "             #defining the change in dimensions between the keypoints of the model compared to the same ones found \n",
    "             #in the scene side\n",
    "        ratio_of_sizes.append(M_kp_size/S_kp_size)\n",
    "        \n",
    "             #defining the change in dimensions between the keypoints of the model compared to the same ones found \n",
    "             #in the scene side\n",
    "        relative_angles.append(M_kp_angle-S_kp_angle)       \n",
    "\n",
    "    #computation of the mean scale factor\n",
    "    if len(ratio_of_sizes) : \n",
    "        Mean_Scale_Factor = (sum(ratio_of_sizes))/len(ratio_of_sizes)\n",
    "        print('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb {}'.format(Mean_Scale_Factor))\n",
    "    else: \n",
    "        # Mean_Scale_Factor = 0\n",
    "        Mean_Scale_Factor = 16.7\n",
    "        print('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa {}'.format(Mean_Scale_Factor))\n",
    "    #computation of the mean relative angle\n",
    "    if len(relative_angles) :\n",
    "        Mean_Relative_Angle = (sum(relative_angles))/len(relative_angles)\n",
    "    else: \n",
    "        Mean_Relative_Angle = 0\n",
    "        \n",
    "        \n",
    "    return Mean_Scale_Factor, Mean_Relative_Angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Accumulator_Array(ACC_ARRAY_CELL_DIMENSION_1, ACC_ARRAY_CELL_DIMENSION_2, Scene_img, Model_img, \n",
    "                      models_images_features, scenes_images_features, Good_Matches, V, i, j, shelf=False, KEYPOINT_INDEX = 1):\n",
    "    #Accumulator dimensions\n",
    "    if shelf:\n",
    "    \tAcc_dim = (int(scenes_images_features[0].shape[0] / ACC_ARRAY_CELL_DIMENSION_2), \n",
    "\t               int(scenes_images_features[0].shape[1] / ACC_ARRAY_CELL_DIMENSION_1))\n",
    "    else:\n",
    "\t    Acc_dim = (int(scenes_images_features[j][0].shape[0] / ACC_ARRAY_CELL_DIMENSION_2), \n",
    "\t               int(scenes_images_features[j][0].shape[1] / ACC_ARRAY_CELL_DIMENSION_1))\n",
    "    Accumulator_Array_Points = {}\n",
    "    #Accumulator array as a matrix of zeroes\n",
    "    Accumulator_Array = np.zeros(Acc_dim)\n",
    "    #print(Accumulator_Array.shape)\n",
    "    #extracting all the keypoints of the scene resulting in good matches\n",
    "    if shelf:\n",
    "    \tscene_pts = np.float32([scenes_images_features[KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "    else:\n",
    "    \tscene_pts = np.float32([scenes_images_features[j][KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "    \n",
    "    #computing the mean scale factor and the mean relative angle occuring between the model and its instances on the scene\n",
    "    r , alpha = KeyPoints_Comparison(models_images_features, scenes_images_features, Good_Matches, shelf)\n",
    "    \n",
    "    #defining the list wich will contain all the G estimation\n",
    "    G_scene = []\n",
    "    for l in range(len(Good_Matches)):\n",
    "        # print((models_images_features[i][KEYPOINT_INDEX][Good_Matches[l].queryIdx].pt))\n",
    "        #rescaling of the voting vectors\n",
    "        Vx_scene = V[l][0] / r\n",
    "        Vy_scene = V[l][1] / r\n",
    "        #computing the estimate position of the center pointed by the l-th voting vector rescaled starting from the l-th keypoint\n",
    "        Gx_scene = scene_pts[l][0][0] - Vx_scene\n",
    "        Gy_scene = scene_pts[l][0][1] - Vy_scene\n",
    "\n",
    "        #collecting the Center estimation in a list \n",
    "        G_scene.append([Gx_scene, Gy_scene])\n",
    "        \n",
    "        \n",
    "        GS_x = int(Gx_scene/(ACC_ARRAY_CELL_DIMENSION_1))\n",
    "        GS_y = int(Gy_scene/(ACC_ARRAY_CELL_DIMENSION_2))\n",
    "        # print([Gx_scene, Gy_scene])\n",
    "        if GS_x in range(Acc_dim[1]):\n",
    "            if GS_y in range(Acc_dim[0]):\n",
    "                # print([GS_y, GS_x])\n",
    "                Accumulator_Array[GS_y,GS_x] += 1\n",
    "\n",
    "                # save the scene points that fall into the current cell of the accumulator array, \n",
    "                if not (GS_y,GS_x, 'S') in Accumulator_Array_Points:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'S')] = []\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'S')].append((Gx_scene, Gy_scene))\n",
    "                else:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'S')].append((Gx_scene, Gy_scene))\n",
    "                '''\n",
    "                # save the model points corresponding to the scene point that fell into the current\n",
    "                # cell of the accumulator array, so to be able to compute then the homography\n",
    "                if not (GS_y,GS_x, 'M') in Accumulator_Array_Points:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'M')] = []\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'M')].append(np.float32(models_images_features[i][KEYPOINT_INDEX][Good_Matches[l].queryIdx].pt))\n",
    "                else:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'M')].append(np.float32(models_images_features[i][KEYPOINT_INDEX][Good_Matches[l].queryIdx].pt))\n",
    "\n",
    "                # save also the list of indexes of good matches to use to be able to draw the bounding box later\n",
    "                if not (GS_y,GS_x, 'L') in Accumulator_Array_Points:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'L')] = []\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'L')].append(l)\n",
    "                else:\n",
    "                    Accumulator_Array_Points[(GS_y, GS_x, 'L')].append(l)\n",
    "                '''\n",
    "        \n",
    "    #print(Accumulator_Array)\n",
    "    \n",
    "    return Accumulator_Array, G_scene, Accumulator_Array_Points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def center_estimation(models_images_features, scenes_images_features, Good_Matches, V, j, shelf=False):\n",
    "    #extracting all the keypoints of the scene resulting in good matches\n",
    "    if shelf:\n",
    "        scene_pts = np.float32([scenes_images_features[KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "    else:\n",
    "        scene_pts = np.float32([scenes_images_features[j][KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "    \n",
    "    #computing the mean scale factor and the mean relative angle occuring between the model and its instances on the scene\n",
    "    r , alpha = KeyPoints_Comparison(models_images_features, scenes_images_features, Good_Matches, shelf)\n",
    "    \n",
    "    #defining the list wich will contain all the G estimation\n",
    "    G_scene = []\n",
    "    for l in range(len(Good_Matches)):\n",
    "        # print((models_images_features[i][KEYPOINT_INDEX][Good_Matches[l].queryIdx].pt))\n",
    "        #rescaling of the voting vectors\n",
    "        Vx_scene = V[l][0] / r\n",
    "        Vy_scene = V[l][1] / r\n",
    "        #computing the estimate position of the center pointed by the l-th voting vector rescaled starting from the l-th keypoint\n",
    "        Gx_scene = scene_pts[l][0][0] - Vx_scene\n",
    "        Gy_scene = scene_pts[l][0][1] - Vy_scene\n",
    "\n",
    "        #collecting the Center estimation in a list \n",
    "        G_scene.append([Gx_scene, Gy_scene])\n",
    "    \n",
    "    return G_scene, r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that solves exceeding dimensions of bounding_boxes in scene\n",
    "def solve_exceeding_dimensions_of_bounding_boxes_in_scene(final_corners_of_bounding_boxes, \n",
    "                                                          difficulty_scenes_images_features,\n",
    "                                                          bounding_box_confidence,\n",
    "                                                          bb_conf_after_exceeding_dim,\n",
    "                                                          shelf=False):\n",
    "    final_corners_of_bounding_boxes_without_exceeding_dimensions = []\n",
    "    # adjust bounding boxes that go out the dimensions of the scene image\n",
    "    if shelf:\n",
    "        scene_height = difficulty_scenes_images_features[IMAGE_INDEX].shape[0]\n",
    "        scene_width = difficulty_scenes_images_features[IMAGE_INDEX].shape[1]\n",
    "    else:\n",
    "        scene_height = difficulty_scenes_images_features[j][IMAGE_INDEX].shape[0]\n",
    "        scene_width = difficulty_scenes_images_features[j][IMAGE_INDEX].shape[1]\n",
    "\n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes):\n",
    "        \n",
    "        old_top_left_corner = fin_top_left_corner\n",
    "        old_bottom_right_corner = fin_bottom_right_corner\n",
    "        \n",
    "        fin_top_left_corner = list(fin_top_left_corner)\n",
    "        fin_bottom_right_corner = list(fin_bottom_right_corner)\n",
    "        \n",
    "        \n",
    "        # top left corner that goes out on the left of the scene image\n",
    "        if fin_top_left_corner[0] < 0:\n",
    "            fin_top_left_corner[0] = 0\n",
    "        # bottom right corner that goes out on the left of the scene image\n",
    "        if fin_bottom_right_corner[0] < 0:\n",
    "            fin_bottom_right_corner[0] = 0\n",
    "        # top left corner that goes out on the right of the scene image\n",
    "        if fin_top_left_corner[0] > scene_width:\n",
    "            fin_top_left_corner[0] = scene_width\n",
    "        # bottom right corner that goes out on the right of the scene image\n",
    "        if fin_bottom_right_corner[0] > scene_width:\n",
    "            fin_bottom_right_corner[0] = scene_width\n",
    "        # top left corner that goes out on the top of the scene image\n",
    "        if fin_top_left_corner[1] < 0:\n",
    "            fin_top_left_corner[1] = 0\n",
    "        # bottom right corner that goes out on the top of the scene image\n",
    "        if fin_bottom_right_corner[1] < 0:\n",
    "            fin_bottom_right_corner[1] = 0  \n",
    "        # top left corner that goes out on the bottom of the scene image\n",
    "        if fin_top_left_corner[1] > scene_height:\n",
    "            fin_top_left_corner[1] = scene_height\n",
    "        # bottom right corner that goes out on the bottom of the scene image\n",
    "        if fin_bottom_right_corner[1] > scene_height:\n",
    "            fin_bottom_right_corner[1] = scene_height\n",
    "\n",
    "        fin_top_left_corner = tuple(fin_top_left_corner)\n",
    "        fin_bottom_right_corner = tuple(fin_bottom_right_corner)\n",
    "        final_corners_of_bounding_boxes_without_exceeding_dimensions.append([ (int(fin_top_left_corner[0]), \n",
    "                                                                               int(fin_top_left_corner[1])), \n",
    "                                                                             (int(fin_bottom_right_corner[0]), \n",
    "                                                                              int(fin_bottom_right_corner[1]))])\n",
    "        # keep track of min_votes confidence after solving exceeding dimensions\n",
    "        bb_conf_after_exceeding_dim[ (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                                    (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1])) ] = \\\n",
    "                                    bounding_box_confidence[ (old_top_left_corner, old_bottom_right_corner) ]\n",
    "        \n",
    "    return final_corners_of_bounding_boxes_without_exceeding_dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve_color_problem_with_N_x_M_bins(final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                    difficulty_scenes_images_features, bb_conf_after_exceeding_dim,\n",
    "                                    bb_conf_after_color_solver,\n",
    "                                    i, j, N=3, M=4, \n",
    "                                    COLOR_DIFF_IN_SINGLE_CHANNEL_TRES=50, \n",
    "                                    MAX_NUM_OF_NO_GOOD_CELLS = 5,\n",
    "                                    shelf=False):\n",
    "    final_corners_of_bounding_boxes_after_color_problem = []\n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes_without_exceeding_dimensions):\n",
    "    \n",
    "        model_img = model_images_features[i][IMAGE_INDEX]\n",
    "        # smooth\n",
    "        blur_model_img = model_img #cv2.GaussianBlur(model_img,(45,45),0) #(45,45)\n",
    "        \n",
    "        means_bins_model_img = split_image_into_N_x_M_bins_with_intensity_means(\n",
    "            blur_model_img, \n",
    "            N, M)\n",
    "        \n",
    "        \n",
    "        \n",
    "        print(fin_top_left_corner[1], fin_top_left_corner[0], fin_bottom_right_corner[1], fin_bottom_right_corner[0])\n",
    "        \n",
    "        if shelf:\n",
    "            means_bins_scene_img = split_image_into_N_x_M_bins_with_intensity_means(\n",
    "                difficulty_scenes_images_features[IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]], \n",
    "                N, M)\n",
    "        else:\n",
    "            means_bins_scene_img = split_image_into_N_x_M_bins_with_intensity_means(\n",
    "                difficulty_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]], \n",
    "                N, M)\n",
    "    \n",
    "        if not means_bins_scene_img or not means_bins_scene_img:\n",
    "            return []\n",
    "    \n",
    "        good = True\n",
    "        num_of_No_good = 0\n",
    "\n",
    "        \n",
    "        # cycle the model dictionary and get the diff of the means by getting the keys of the scene dictionary\n",
    "        for k, v in means_bins_model_img.items():\n",
    "            \n",
    "            # do not consider all bins that are on the top and on the bottom\n",
    "            if k[0] != 0 and k[0] != N:\n",
    "            \n",
    "                print('CHIAVE')\n",
    "                print(k[0], k[1])\n",
    "                means_k_scene = means_bins_scene_img.get(k)\n",
    "                # I consider a bounding box as good if and only if all the difference of the intiensities from the \n",
    "                # bins of the model, in all 3 channels, are below a certain treshold\n",
    "                diff_r = np.absolute(v[0] - means_k_scene[0])\n",
    "                diff_g = np.absolute(v[1] - means_k_scene[1])\n",
    "                diff_b = np.absolute(v[2] - means_k_scene[2])\n",
    "\n",
    "                print('Bin ({}, {}):'.format(k[0], k[1]))\n",
    "                print('  - diff_r: ', diff_r)\n",
    "                print('  - diff_g: ', diff_g)\n",
    "                print('  - diff_b: ', diff_b)\n",
    "\n",
    "\n",
    "                if diff_r >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES or diff_g >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES or \\\n",
    "                    diff_b >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES:\n",
    "                    print('___________________NO GOOD___________________')\n",
    "                    good = False\n",
    "                    num_of_No_good += 1\n",
    "        \n",
    "        \n",
    "        if  num_of_No_good <= MAX_NUM_OF_NO_GOOD_CELLS: #good:\n",
    "            final_corners_of_bounding_boxes_after_color_problem.append([ (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                                                        (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))])\n",
    "            \n",
    "            # add color bins solver confidence\n",
    "            color_conf = (1 - (num_of_No_good / MAX_NUM_OF_NO_GOOD_CELLS)) \n",
    "            bb_conf_after_color_solver[(int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                                       (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))] = \\\n",
    "                                        bb_conf_after_exceeding_dim[(fin_top_left_corner, fin_bottom_right_corner)]\n",
    "            bb_conf_after_color_solver[(int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                                       (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))].append(color_conf)\n",
    "                                            \n",
    "\n",
    "    return final_corners_of_bounding_boxes_after_color_problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve_color_problem_with_difference_N_x_M_bins(final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                                   difficulty_scenes_images_features, \n",
    "                                                   i, j, N=3, M=4, \n",
    "                                                   COLOR_DIFF_IN_SINGLE_CHANNEL_TRES = 50, \n",
    "                                                   MAX_NUM_OF_NO_GOOD_CELLS = 5, shelf=False):\n",
    "    final_corners_of_bounding_boxes_after_color_problem = []\n",
    "    \n",
    "    model_img = np.copy(model_images_features[i][IMAGE_INDEX])\n",
    "    \n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes_without_exceeding_dimensions):\n",
    "        \n",
    "        # model probably found in the scene\n",
    "        if shelf:\n",
    "            model_found_in_the_scene = np.copy(difficulty_scenes_images_features[IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]])\n",
    "        else:\n",
    "            model_found_in_the_scene = np.copy(difficulty_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]])\n",
    "        # define new dimension that the model image should now have, \n",
    "        # which is the dimension of the probably found model in the scene \n",
    "        new_dim = (model_found_in_the_scene.shape[1], model_found_in_the_scene.shape[0])\n",
    "        \n",
    "        # model image resizing\n",
    "        resized_model_img = cv2.resize(model_img, new_dim, interpolation = cv2.INTER_AREA)\n",
    "        \n",
    "        # ValueError: operands could not be broadcast together with shapes (63,85,3) (85,63,3) \n",
    "\n",
    "        resized_model_img = resized_model_img.reshape(model_found_in_the_scene.shape[0], model_found_in_the_scene.shape[1],\n",
    "                                                     model_found_in_the_scene.shape[2])\n",
    "        \n",
    "        # perform difference of intensities between the scaled model image and the probably found model in the scene image \n",
    "        diff_image = resized_model_img -  model_found_in_the_scene\n",
    "        \n",
    "        # plot the resized model image\n",
    "        plt.imshow(cv2.cvtColor(resized_model_img, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "        \n",
    "        # plot the probably model found in the scene\n",
    "        plt.imshow(cv2.cvtColor(model_found_in_the_scene, cv2.COLOR_BGR2RGB))\n",
    "        plt.show() \n",
    "        \n",
    "        \n",
    "        \n",
    "        # divide the image containing the difference of intensities into N x M bins\n",
    "        means_bins_diff_img = split_image_into_N_x_M_bins_with_intensity_means(diff_image, \n",
    "                                                                               n_bins_width = N, \n",
    "                                                                               n_bins_heigth = M)\n",
    "        \n",
    "        if not means_bins_diff_img:\n",
    "            return []\n",
    "    \n",
    "        good = True\n",
    "        num_of_No_good = 0\n",
    "        \n",
    "        # cycle the model dictionary and get the means by getting the keys of the scene dictionary\n",
    "        for k, v in means_bins_diff_img.items():\n",
    "            means_k_diff = means_bins_diff_img.get(k)\n",
    "            \n",
    "            # PRINT ALL THE DIFFERENCES\n",
    "            print('Bin ({}, {}):'.format(k[0], k[1]))\n",
    "            print('  - diff_r: ', means_k_diff[0])\n",
    "            print('  - diff_g: ', means_k_diff[1])\n",
    "            print('  - diff_b: ', means_k_diff[2])\n",
    "            \n",
    "            if means_k_diff[0] >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES or \\\n",
    "                means_k_diff[1] >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES or \\\n",
    "                means_k_diff[2] >= COLOR_DIFF_IN_SINGLE_CHANNEL_TRES:\n",
    "                print('___________________NO GOOD___________________')\n",
    "                num_of_No_good += 1\n",
    "               \n",
    "                good = False\n",
    "        print('Num of No Good Bins: {}'.format(num_of_No_good))\n",
    "            \n",
    "        if num_of_No_good <= MAX_NUM_OF_NO_GOOD_CELLS: #good:\n",
    "            final_corners_of_bounding_boxes_after_color_problem.append(\n",
    "                [ (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                 (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))])\n",
    "            \n",
    "    \n",
    "    return final_corners_of_bounding_boxes_after_color_problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that plots the final merged bounding boxes\n",
    "def plot_final_bounding_boxes(img, final_corners_of_bounding_boxes, difficulty_scenes_images_features, j, thickness=20, \n",
    "                              shelf=False):\n",
    "    print('Final corners of bounding boxes:')\n",
    "    print(final_corners_of_bounding_boxes)\n",
    "    \n",
    "    if img is None:\n",
    "        if shelf:\n",
    "            img = np.copy(difficulty_scenes_images_features[IMAGE_INDEX])\n",
    "        else:\n",
    "            img = np.copy(difficulty_scenes_images_features[j][IMAGE_INDEX])\n",
    "    print(img.shape)\n",
    "\n",
    "    \n",
    "    # print final bounding boxes on an image\n",
    "    for top_left_corner, bottom_right_corner in final_corners_of_bounding_boxes:\n",
    "        scene_img_with_FINAL_bounding_boxes = cv2.rectangle(img, top_left_corner, bottom_right_corner, (0,255,0), thickness)\n",
    "\n",
    "    \n",
    "    \n",
    "    # plot image with final bounding boxes (with bounding boxes that do not overlap each other)\n",
    "    if(len(final_corners_of_bounding_boxes) > 0):\n",
    "        print('  - Found {} instances of model {} in scene {}'.format(len(final_corners_of_bounding_boxes),i,j))\n",
    "        # plotting the bounding box\n",
    "        plt.imshow(cv2.cvtColor(scene_img_with_FINAL_bounding_boxes, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "        \n",
    "        return scene_img_with_FINAL_bounding_boxes\n",
    "    else:\n",
    "        print('  - Model {} NOT found in the scene {}'.format(i,j))\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCorrelationComparingHistograms(img1, img2):\n",
    "    hsv_img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2HSV)\n",
    "    hsv_img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2HSV)\n",
    "\n",
    "    h_bins = 50\n",
    "    s_bins = 60\n",
    "    histSize = [h_bins, s_bins]\n",
    "    # hue varies from 0 to 179, saturation from 0 to 255\n",
    "    h_ranges = [0, 180]\n",
    "    s_ranges = [0, 256]\n",
    "    ranges = h_ranges + s_ranges # concat lists\n",
    "    # Use the 0-th and 1-st channels\n",
    "    channels = [0, 1]\n",
    "    \n",
    "    hist_img1 = cv2.calcHist([hsv_img1], channels, None, histSize, ranges, accumulate=False)\n",
    "    cv2.normalize(hist_img1, hist_img1, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX)\n",
    "\n",
    "    hist_img2 = cv2.calcHist([hsv_img2], channels, None, histSize, ranges, accumulate=False)\n",
    "    cv2.normalize(hist_img2, hist_img2, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX)\n",
    "\n",
    "    \n",
    "    return cv2.compareHist(hist_img1, hist_img2, 2) * 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve_color_problem_with_HSV_Histogram_Correlation(final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                                   difficulty_scenes_images_features, \n",
    "                                                   i, j,\n",
    "                                                   bb_conf_after_exceeding_dim,\n",
    "                                                   bb_conf_after_hsv_corr,\n",
    "                                                   cut_factor,\n",
    "                                                   TRES_MINIMUM_HISTOGRAM_CORR, \n",
    "                                                   shelf=False):\n",
    "    final_corners_of_bounding_boxes_after_color_problem = []\n",
    "    \n",
    "    model_img = np.copy(model_images_features[i][IMAGE_INDEX])\n",
    "    \n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes_without_exceeding_dimensions):\n",
    "        \n",
    "        # model probably found in the scene\n",
    "        if shelf:\n",
    "            model_found_in_the_scene = np.copy(difficulty_scenes_images_features[IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]])\n",
    "        else:\n",
    "            model_found_in_the_scene = np.copy(difficulty_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]])\n",
    "        \n",
    "        # define new dimension that the model image should now have, \n",
    "        # which is the dimension of the probably found model in the scene \n",
    "        new_dim = (model_found_in_the_scene.shape[1], model_found_in_the_scene.shape[0])\n",
    "        \n",
    "        # model image resizing\n",
    "        resized_model_img = cv2.resize(model_img, new_dim, interpolation = cv2.INTER_AREA)\n",
    "        resized_model_img = resized_model_img.reshape(model_found_in_the_scene.shape[0], model_found_in_the_scene.shape[1],\n",
    "                                                     model_found_in_the_scene.shape[2])\n",
    "        \n",
    "        \n",
    "        height_resized_model_img, width_resized_model_img = resized_model_img.shape[:2]\n",
    "        height_model_found_in_the_scene, width_model_found_in_the_scene = model_found_in_the_scene.shape[:2]\n",
    "\n",
    "        # cut 1/10 of the the top and the bottom image before comparing histograms\n",
    "        resized_model_img = resized_model_img[\\\n",
    "                            int(height_resized_model_img/cut_factor):height_resized_model_img-int(height_resized_model_img/cut_factor),\n",
    "                                0:width_resized_model_img ]\n",
    "        \n",
    "        model_found_in_the_scene = model_found_in_the_scene[\\\n",
    "                            int(height_model_found_in_the_scene/cut_factor):height_model_found_in_the_scene-int(height_model_found_in_the_scene/cut_factor),\n",
    "                                0:width_model_found_in_the_scene ]\n",
    "                \n",
    "        histogram_correlation = getCorrelationComparingHistograms(resized_model_img, model_found_in_the_scene)\n",
    "        plt.imshow(cv2.cvtColor(model_found_in_the_scene, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "        plt.imshow(cv2.cvtColor(resized_model_img, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "        print('  -Histogram HSV Correlation: {}'.format(histogram_correlation))\n",
    "        \n",
    "        # keep track of the confidence of hsv histograms correlation\n",
    "        hsv_histogram_corr_conf = (histogram_correlation / TRES_MINIMUM_HISTOGRAM_CORR) \n",
    "        bb_conf_after_hsv_corr[(fin_top_left_corner, fin_bottom_right_corner)] = \\\n",
    "                bb_conf_after_exceeding_dim [(fin_top_left_corner, fin_bottom_right_corner)] \n",
    "        bb_conf_after_hsv_corr[(fin_top_left_corner, fin_bottom_right_corner)].append(hsv_histogram_corr_conf)\n",
    "        \n",
    "        \n",
    "        if histogram_correlation >= TRES_MINIMUM_HISTOGRAM_CORR: #good:\n",
    "            print('\\tGOOD HSV CORR')\n",
    "            final_corners_of_bounding_boxes_after_color_problem.append(\n",
    "                [ (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                 (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))])\n",
    "        else:\n",
    "            print('\\tNO GOOD HSV CORR')\n",
    "            \n",
    "    \n",
    "    return final_corners_of_bounding_boxes_after_color_problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_treshold_min_hsv_histogram_correlations(final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                                   difficulty_scenes_images_features, \n",
    "                                                   i, j,\n",
    "                                                   cut_factor, shelf=False):\n",
    "    all_histogram_correlations = []\n",
    "    model_img = np.copy(model_images_features[i][IMAGE_INDEX])\n",
    "    \n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes_without_exceeding_dimensions):\n",
    "        \n",
    "        # model probably found in the scene\n",
    "        if shelf:\n",
    "            model_found_in_the_scene = np.copy(difficulty_scenes_images_features[IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]])\n",
    "        else:\n",
    "            model_found_in_the_scene = np.copy(difficulty_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]])\n",
    "        \n",
    "        # define new dimension that the model image should now have, \n",
    "        # which is the dimension of the probably found model in the scene \n",
    "        new_dim = (model_found_in_the_scene.shape[1], model_found_in_the_scene.shape[0])\n",
    "        \n",
    "        # model image resizing\n",
    "        resized_model_img = cv2.resize(model_img, new_dim, interpolation = cv2.INTER_AREA)\n",
    "        resized_model_img = resized_model_img.reshape(model_found_in_the_scene.shape[0], model_found_in_the_scene.shape[1],\n",
    "                                                     model_found_in_the_scene.shape[2])\n",
    "        \n",
    "        \n",
    "        height_resized_model_img, width_resized_model_img = resized_model_img.shape[:2]\n",
    "        height_model_found_in_the_scene, width_model_found_in_the_scene = model_found_in_the_scene.shape[:2]\n",
    "\n",
    "        # cut 1/10 of the the top and the bottom image before comparing histograms\n",
    "        resized_model_img = resized_model_img[\\\n",
    "                            int(height_resized_model_img/cut_factor):height_resized_model_img-int(height_resized_model_img/cut_factor),\n",
    "                                0:width_resized_model_img ]\n",
    "        \n",
    "        model_found_in_the_scene = model_found_in_the_scene[\\\n",
    "                            int(height_model_found_in_the_scene/cut_factor):height_model_found_in_the_scene-int(height_model_found_in_the_scene/cut_factor),\n",
    "                                0:width_model_found_in_the_scene ]\n",
    "                \n",
    "        all_histogram_correlations.append( getCorrelationComparingHistograms(resized_model_img, model_found_in_the_scene) )\n",
    "    \n",
    "    #print('Current Correlation {}'.format(getCorrelationComparingHistograms(resized_model_img, model_found_in_the_scene)))\n",
    "    if len(all_histogram_correlations) > 0:\n",
    "        print('MEAN {}'.format( np.mean(all_histogram_correlations) ))\n",
    "        print('(Max - Min) / 2 {}'.format( (np.max(all_histogram_correlations) - np.min(all_histogram_correlations))/2 ) )\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_OF_MODELS = 27\n",
    "\n",
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "CENTER_INDEX     = 3\n",
    "V_INDEX          = 4\n",
    "TRESHOLD_INDEX = 5\n",
    "\n",
    "# Dictionary that contains the image, all keypoints and descriptors for each model images\n",
    "model_images_features = {}\n",
    "mean_of_model_intensities_r_g_b = {}\n",
    "\n",
    "for i in range(NUM_OF_MODELS):\n",
    "    model_img = cv2.imread('./models/{}.jpg'.format(i), cv2.COLOR_BGR2GRAY)# cv2.COLOR_BGR2RGB)\n",
    "    # save the blurred model image\n",
    "    blur_model_image = cv2.GaussianBlur(model_img, (15,15),0) #(45,45)\n",
    "\n",
    "    kp_model = sift.detect(blur_model_image)\n",
    "    kp_model, des_model = sift.compute(blur_model_image, kp_model)\n",
    "    model_images_features[i] = [blur_model_image, kp_model, des_model]\n",
    "    \n",
    "    b,g,r = cv2.split(model_images_features[i][IMAGE_INDEX])\n",
    "    # save the mean of the intensities (divided per channel) for every model image\n",
    "    mean_of_model_intensities_r_g_b[i] = [np.mean(r), np.mean(g), np.mean(b)]\n",
    "    # print(mean_of_model_intensities_r_g_b[i][0], mean_of_model_intensities_r_g_b[i][1], mean_of_model_intensities_r_g_b[i][2])\n",
    "    # print(model_images_features[i][IMAGE_INDEX][0].shape)\n",
    "    # print(mean_of_model_intensities[i])\n",
    "\n",
    "    plt.imshow(cv2.cvtColor(model_img, cv2.COLOR_BGR2RGB))\n",
    "    plt.show()\n",
    "    plt.imshow(cv2.cvtColor(model_images_features[i][IMAGE_INDEX], cv2.COLOR_BGR2RGB))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "CENTER_INDEX     = 3\n",
    "V_INDEX          = 4\n",
    "NUM_OF_MODELS = 27\n",
    "\n",
    "for i in range(NUM_OF_MODELS):\n",
    "\n",
    "    model_img_z = model_images_features[i][IMAGE_INDEX]\n",
    "\n",
    "    #Find or update the center of the model image information collected in the model features dictionary\n",
    "    InstanceCenter(model_img_z,model_images_features, i, CENTER_INDEX=3) \n",
    "\n",
    "    # Print the model images\n",
    "    #Print_Center_Keypoint(model_img_z, model_images_features, good,Img_scale=1000)\n",
    "\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that split an image into (n_bins_heigth x n_bins_width) bins and returns a dictonary\n",
    "# that contains the means of the 3 color channels of the (n_bins_heigth x n_bins_width) bins\n",
    "def split_image_into_N_x_M_bins_with_intensity(image, n_bins_width = 3, n_bins_heigth = 4):\n",
    "    img_N_x_M_bins = {}\n",
    "    \n",
    "    img2 = np.copy(image)\n",
    "    \n",
    "    img_width = image.shape[1]\n",
    "    img_height = image.shape[0]\n",
    "    \n",
    "    step_width = int(img_width / n_bins_width)\n",
    "    step_height = int(img_height / n_bins_heigth)\n",
    "    \n",
    "    r = 0\n",
    "    c = 0\n",
    "    \n",
    "    print('img_height: ', img_height)\n",
    "    print('step_height: ', step_height)\n",
    "    if img_height != 0 and step_height != 0 and img_width != 0 and step_width != 0:\n",
    "        for row in np.arange(0, img_height, step_height):\n",
    "            c = 0\n",
    "            cv2.line(img2,(0, row),(img_width, row),(0,0,0),3) \n",
    "            for col in np.arange(0, img_width, step_width):\n",
    "                # print('row {} col {}'.format(r,c))\n",
    "                if row + 2 * step_height > img_height and col + 2 * step_width > img_width:\n",
    "                    partial_bin = image[row:, col:]\n",
    "\n",
    "                elif row + 2 * step_height > img_height:\n",
    "                    partial_bin = image[row:, col : col + step_width]\n",
    "\n",
    "                elif col + 2 * step_width > img_width:\n",
    "                    partial_bin = image[row : row + step_height, col :]\n",
    "                else:\n",
    "                    partial_bin = image[row : row + step_height, col : col + step_width]\n",
    "\n",
    "                cv2.line(img2,(col, 0),(col, img_height),(0,0,0),3) \n",
    "                if r < n_bins_heigth and c < n_bins_width:\n",
    "                    # save the 3 channels (r,g,b) of each bin\n",
    "                    img_N_x_M_bins[r, c] = partial_bin\n",
    "                    # IF YOU WANT TO VISUALIZE EACH BIN UNCOMMENT THE FOLLOWING LINES:\n",
    "                    # plot each bin in blue channel color\n",
    "                    # plt.imshow(cv2.cvtColor(partial_b_channel, cv2.COLOR_BGR2RGB))\n",
    "                    # plt.show()\n",
    "\n",
    "                    # salva le medie e non le immagini dei bins\n",
    "\n",
    "                c += 1\n",
    "            r += 1\n",
    "        plt.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()  \n",
    "    return img_N_x_M_bins\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def solve_color_problem_with_N_x_M_bins_and_histogram_HSV_correlation(\n",
    "                                    final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                    difficulty_scenes_images_features, \n",
    "                                    TRES_MINIMUM_HISTOGRAM_CORR,\n",
    "                                    i, j, N=3, M=4, \n",
    "                                    MIN_NUM_OF_GOOD_CELLS = 3,\n",
    "                                    shelf=False):\n",
    "    final_corners_of_bounding_boxes_after_color_problem = []\n",
    "    for index, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes_without_exceeding_dimensions):\n",
    "    \n",
    "        model_img = model_images_features[i][IMAGE_INDEX]\n",
    "        # smooth\n",
    "        blur_model_img = model_img #cv2.GaussianBlur(model_img,(45,45),0) #(45,45)\n",
    "        \n",
    "        means_bins_model_img = split_image_into_N_x_M_bins_with_intensity(\n",
    "            blur_model_img, \n",
    "            N, M)\n",
    "        \n",
    "        \n",
    "        \n",
    "        print(fin_top_left_corner[1], fin_top_left_corner[0], fin_bottom_right_corner[1], fin_bottom_right_corner[0])\n",
    "        \n",
    "        if shelf:\n",
    "            means_bins_scene_img = split_image_into_N_x_M_bins_with_intensity(\n",
    "                difficulty_scenes_images_features[IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]], \n",
    "                N, M)\n",
    "        else:\n",
    "            means_bins_scene_img = split_image_into_N_x_M_bins_with_intensity(\n",
    "                difficulty_scenes_images_features[j][IMAGE_INDEX][fin_top_left_corner[1]:fin_bottom_right_corner[1], \n",
    "                                                                       fin_top_left_corner[0]:fin_bottom_right_corner[0]], \n",
    "                N, M)\n",
    "    \n",
    "        if not means_bins_scene_img or not means_bins_scene_img:\n",
    "            return []\n",
    "    \n",
    "        good = True\n",
    "        num_of_good = 0\n",
    "\n",
    "        \n",
    "        # cycle the model dictionary and get the diff of the means by getting the keys of the scene dictionary\n",
    "        for k, v in means_bins_model_img.items():\n",
    "            \n",
    "            # do not consider all bins that are on the top and on the bottom\n",
    "            if k[0] != 0 and k[0] != N:\n",
    "            \n",
    "                print('CHIAVE')\n",
    "                print(k[0], k[1])\n",
    "                means_k_scene = means_bins_scene_img.get(k)\n",
    "                # I consider a bounding box as good if and only if all the difference of the intiensities from the \n",
    "                # bins of the model, in all 3 channels, are below a certain treshold\n",
    "                #diff_r = np.absolute(v[0] - means_k_scene[0])\n",
    "                #diff_g = np.absolute(v[1] - means_k_scene[1])\n",
    "                #diff_b = np.absolute(v[2] - means_k_scene[2])\n",
    "\n",
    "                print('Bin ({}, {}):'.format(k[0], k[1]))\n",
    "                #print('  - diff_r: ', diff_r)\n",
    "                #print('  - diff_g: ', diff_g)\n",
    "                #print('  - diff_b: ', diff_b)\n",
    "                histogram_correlation = getCorrelationComparingHistograms(v, means_k_scene)\n",
    "                print('histogram correlation between current bins: {}'.format(histogram_correlation))\n",
    "\n",
    "\n",
    "                if histogram_correlation >= TRES_MINIMUM_HISTOGRAM_CORR:\n",
    "                    print('___________________GOOD HISTOGRAM CORRELATION______________________')\n",
    "                    num_of_good += 1\n",
    "                else:\n",
    "                    good = False                    \n",
    "                    print('___________________NO GOOD HISTOGRAM CORRELATION___________________')\n",
    "\n",
    "            \n",
    "        if  num_of_good >= MIN_NUM_OF_GOOD_CELLS: #good:\n",
    "            final_corners_of_bounding_boxes_after_color_problem.append([ (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                                                        (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))])\n",
    "            \n",
    "    \n",
    "    return final_corners_of_bounding_boxes_after_color_problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nearest_corner_in_dict(corner, bb_conf_after_hsv_corr):\n",
    "    i = 0\n",
    "    min_dist = 100000\n",
    "    for top_left, bottom_right in bb_conf_after_hsv_corr.keys():\n",
    "        if i == 0:\n",
    "            nearest = (top_left, bottom_right)\n",
    "            min_dist = distance_2_points(top_left, corner[0])\n",
    "\n",
    "        min_dist_tmp = distance_2_points(top_left, corner[0])\n",
    "        if min_dist_tmp < min_dist:\n",
    "            nearest = (top_left, bottom_right)\n",
    "            min_dist = min_dist_tmp\n",
    "        #print(top_left, bottom_right)\n",
    "        #print(nearest)\n",
    "        #print(min_dist)\n",
    "\n",
    "        i += 1\n",
    "    return nearest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that merges adjacent bounding boxes that overlap between each other       \n",
    "def merge_overlapping_bounding_boxes(corners_of_bounding_boxes, bb_conf_after_hsv_corr, \n",
    "                                     final_bb_conf, i, j, shelf_index,\n",
    "                                     DISTANCE_BOUNDING_BOXES_TRESHOLD=200):\n",
    "    # top left and bottom right corners of the final bounding boxes\n",
    "    final_corners_of_bounding_boxes = []\n",
    "    \n",
    "    for index_1, [top_left_corner, bottom_right_corner] in enumerate(corners_of_bounding_boxes):\n",
    "            # add first couple of top left and bottom right corners of the final bounding boxes\n",
    "            if len(final_corners_of_bounding_boxes) == 0:\n",
    "                final_corners_of_bounding_boxes.append([ (int(top_left_corner[0]), int(top_left_corner[1])), \n",
    "                                                            (int(bottom_right_corner[0]), int(bottom_right_corner[1])) ])\n",
    "                # keep track of confidences\n",
    "                final_bb_conf[(int(top_left_corner[0]), int(top_left_corner[1])), \n",
    "                              (int(bottom_right_corner[0]), int(bottom_right_corner[1])),\n",
    "                             i, j, shelf_index] = \\\n",
    "                            bb_conf_after_hsv_corr[(int(top_left_corner[0]), int(top_left_corner[1])), \n",
    "                              (int(bottom_right_corner[0]), int(bottom_right_corner[1]))]\n",
    "                final_bb_conf[(int(top_left_corner[0]), int(top_left_corner[1])), \n",
    "                              (int(bottom_right_corner[0]), int(bottom_right_corner[1])),\n",
    "                             i, j, shelf_index].append((i, j, shelf_index))\n",
    "\n",
    "            \n",
    "            \n",
    "            for index_2, [fin_top_left_corner, fin_bottom_right_corner] in enumerate(final_corners_of_bounding_boxes):\n",
    "                # if a corner is already in the final corners then I don't add it\n",
    "                if top_left_corner == fin_top_left_corner and bottom_right_corner == fin_bottom_right_corner:\n",
    "                    break\n",
    "                # if a corner is near (below DISTANCE_BOUNDING_BOXES_TRESHOLD) a final corner, then I mean the two into a\n",
    "                # single one\n",
    "                if ( distance_2_points(top_left_corner, fin_top_left_corner) < DISTANCE_BOUNDING_BOXES_TRESHOLD and \n",
    "                distance_2_points(fin_bottom_right_corner, bottom_right_corner) < DISTANCE_BOUNDING_BOXES_TRESHOLD ) :\n",
    "                    \n",
    "                    sum_top_left = tuple(map(operator.add, top_left_corner, fin_top_left_corner))\n",
    "                    sum_bottom_right = tuple(map(operator.add, bottom_right_corner ,fin_bottom_right_corner))\n",
    "                    mean_top_left = (sum_top_left[0]/2, sum_top_left[1]/2)\n",
    "                    mean_bottom_right = (sum_bottom_right[0]/2, sum_bottom_right[1]/2)\n",
    "                    print('mean_top_left', mean_top_left)\n",
    "                    final_corners_of_bounding_boxes[index_2] = [ (int(mean_top_left[0]), int(mean_top_left[1])), \n",
    "                                                            (int(mean_bottom_right[0]), int(mean_bottom_right[1]))]\n",
    "                    \n",
    "                    # keep track of confidences\n",
    "                    final_bb_conf.pop((fin_top_left_corner, fin_bottom_right_corner, i, j, shelf_index), None)\n",
    "                    \n",
    "                    corner = ( (int(fin_top_left_corner[0]), int(fin_top_left_corner[1])), \n",
    "                                  (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1])) )\n",
    "                    nearest_corner = nearest_corner_in_dict(corner, bb_conf_after_hsv_corr)\n",
    "                    \n",
    "                    print(nearest_corner)\n",
    "                    \n",
    "                    final_bb_conf[(int(mean_top_left[0]), int(mean_top_left[1])), \n",
    "                                  (int(mean_bottom_right[0]), int(mean_bottom_right[1])),\n",
    "                                 i, j, shelf_index] = \\\n",
    "                                        bb_conf_after_hsv_corr[nearest_corner]\n",
    "                        \n",
    "                        #int(fin_top_left_corner[0]), int(fin_top_left_corner[1])),\n",
    "                        #                                       (int(fin_bottom_right_corner[0]), int(fin_bottom_right_corner[1]))]\n",
    "                    final_bb_conf[(int(mean_top_left[0]), int(mean_top_left[1])), \n",
    "                                  (int(mean_bottom_right[0]), int(mean_bottom_right[1])),\n",
    "                                 i, j, shelf_index].append((i, j, shelf_index))\n",
    "\n",
    "\n",
    "            \n",
    "        \n",
    "                    break\n",
    "                # if my corner is'n near any final corner then I add it to the final corners\n",
    "                if index_2 == len(final_corners_of_bounding_boxes) - 1:\n",
    "                    final_corners_of_bounding_boxes.append([ (int(top_left_corner[0]), int(top_left_corner[1])), \n",
    "                                                            (int(bottom_right_corner[0]), int(bottom_right_corner[1]))])\n",
    "                    # keep track of confidences\n",
    "                    final_bb_conf[ top_left_corner, bottom_right_corner , \n",
    "                                  i, j, shelf_index] = bb_conf_after_hsv_corr[(top_left_corner, bottom_right_corner)]\n",
    "                    final_bb_conf[ top_left_corner, bottom_right_corner, i, j, shelf_index].append((i, j, shelf_index))\n",
    "                    \n",
    "    \n",
    "\n",
    "    return final_corners_of_bounding_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tres_min_hsv_corr = [169, 71.5, 105, 195, 68, 70, 76, 111, 26, 11, 18, 59, 14, 14, 70, 75, 150, 57, 60, 43, 31, 22, 115, 10.5, 70, 150, 64]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### STPEC DIVINDING SHELVES\n",
    "\n",
    "IMAGE_INDEX      = 0\n",
    "KEYPOINT_INDEX   = 1\n",
    "DESCRIPTOR_INDEX = 2\n",
    "\n",
    "models_to_test = list(range(0, 24))\n",
    "scenes_to_test = [1,2,3,4,5] #[2]#[1,2,3,4,5]\n",
    "\n",
    "\n",
    "h_results = {}\n",
    "\n",
    "final_shelf_images_with_bb = {}\n",
    "\n",
    "tres_to_get_min_keypoints = {}\n",
    "\n",
    "\n",
    "min_votes_confidence = {}\n",
    "\n",
    "bounding_box_confidence = {}\n",
    "\n",
    "bb_conf_after_exceeding_dim = {}\n",
    "\n",
    "bb_conf_after_color_solver = {}\n",
    "\n",
    "final_bb_conf = {}\n",
    "\n",
    "bb_conf_after_hsv_corr = {}\n",
    "\n",
    "\n",
    "#dimension of a single cell of the accumulator array\n",
    "ACC_ARRAY_CELL_DIMENSION_1 = 30\n",
    "ACC_ARRAY_CELL_DIMENSION_2 = 30\n",
    "\n",
    "\n",
    "# Parameters to tune\n",
    "# minimum number of votes to consider G as a valid point\n",
    "MIN_VOTES = 1\n",
    "#treshold passed to the Matching function\n",
    "# MATCHING_TRESHOLD = 0.55\n",
    "\n",
    "# distance to merge two bounding boxes\n",
    "DISTANCE_BOUNDING_BOXES_TRESHOLD = 50\n",
    "\n",
    "# color parameters\n",
    "COLOR_DIFF_IN_SINGLE_CHANNEL_TRES_FIRST_BINS = 79#66#60#65.9#70#75#80#92 \n",
    "COLOR_DIFF_IN_SINGLE_CHANNEL_TRES_FIRST_DIFFERENCE = 145\n",
    "\n",
    "# number of bins for the color problem\n",
    "N_BINS_ON_WIDTH = 3\n",
    "\n",
    "M_BINS_ON_HEIGHT = 4\n",
    "\n",
    "# if the num of no good cells is <= of this number then I take the bounding box\n",
    "MAX_NUM_OF_NO_GOOD_CELLS = 3#2\n",
    "\n",
    "# only if you use solve_color_problem_with_N_x_M_bins_and_histogram_HSV_correlation\n",
    "MIN_NUM_OF_GOOD_CELLS = 3\n",
    "\n",
    "# treshold for keypoints parameters\n",
    "MIN_TRESHOLD = 0.2\n",
    "MAX_TRESHOLD = 0.65#0.53#0.50\n",
    "STEP_SIZE = 0.05\n",
    "\n",
    "# MIN_NUM_OF_KEYPOINTS_IN_SCENE = 125#15#8#15#10#15 #10\n",
    "\n",
    "# ERROR = 60#40\n",
    "\n",
    "# minimum treshold of correlation of hsv histograms\n",
    "TRES_MINIMUM_HISTOGRAM_CORR = 50#70\n",
    "# amount of cutting (1/N) on top and of bottom of of the images when considering HSV Hist Correlation\n",
    "CUT_FACTOR_FOR_HIST_CORR = 7\n",
    "\n",
    "# cycle scenes\n",
    "for j in scenes_to_test:\n",
    "    h_results[j] = {}\n",
    "    \n",
    "    final_shelf_images_with_bb[j] = {}\n",
    "    \n",
    "    # cycle shelves in scene\n",
    "    for shelf_index, shelf_features in scenes_shelves_features[j].items():\n",
    "        h_results[j][shelf_index] = {}\n",
    "        \n",
    "        final_shelf_images_with_bb[j][shelf_index] = np.copy(shelf_features[IMAGE_INDEX])\n",
    "        \n",
    "        if shelf_features[IMAGE_INDEX].shape[0] < 80: \n",
    "            ERROR = int(shelf_features[IMAGE_INDEX].shape[0] / 0.9)\n",
    "        else:\n",
    "            ERROR = int(shelf_features[IMAGE_INDEX].shape[0] / 3)\n",
    "            ERROR_width = int(shelf_features[IMAGE_INDEX].shape[0] / 3.5)\n",
    "        \n",
    "        # cycle models\n",
    "        for i in models_to_test:\n",
    "            \n",
    "\n",
    "            \n",
    "            model_img = np.copy(model_images_features[i][IMAGE_INDEX])\n",
    "\n",
    "            height_of_model, width_of_model = model_img.shape[:2]\n",
    "            \n",
    "            r_model = height_of_model / width_of_model\n",
    "\n",
    "\n",
    "            print(('_' * 30) + 'Finding model {} in shelf {} of scene {}'.format(i, shelf_index, j) + ('_' * 30))\n",
    "\n",
    "            show(shelf_features[IMAGE_INDEX])\n",
    "\n",
    "            shelf_img = np.copy(shelf_features[IMAGE_INDEX])\n",
    "            \n",
    "            height_of_shelf, width_of_shelf = shelf_img.shape[:2]\n",
    "            \n",
    "\n",
    "\n",
    "            # dictionary that will contain information for the final output\n",
    "            h_results[j][shelf_index][i] = {}\n",
    "            h_results[j][shelf_index][i]['count'] = 0\n",
    "\n",
    "\n",
    "            # cycle all tresholds -> lo tolgo momentaneamente\n",
    "            #for treshold in np.arange(MIN_TRESHOLD, MAX_TRESHOLD, STEP_SIZE):\n",
    "                #Finding matches between the keypoints of the scene and the keypoints found in the model\n",
    "                # Special move\n",
    "            #for special_move in [1,2]:\n",
    "            good = Matching(model_images_features[i][DESCRIPTOR_INDEX], shelf_features[DESCRIPTOR_INDEX], \n",
    "                        Treshold=MAX_TRESHOLD)\n",
    "                            #    Treshold=treshold)\n",
    "\n",
    "                # if the number of keypoints of the current model in the current scene is greater than th min required, then \n",
    "                # I save the treshold for this model\n",
    "            #    if len(good) >= MIN_NUM_OF_KEYPOINTS_IN_SCENE:\n",
    "                    # Updating the model feature dictionary appending the information of the treshold\n",
    "            #         tres_to_get_min_keypoints[i] = treshold\n",
    "            #         break\n",
    "\n",
    "            V = VotingVectors(good, model_images_features, i)\n",
    "\n",
    "            \n",
    "            ACC, G_scene, Accumulator_Array_Points = Accumulator_Array(ACC_ARRAY_CELL_DIMENSION_1,\n",
    "                                                                          ACC_ARRAY_CELL_DIMENSION_2,\n",
    "                                                                          shelf_img, \n",
    "                                                                          model_img,\n",
    "                                                                          model_images_features, \n",
    "                                                                          shelf_features, \n",
    "                                                                          good, \n",
    "                                                                          V, \n",
    "                                                                          i, j,\n",
    "                                                                          shelf = True)\n",
    "            \n",
    "            dst_pts = np.float32([ shelf_features[KEYPOINT_INDEX][m.trainIdx].pt for m in good ]).reshape(-1,1,2)\n",
    "            shelf_img_with_keypoints = np.copy(shelf_img)\n",
    "            \n",
    "            # plot scene image with keypoints\n",
    "            for d in dst_pts:\n",
    "                cv2.circle(shelf_img_with_keypoints, (int(d[0][0]), int(d[0][1])), 1, (0,0,0),10)\n",
    "                \n",
    "            # create an image to plot the estimated centers\n",
    "            shelf_img_with_estimated_centers = np.copy(shelf_img)\n",
    "\n",
    "            for g in G_scene :\n",
    "                #Drawing a dot in the position of G\n",
    "                #print(g)\n",
    "                #if np.max(ACC)>=20:\n",
    "                cv2.circle(shelf_img,(int(g[0]), int(g[1])), 5, (255,55,236), 5)\n",
    "                cv2.circle(shelf_img_with_estimated_centers,(int(g[0]), int(g[1])), 3, (255,55,236), 3)\n",
    "        \n",
    "            # List of indexes of highlighted cells (list of cells that have a num votes >= MIN_VOTES)\n",
    "            highlighted_cells_of_current_model_in_current_shelf = []\n",
    "            '''\n",
    "            print('List of highlighted cells:')\n",
    "            '''\n",
    "            # print(ACC.shape[0], ACC.shape[1])\n",
    "            for t in range(ACC.shape[0]):\n",
    "                for w in range(ACC.shape[1]):\n",
    "                    #print(t,w)\n",
    "                    # If a cell of the accumulator array has more than MIN_VOTES than it means that the model has been found\n",
    "                    if ACC[t,w] >= MIN_VOTES:\n",
    "                        print('  - Num votes: {} in cell {} '.format(ACC[t,w], (t,w)))\n",
    "                        highlighted_cells_of_current_model_in_current_shelf.append([t, w])\n",
    "                        # keep track of the confidence by using the num of votes\n",
    "                        min_votes_confidence[(t,w)] = ( ACC[t,w] / MIN_VOTES )\n",
    "\n",
    "\n",
    "            # estimated centers of the bounding boxes\n",
    "            G_scenes, r = center_estimation(model_images_features, shelf_features, good, V, j, shelf=True)\n",
    "\n",
    "            # print('r: ', r)\n",
    "            \n",
    "            \n",
    "            \n",
    "            # getting the scaled height and width of the model in the scene\n",
    "            model_height_in_the_scene = model_images_features[i][IMAGE_INDEX].shape[0] / r                                               \n",
    "            model_width_in_the_scene = model_images_features[i][IMAGE_INDEX].shape[1] / r\n",
    "\n",
    "            shelf_img_copy = np.copy(shelf_features[IMAGE_INDEX])\n",
    "\n",
    "\n",
    "\n",
    "            corners_of_bounding_boxes = []\n",
    "\n",
    "            for c in highlighted_cells_of_current_model_in_current_shelf:\n",
    "                # Compute G (the center of the cereal box in the scene) as the mean of points that fall \n",
    "                # into the higlighted cell\n",
    "                G_mean = np.mean(Accumulator_Array_Points[(c[0], c[1], 'S')], axis=0)\n",
    "                cv2.circle(shelf_img,(int(G_mean[0]), int(G_mean[1])), 8, (0,0,0), 5)\n",
    "\n",
    "                cv2.rectangle(shelf_img,(c[1]*ACC_ARRAY_CELL_DIMENSION_1,c[0]*ACC_ARRAY_CELL_DIMENSION_2),\n",
    "                              ((c[1]+1)*ACC_ARRAY_CELL_DIMENSION_1,(c[0]+1)*ACC_ARRAY_CELL_DIMENSION_2),(0,255,0),10)\n",
    "\n",
    "            \n",
    "                # I compute the top left corner and bottom right corner of the bounding box so that I will be able to draw\n",
    "                # the bounding box with the rectangle function of OpenCV\n",
    "                top_left_corner_of_bounding_box = ( int(int(G_mean[0]) - (model_width_in_the_scene / 2) ), \n",
    "                                                int(int(G_mean[1]) - (model_height_in_the_scene / 2) ) )\n",
    "                bottom_right_corner_of_bounding_box = ( int(int(G_mean[0]) + (model_width_in_the_scene / 2) ), \n",
    "                                                int(int(G_mean[1]) + (model_height_in_the_scene / 2) ) )\n",
    "                \n",
    "                # (w,h)\n",
    "                top_right_corner_of_bounding_box = (bottom_right_corner_of_bounding_box[0], \n",
    "                                                    top_left_corner_of_bounding_box[1])\n",
    "                \n",
    "                bottom_left_corner_of_bounding_box = (top_left_corner_of_bounding_box[0],\n",
    "                                                     bottom_right_corner_of_bounding_box[1])\n",
    "                \n",
    "                \n",
    "                height_bb_before_processing = distance_2_points(top_left_corner_of_bounding_box, \n",
    "                                                                bottom_left_corner_of_bounding_box)\n",
    "                \n",
    "                width_bb_before_processing = distance_2_points(top_left_corner_of_bounding_box, \n",
    "                                                               top_right_corner_of_bounding_box)\n",
    "                \n",
    "                \n",
    "\n",
    "                \n",
    "                # consider a bounding box good only if it does not exceed WIDTH_MAX and HEIGHT_MAX dimensions\n",
    "                max_height = height_of_shelf + ERROR\n",
    "                max_width = max_height / r_model #int(width_of_shelf / r_model) # int((height_of_shelf + ERROR) / r_model)\n",
    "                \n",
    "                #print('r_model: {}'.format(r_model))                \n",
    "                #print('max_height: {}'.format(max_height))\n",
    "                #print('max_width: {}'.format(max_width))\n",
    "\n",
    "                #print('ERROR: {}'.format(ERROR))\n",
    "\n",
    "                if height_bb_before_processing <= max_height and width_bb_before_processing <= max_width:\n",
    "                    corners_of_bounding_boxes.append([top_left_corner_of_bounding_box, \n",
    "                                                      bottom_right_corner_of_bounding_box])\n",
    "                    # keep track of the confidence of bounding boxes\n",
    "                    bounding_box_confidence[(top_left_corner_of_bounding_box, \n",
    "                                                      bottom_right_corner_of_bounding_box)] = [min_votes_confidence[(c[0], \n",
    "                                                                                                                     c[1])]]\n",
    "\n",
    "                shelf_img_with_bounding_boxes = cv2.rectangle(shelf_img_copy,\n",
    "                                              top_left_corner_of_bounding_box,\n",
    "                                              bottom_right_corner_of_bounding_box,\n",
    "                                              (0,255,0), 10)\n",
    "                '''\n",
    "                print('Bounding box:')\n",
    "                print('  - top left corner: {}'.format((top_left_corner_of_bounding_box[0], \n",
    "                                                top_left_corner_of_bounding_box[1])))\n",
    "                print('  - bottom right corner: {}'.format((bottom_right_corner_of_bounding_box[0], \n",
    "                                                bottom_right_corner_of_bounding_box[1])))\n",
    "\n",
    "                '''\n",
    "            # solve exceeding dimensions of bounding boxes in the scene\n",
    "            final_corners_of_bounding_boxes_without_exceeding_dimensions = solve_exceeding_dimensions_of_bounding_boxes_in_scene(\n",
    "                corners_of_bounding_boxes, shelf_features, bounding_box_confidence, bb_conf_after_exceeding_dim, shelf=True)\n",
    "\n",
    "            \n",
    "            \n",
    "            # solve color problem\n",
    "            #final_corners_of_bounding_boxes_after_color_problem = solve_color_problem(\n",
    "            #    final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "            #    h_scenes_images_features, j)\n",
    "\n",
    "            # solve color problem with bins (first difference, then bins)\n",
    "            #final_corners_of_bounding_boxes_after_color_problem = solve_color_problem_with_difference_N_x_M_bins(\n",
    "            #                                           final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "            #                                           shelf_features,\n",
    "            #                                           i, j, N=N_BINS_ON_WIDTH, M=M_BINS_ON_HEIGHT, \n",
    "            #                                           COLOR_DIFF_IN_SINGLE_CHANNEL_TRES=COLOR_DIFF_IN_SINGLE_CHANNEL_TRES_FIRST_DIFFERENCE,\n",
    "            #                                           MAX_NUM_OF_NO_GOOD_CELLS=MAX_NUM_OF_NO_GOOD_CELLS, shelf=True)\n",
    "\n",
    "            # solve color problem with bins (first bins, then difference)\n",
    "            \n",
    "            final_corners_of_bounding_boxes_after_color_problem = solve_color_problem_with_N_x_M_bins(\n",
    "                final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                # final_corners_of_bounding_boxes_after_color_problem, # this or the previous line (this if I use also the color solver with difference)\n",
    "                shelf_features, bb_conf_after_exceeding_dim, bb_conf_after_color_solver, i, j, N=N_BINS_ON_WIDTH, M=M_BINS_ON_HEIGHT, \n",
    "                COLOR_DIFF_IN_SINGLE_CHANNEL_TRES=COLOR_DIFF_IN_SINGLE_CHANNEL_TRES_FIRST_BINS, \n",
    "                MAX_NUM_OF_NO_GOOD_CELLS=MAX_NUM_OF_NO_GOOD_CELLS,\n",
    "                shelf=True)\n",
    "            \n",
    "            \n",
    "            '''\n",
    "            get_treshold_min_hsv_histogram_correlations(final_corners_of_bounding_boxes_after_color_problem, \n",
    "                                                   shelf_features,\n",
    "                                                   i, j,\n",
    "                                                   CUT_FACTOR_FOR_HIST_CORR, shelf=True)\n",
    "            '''\n",
    "            \n",
    "            final_corners_of_bounding_boxes_after_color_problem = solve_color_problem_with_HSV_Histogram_Correlation(\n",
    "                                                    final_corners_of_bounding_boxes_after_color_problem, \n",
    "                                                    shelf_features, \n",
    "                                                    i, j,\n",
    "                                                    bb_conf_after_color_solver,\n",
    "                                                    bb_conf_after_hsv_corr,\n",
    "                                                    CUT_FACTOR_FOR_HIST_CORR,\n",
    "                                                    #TRES_MINIMUM_HISTOGRAM_CORR=TRES_MINIMUM_HISTOGRAM_CORR,\n",
    "                                                    TRES_MINIMUM_HISTOGRAM_CORR = tres_min_hsv_corr[i],\n",
    "                                                    shelf=True)\n",
    "            \n",
    "            '''\n",
    "            # using HSV histogram correlation between each of the NxM bins\n",
    "            final_corners_of_bounding_boxes_after_color_problem = \\\n",
    "                solve_color_problem_with_N_x_M_bins_and_histogram_HSV_correlation(\n",
    "                                    final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                    shelf_features, \n",
    "                                    TRES_MINIMUM_HISTOGRAM_CORR,\n",
    "                                    i, j, N=N_BINS_ON_WIDTH, M=M_BINS_ON_HEIGHT, \n",
    "                                    MIN_NUM_OF_GOOD_CELLS = MIN_NUM_OF_GOOD_CELLS,\n",
    "                                    shelf=True)\n",
    "            '''\n",
    "            \n",
    "            # merge overlapping bounding boxes\n",
    "            final_corners_of_bounding_boxes = merge_overlapping_bounding_boxes(\n",
    "                final_corners_of_bounding_boxes_after_color_problem,\n",
    "                bb_conf_after_hsv_corr, \n",
    "                final_bb_conf,\n",
    "                i, j,\n",
    "                shelf_index,\n",
    "                DISTANCE_BOUNDING_BOXES_TRESHOLD)\n",
    "        \n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "            '''\n",
    "            print('Model {}'.format(i))\n",
    "            #plotting the model image\n",
    "            plt.imshow(cv2.cvtColor(model_img, cv2.COLOR_BGR2RGB))\n",
    "            plt.show()\n",
    "\n",
    "\n",
    "\n",
    "            print('  - {} Keypoints in scene {}'.format(len(dst_pts), j))\n",
    "            # plotting keypoints in the scene image\n",
    "            plt.imshow(cv2.cvtColor(shelf_img_with_keypoints, cv2.COLOR_BGR2RGB))\n",
    "            plt.show()\n",
    "\n",
    "            print('  - {} estimated centers in scene {}'.format(len(G_scene), j))\n",
    "            # plotting keypoints in the scene image\n",
    "            plt.imshow(cv2.cvtColor(shelf_img_with_estimated_centers, cv2.COLOR_BGR2RGB))\n",
    "            plt.show()\n",
    "\n",
    "\n",
    "\n",
    "            print('  - Found {} highligthed cells in scene {}'.format(len(highlighted_cells_of_current_model_in_current_shelf), j))\n",
    "            #plotting the scene image\n",
    "            plt.imshow(cv2.cvtColor(shelf_img, cv2.COLOR_BGR2RGB))\n",
    "            plt.show()\n",
    "            \n",
    "            shelf_img_sctioned = np.copy(shelf_img)\n",
    "            plot_Img_sectioned(shelf_img_sctioned, ACC_ARRAY_CELL_DIMENSION_1, ACC_ARRAY_CELL_DIMENSION_2)\n",
    "            print(ACC[ACC != 0])\n",
    "            \n",
    "            # if I don't find any cell that has the min num of votes, then I don't print the bounding box\n",
    "            if(len(highlighted_cells_of_current_model_in_current_shelf) > 0):\n",
    "                print('  - Found {} instances of model {} in scene {}'.format(len(highlighted_cells_of_current_model_in_current_shelf), i, j))\n",
    "                # plotting the bounding box\n",
    "                plt.imshow(cv2.cvtColor(shelf_img_with_bounding_boxes, cv2.COLOR_BGR2RGB))\n",
    "                plt.show()\n",
    "            else:\n",
    "                print('  - Model {} NOT found in the scene {}'.format(i,j))\n",
    "\n",
    "            \n",
    "            # plot image with final bounding boxes (with bounding boxes that do not overlap each other)\n",
    "            plot_final_bounding_boxes(None, final_corners_of_bounding_boxes_without_exceeding_dimensions, \n",
    "                                      shelf_features, j, thickness=5, shelf=True)\n",
    "            '''\n",
    "            print('AFTER COLOR CORRECTION:')\n",
    "            # plot image with final bounding boxes, after color problem\n",
    "            h_img_bb = plot_final_bounding_boxes(final_shelf_images_with_bb[j][shelf_index], \n",
    "                                                 final_corners_of_bounding_boxes, \n",
    "                                                 shelf_features, j, thickness=5, shelf=True)\n",
    "\n",
    "            if h_img_bb is not None:\n",
    "                final_shelf_images_with_bb[j][shelf_index] = h_img_bb\n",
    "\n",
    "            h_results[j][shelf_index][i]['count'] += len(final_corners_of_bounding_boxes)\n",
    "            \n",
    "            for index_1, [top_left_corner, bottom_right_corner] in enumerate(final_corners_of_bounding_boxes):\n",
    "            \n",
    "                top_right_corner = (bottom_right_corner[0], top_left_corner[1])\n",
    "\n",
    "                if index_1 == 0:\n",
    "                    h_results[j][shelf_index][i]['width'] = []\n",
    "                    h_results[j][shelf_index][i]['height'] = []\n",
    "                    h_results[j][shelf_index][i]['pos'] = []\n",
    "\n",
    "                # compute width and height of the bounding box\n",
    "                width_of_bounding_box = distance_2_points(top_left_corner, top_right_corner)\n",
    "                height_of_bounding_box = distance_2_points(top_right_corner, bottom_right_corner)\n",
    "\n",
    "\n",
    "                # Save width and height measures of the bounding box\n",
    "                h_results[j][shelf_index][i]['width'].append(width_of_bounding_box)\n",
    "                h_results[j][shelf_index][i]['height'].append(height_of_bounding_box)\n",
    "\n",
    "                # Save the position of the bounding box\n",
    "                h_results[j][shelf_index][i]['pos'].append(\n",
    "                    (int(top_left_corner[0]), int(top_left_corner[1])) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_result(e_results, idx_of_scene, idx_shelf):\n",
    "    for i in models_to_test:\n",
    "        print('Product {} - {} instance/s found:'.format(i, e_results[idx_of_scene][idx_shelf][i]['count']))\n",
    "        n = 0\n",
    "        if e_results[idx_of_scene][idx_shelf][i].get('width', None):\n",
    "            found_instances = e_results[idx_of_scene][idx_shelf][i].get('width')\n",
    "            for index in range(len(found_instances)):\n",
    "                n += 1\n",
    "                print('\\tInstance {} position: {}, width: {}px, height: {}px'.format(n, \n",
    "                                                            e_results[idx_of_scene][idx_shelf][i]['pos'][index], \n",
    "                                                            e_results[idx_of_scene][idx_shelf][i]['width'][index], \n",
    "                                                            e_results[idx_of_scene][idx_shelf][i]['height'][index]))\n",
    "                for k, v in final_bb_conf.items():\n",
    "                    if e_results[idx_of_scene][idx_shelf][i]['pos'][index] == k[0]:\n",
    "                        print('\\t\\tFINAL CORR: {}'.format(final_bb_conf[k][6]))\n",
    "    print('_' * 80 + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "print(len(bounding_box_confidence.keys()))\n",
    "print(bounding_box_confidence)\n",
    "\n",
    "\n",
    "print('\\n\\nDopo exceeding dimensions\\n \\n')\n",
    "print(len(bb_conf_after_exceeding_dim.keys()))\n",
    "print(bb_conf_after_exceeding_dim)\n",
    "print('\\n')\n",
    "'''\n",
    "\n",
    "#print('Ultimo step:\\n')\n",
    "#print(len(final_bb_conf))\n",
    "#print(final_bb_conf)\n",
    "\n",
    "for k, v in final_bb_conf.items():\n",
    "    #print(k[0])\n",
    "    final_conf_value = 0.7 * v[0] + 5 * v[1] + 0.8 *v[2] \n",
    "    final_bb_conf[k].append(final_conf_value)\n",
    "    final_bb_conf[k].append(final_conf_value)\n",
    "    final_bb_conf[k].append(final_conf_value)\n",
    "    final_bb_conf[k].append(final_conf_value)\n",
    "    final_bb_conf[k].append(final_conf_value)\n",
    "    \n",
    "#print('\\n')\n",
    "#print(final_bb_conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in scenes_to_test:\n",
    "    print( ('_' * 40) + 'scene {}'.format(j) + ('_' * 40))\n",
    "    for shelf_index, shelf_features in scenes_shelves_features[j].items():\n",
    "        img = final_shelf_images_with_bb[j].get(shelf_index, None)\n",
    "        if img is not None:\n",
    "            print('shelf {}'.format(shelf_index))\n",
    "            plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n",
    "            plt.show()\n",
    "            print_result(h_results, j, shelf_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#final_bb_conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_result(e_results, idx_of_scene): #, confidence_value):\n",
    "    for i in models_to_test:\n",
    "        print('Product {} - {} instance/s found:'.format(i, e_results[idx_of_scene][i]['count']))\n",
    "        n = 0\n",
    "        if e_results[idx_of_scene][i].get('width', None):\n",
    "            found_instances = e_results[idx_of_scene][i].get('width')\n",
    "            for index in range(len(found_instances)):\n",
    "                n += 1\n",
    "                print('\\tInstance {} position: {}, width: {}px, height: {}px'.format(n, \n",
    "                                                            e_results[idx_of_scene][i]['pos'][index], \n",
    "                                                            e_results[idx_of_scene][i]['width'][index], \n",
    "                                                            e_results[idx_of_scene][i]['height'][index]))\n",
    "                \n",
    "                print('\\t\\tWeigthed coeff: num_votes\\tcolor\\thsv_corr:\\n\\t\\t\\t\\t{}\\t\\t{}\\t{}'.format(\n",
    "                    e_results[idx_of_scene][i]['conf_values'][index][0], \n",
    "                    e_results[idx_of_scene][i]['conf_values'][index][1], \n",
    "                    e_results[idx_of_scene][i]['conf_values'][index][2]))\n",
    "                \n",
    "                print('\\t\\tFINAL CONFIDENCE: {}'.format(e_results[idx_of_scene][i]['final_conf'][index]))\n",
    "                print('\\t' + ('-' * 80))\n",
    "    print('_' * 80 + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_scene_images = {}\n",
    "\n",
    "h_final_results = {}\n",
    "\n",
    "TRES_CONF = 4.97#5.12#6.0#6.5 #8.5 #6.87\n",
    "\n",
    "models_to_test = list(range(0,24))\n",
    "\n",
    "final_bb_conf_after_confidence_tresholding = {}\n",
    "\n",
    "for j in scenes_to_test:\n",
    "    h_final_results[j] = {}\n",
    "    final_scene_images[j] = np.copy(h_scenes_images_features[j][IMAGE_INDEX])\n",
    "    for i in models_to_test:\n",
    "        # initialize lists that will contain final output info\n",
    "        \n",
    "        h_final_results[j][i] = {}\n",
    "        h_final_results[j][i]['count'] = 0\n",
    "        h_final_results[j][i]['width'] = []\n",
    "        h_final_results[j][i]['height'] = []\n",
    "        h_final_results[j][i]['pos'] = []\n",
    "        h_final_results[j][i]['conf_values'] = []\n",
    "        h_final_results[j][i]['final_conf'] = []\n",
    "\n",
    "        \n",
    "for k, v in final_bb_conf.items():\n",
    "    # print(k)\n",
    "    # corner coord in shelf\n",
    "    top_left_corner_in_shelf = k[0]\n",
    "    bottom_right_corner_in_shelf = k[1]\n",
    "    model_index = v[3][0]\n",
    "    scene_index = v[3][1]\n",
    "    shelf_index = v[3][2]\n",
    "    \n",
    "    num_votes_weight = v[0]\n",
    "    color_weight = v[1]\n",
    "    hsv_correlation_weigth = v[2]\n",
    "    \n",
    "    confidence_value = v[8]\n",
    "    \n",
    "    \n",
    "    #print(shelf_index)\n",
    "    \n",
    "    \n",
    "    # shift top left and bottom right corners of the bounding box as the height of their shelf\n",
    "    # these are the corners coord in scene\n",
    "    top_left_corner_in_scene = (top_left_corner_in_shelf[0], \n",
    "                                top_left_corner_in_shelf[1] + \n",
    "                                height_coord_of_shelves_in_all_scenes[scene_index][shelf_index])\n",
    "    bottom_right_corner_in_scene = (bottom_right_corner_in_shelf[0], \n",
    "                                    bottom_right_corner_in_shelf[1] + \n",
    "                                    height_coord_of_shelves_in_all_scenes[scene_index][shelf_index])\n",
    "    #print(confidence_value)\n",
    "    \n",
    "    # print(confidence_value)\n",
    "    # consider a bounding box correct only if its confidence is greater than the treshold\n",
    "    if confidence_value >= TRES_CONF:\n",
    "        final_bb_conf_after_confidence_tresholding[k] = v\n",
    "        \n",
    "        final_scene_images[scene_index] = cv2.rectangle(final_scene_images[scene_index], \n",
    "                                                        top_left_corner_in_scene, bottom_right_corner_in_scene, (0,255,0), 10)\n",
    "        \n",
    "        cv2.putText(final_scene_images[scene_index], str(model_index), (top_left_corner_in_scene[0], \n",
    "                                                                        top_left_corner_in_scene[1]+20),\n",
    "                                                                        cv2.FONT_HERSHEY_SIMPLEX, \n",
    "                                                                        0.9, (0,0,0), 3)\n",
    "\n",
    "        # increment the counter info\n",
    "        #print('scene_index {}'.format(scene_index))\n",
    "        #print('model_index {}'.format(model_index))\n",
    "        #print(h_final_results[scene_index][model_index]['count'])\n",
    "        h_final_results[scene_index][model_index]['count'] += 1\n",
    "\n",
    "        # top right corner of the bounding box in scene\n",
    "        top_right_corner_in_scene = (bottom_right_corner_in_scene[0], top_left_corner_in_scene[1])\n",
    "\n",
    "        # compute width and height of the bounding box\n",
    "        width_of_bounding_box = distance_2_points(top_left_corner_in_scene, top_right_corner_in_scene)\n",
    "        height_of_bounding_box = distance_2_points(top_right_corner_in_scene, bottom_right_corner_in_scene)\n",
    "\n",
    "\n",
    "        # Save width and height measures of the bounding box\n",
    "        h_final_results[scene_index][model_index]['width'].append(width_of_bounding_box)\n",
    "        h_final_results[scene_index][model_index]['height'].append(height_of_bounding_box)\n",
    "\n",
    "        # Save the position of the bounding box\n",
    "        h_final_results[scene_index][model_index]['pos'].append(\n",
    "            (int(top_left_corner_in_scene[0]), int(top_left_corner_in_scene[1])) )\n",
    "\n",
    "        # Save the three weigthed values for the final confidence of the bounding box        \n",
    "        h_final_results[scene_index][model_index]['conf_values'].append([num_votes_weight, color_weight, hsv_correlation_weigth])\n",
    "\n",
    "        # Save the final confidence\n",
    "        h_final_results[scene_index][model_index]['final_conf'].append(confidence_value)\n",
    "\n",
    "\n",
    "        \n",
    "for j in scenes_to_test:\n",
    "    plt.imshow(cv2.cvtColor(final_scene_images[j], cv2.COLOR_BGR2RGB))\n",
    "    plt.show()\n",
    "    print_result(h_final_results, j)\n",
    "    \n",
    "\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of the excluded bounding boxes due to a less confidence value wrt to another overlapped bounding box\n",
    "excluded_bounding_boxes = {}\n",
    "\n",
    "\n",
    "# if the top left corners of the bounding boxes have a distance <= DISTANCE_2_OVERLAPPED_BOUNDING_BOXES\n",
    "# then they are overlapped and I will take the one with the higher confidence\n",
    "DISTANCE_2_OVERLAPPED_BOUNDING_BOXES = 23\n",
    "\n",
    "for k_1, v_1 in final_bb_conf_after_confidence_tresholding.items():\n",
    "    #print('c')\n",
    "    # corner coord in shelf\n",
    "    top_left_corner_in_shelf_1 = k_1[0]\n",
    "    bottom_right_corner_in_shelf_1 = k_1[1]\n",
    "    model_index_1 = v_1[3][0]\n",
    "    scene_index_1 = v_1[3][1]\n",
    "    shelf_index_1 = v_1[3][2]\n",
    "    \n",
    "    num_votes_weight_1 = v_1[0]\n",
    "    color_weight_1 = v_1[1]\n",
    "    hsv_correlation_weigth_1 = v_1[2]\n",
    "    \n",
    "    confidence_value_1 = v_1[6]    \n",
    "    \n",
    "    # shift top left and bottom right corners of the bounding box as the height of their shelf\n",
    "    # these are the corners coord in scene\n",
    "    top_left_corner_in_scene_1 = (top_left_corner_in_shelf_1[0], \n",
    "                                top_left_corner_in_shelf_1[1] + \n",
    "                                height_coord_of_shelves_in_all_scenes[scene_index_1][shelf_index_1])\n",
    "    bottom_right_corner_in_scene_1 = (bottom_right_corner_in_shelf_1[0], \n",
    "                                    bottom_right_corner_in_shelf_1[1] + \n",
    "                                    height_coord_of_shelves_in_all_scenes[scene_index_1][shelf_index_1])\n",
    "    \n",
    "    for k_2, v_2 in final_bb_conf_after_confidence_tresholding.items():\n",
    "        #print('a')\n",
    "        # If I am comparing the same bounding box, then I break\n",
    "        if k_1 == k_2:\n",
    "            break\n",
    "        # If I already excluded the current bounding box, then I don't consider it anymore\n",
    "        #if k_2 in excluded_bounding_boxes:\n",
    "        #    break\n",
    "        #print('b')\n",
    " \n",
    "        # corner coord in shelf\n",
    "        top_left_corner_in_shelf_2 = k_2[0]\n",
    "        bottom_right_corner_in_shelf_2 = k_2[1]\n",
    "        model_index_2 = v_2[3][0]\n",
    "        scene_index_2 = v_2[3][1]\n",
    "        shelf_index_2 = v_2[3][2]\n",
    "\n",
    "        num_votes_weight_2 = v_2[0]\n",
    "        color_weight_2 = v_2[1]\n",
    "        hsv_correlation_weigth_2 = v_2[2]\n",
    "\n",
    "        confidence_value_2 = v_2[6]    \n",
    "\n",
    "        # shift top left and bottom right corners of the bounding box as the height of their shelf\n",
    "        # these are the corners coord in scene\n",
    "        top_left_corner_in_scene_2 = (top_left_corner_in_shelf_2[0], \n",
    "                                    top_left_corner_in_shelf_2[1] + \n",
    "                                    height_coord_of_shelves_in_all_scenes[scene_index_2][shelf_index_2])\n",
    "        bottom_right_corner_in_scene_2 = (bottom_right_corner_in_shelf_2[0], \n",
    "                                        bottom_right_corner_in_shelf_2[1] + \n",
    "                                        height_coord_of_shelves_in_all_scenes[scene_index_2][shelf_index_2])\n",
    "\n",
    "        \n",
    "        #print(top_left_corner_in_scene_1)\n",
    "        # if the top left corners of the bounding boxes have a distance <= DISTANCE_2_OVERLAPPED_BOUNDING_BOXES\n",
    "        # then they are overlapped and I will discard the one with the lower confidence\n",
    "        if distance_2_points(top_left_corner_in_scene_1, top_left_corner_in_scene_2) <= DISTANCE_2_OVERLAPPED_BOUNDING_BOXES:\n",
    "            # I manage 2 overlapping bounding boxes only if they are on the same scene and shelf\n",
    "            if scene_index_1 == scene_index_2 and shelf_index_1 == shelf_index_2:\n",
    "                print(top_left_corner_in_scene_1, top_left_corner_in_scene_2, scene_index_1, shelf_index_2)\n",
    "\n",
    "                if confidence_value_1 <= confidence_value_2:\n",
    "                    excluded_bounding_boxes[k_1] = 1\n",
    "                else:\n",
    "                    excluded_bounding_boxes[k_2] = 1\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(excluded_bounding_boxes)\n",
    "print()\n",
    "print(len(excluded_bounding_boxes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_bb_conf_after_overlapped_bounding_boxes = {}\n",
    "\n",
    "final_scene_images = {}\n",
    "\n",
    "h_final_results = {}\n",
    "\n",
    "models_to_test = list(range(0,24))\n",
    "\n",
    "\n",
    "for k,v in final_bb_conf_after_confidence_tresholding.items():\n",
    "    if k not in excluded_bounding_boxes:\n",
    "        final_bb_conf_after_overlapped_bounding_boxes[k] = v\n",
    "        \n",
    "        \n",
    "\n",
    "for j in scenes_to_test:\n",
    "    h_final_results[j] = {}\n",
    "    final_scene_images[j] = np.copy(h_scenes_images_features[j][IMAGE_INDEX])\n",
    "    for i in models_to_test:\n",
    "        # initialize lists that will contain final output info\n",
    "        \n",
    "        h_final_results[j][i] = {}\n",
    "        h_final_results[j][i]['count'] = 0\n",
    "        h_final_results[j][i]['width'] = []\n",
    "        h_final_results[j][i]['height'] = []\n",
    "        h_final_results[j][i]['pos'] = []\n",
    "        h_final_results[j][i]['conf_values'] = []\n",
    "        h_final_results[j][i]['final_conf'] = []\n",
    "\n",
    "        \n",
    "for k, v in final_bb_conf_after_overlapped_bounding_boxes.items():\n",
    "    # corner coord in shelf\n",
    "    top_left_corner_in_shelf = k[0]\n",
    "    bottom_right_corner_in_shelf = k[1]\n",
    "    model_index = v[3][0]\n",
    "    scene_index = v[3][1]\n",
    "    shelf_index = v[3][2]\n",
    "    \n",
    "    num_votes_weight = v[0]\n",
    "    color_weight = v[1]\n",
    "    hsv_correlation_weigth = v[2]\n",
    "    \n",
    "    confidence_value = v[6]\n",
    "    \n",
    "    \n",
    "    #print(shelf_index)\n",
    "    \n",
    "    \n",
    "    # shift top left and bottom right corners of the bounding box as the height of their shelf\n",
    "    # these are the corners coord in scene\n",
    "    top_left_corner_in_scene = (top_left_corner_in_shelf[0], \n",
    "                                top_left_corner_in_shelf[1] + \n",
    "                                height_coord_of_shelves_in_all_scenes[scene_index][shelf_index])\n",
    "    bottom_right_corner_in_scene = (bottom_right_corner_in_shelf[0], \n",
    "                                    bottom_right_corner_in_shelf[1] + \n",
    "                                    height_coord_of_shelves_in_all_scenes[scene_index][shelf_index])\n",
    "\n",
    "    final_scene_images[scene_index] = cv2.rectangle(final_scene_images[scene_index], \n",
    "                                                    top_left_corner_in_scene, bottom_right_corner_in_scene, (0,255,0), 10)\n",
    "\n",
    "    cv2.putText(final_scene_images[scene_index], str(model_index), (top_left_corner_in_scene[0], \n",
    "                                                                    top_left_corner_in_scene[1]+20),\n",
    "                                                                    cv2.FONT_HERSHEY_SIMPLEX, \n",
    "                                                                    0.9, (0,0,0), 3)\n",
    "\n",
    "    # increment the counter info\n",
    "    h_final_results[scene_index][model_index]['count'] += 1\n",
    "\n",
    "    # top right corner of the bounding box in scene\n",
    "    top_right_corner_in_scene = (bottom_right_corner_in_scene[0], top_left_corner_in_scene[1])\n",
    "\n",
    "    # compute width and height of the bounding box\n",
    "    width_of_bounding_box = distance_2_points(top_left_corner_in_scene, top_right_corner_in_scene)\n",
    "    height_of_bounding_box = distance_2_points(top_right_corner_in_scene, bottom_right_corner_in_scene)\n",
    "\n",
    "\n",
    "    # Save width and height measures of the bounding box\n",
    "    h_final_results[scene_index][model_index]['width'].append(width_of_bounding_box)\n",
    "    h_final_results[scene_index][model_index]['height'].append(height_of_bounding_box)\n",
    "\n",
    "    # Save the position of the bounding box\n",
    "    h_final_results[scene_index][model_index]['pos'].append(\n",
    "        (int(top_left_corner_in_scene[0]), int(top_left_corner_in_scene[1])) )\n",
    "\n",
    "    # Save the three weigthed values for the final confidence of the bounding box        \n",
    "    h_final_results[scene_index][model_index]['conf_values'].append([num_votes_weight, color_weight, hsv_correlation_weigth])\n",
    "\n",
    "    # Save the final confidence\n",
    "    h_final_results[scene_index][model_index]['final_conf'].append(confidence_value)\n",
    "\n",
    "\n",
    "        \n",
    "for j in scenes_to_test:\n",
    "    plt.imshow(cv2.cvtColor(final_scene_images[j], cv2.COLOR_BGR2RGB))\n",
    "    plt.show()\n",
    "    print_result(h_final_results, j)\n",
    "    \n",
    "\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}