Final_pipline.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Final_pipline.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyN2MGy84kEBPpQcx66ofGdl"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","metadata":{"id":"1aw_oLycfyPO","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1632032993572,"user_tz":-330,"elapsed":419,"user":{"displayName":"Vishal Vanpariya","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjI-opVAF50hU4bs6Osmpwzh7ACFjjBuxgYjhKh7w=s64","userId":"03081609474255309688"}},"outputId":"9d4a5768-8fba-4f11-d528-5ab69dca9fc0"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}]},{"cell_type":"code","metadata":{"id":"PX0uiGTExZyh","executionInfo":{"status":"ok","timestamp":1632032996346,"user_tz":-330,"elapsed":2355,"user":{"displayName":"Vishal Vanpariya","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjI-opVAF50hU4bs6Osmpwzh7ACFjjBuxgYjhKh7w=s64","userId":"03081609474255309688"}}},"source":["## Importing library\n","import tensorflow as tf\n","from google.colab import files\n","import numpy as np\n","from io import BytesIO\n","from PIL import Image\n","import matplotlib.pyplot as plt\n","import cv2"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"HiyBNb4jvilK","executionInfo":{"status":"ok","timestamp":1632035029866,"user_tz":-330,"elapsed":430,"user":{"displayName":"Vishal Vanpariya","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjI-opVAF50hU4bs6Osmpwzh7ACFjjBuxgYjhKh7w=s64","userId":"03081609474255309688"}}},"source":["## Final model Class define\n","class Final_Model:\n","\n","    def __init__(self):\n","        \"\"\"\n","        Final_Model constructer\n","        \"\"\"\n","        self.model = self.get_model()\n","        self.model.load_weights(\"/content/drive/MyDrive/Colab Notebooks/AAIC/2nd case study/Models/RESNET50.h5\")\n","    \n","    def predict(self,image):\n","        \"\"\"\n","        This function return predicted value\n","\n","        Input:\n","            image: 320x320x3 size array\n","        Output:\n","            Pred: predicted dimensions (x_center,y_center,box oriantation,box height,box width) \n","            Edges: 4x2 array, contains points of corner\n","        \"\"\"\n","\n","        height, width = image.shape[:2]\n","        tt,ttt =  1,1\n","        T = np.float32([[1, 0, 50 * tt], [0, 1, 50 * ttt]])\n","        image = cv2.warpAffine(image, T, (width, height))\n","\n","        pred = self.model.predict(np.array([image]))[0]\n","        edges = self.grasp_to_bbox_pred(pred[0], pred[1],pred[2],pred[3],pred[4]) - np.array([50,50])\n","\n","        return pred, edges\n","\n","    def grasp_to_bbox_pred(self,x, y, angle, h, w):\n","        \"\"\"\n","        This function return corner points using given dimensions (x_center,y_center,box oriantation,box height,box width) \n","\n","        Input:\n","            x: x cordinate of box center\n","            y: y cordinate of box center\n","            angle: Oriantation of box respect to positive x-axis\n","            h: height of box.\n","            w: width of box\n","        Output:\n","            Edges: 4x2 array, contains points of corner\n","        \"\"\"\n","        edge1 = (x - w/2, y + h/2)\n","        edge2 = (x + w/2, y + h/2)\n","        edge3 = (x + w/2, y - h/2)\n","        edge4 = (x - w/2, y - h/2)\n","        a = np.array([edge1, edge2, edge3, edge4])\n","        a = a - (x,y)\n","        angle = angle * (3.14/180)\n","        fun = lambda x: (x[:,0] * np.cos(angle) - x[:,1] * np.sin(angle), x[:,0] * np.sin(angle) + x[:,1] * np.cos(angle))\n","        a = fun(a)\n","        a = np.array(a).T\n","        a = a + (x,y)\n","        return a\n","\n","    def plot_predict(self,image):\n","        \"\"\"\n","        This function return predicted values and plot predicted values on image\n","\n","        Input:\n","            image: 320x320x3 size array\n","        Output:\n","            Pred: predicted dimensions (x_center,y_center,box oriantation,box height,box width) \n","            Edges: 4x2 array, contains points of corner\n","        \"\"\"\n","        plot_im = image\n","\n","        height, width = image.shape[:2]\n","        tt,ttt =  1,1\n","        T = np.float32([[1, 0, 50 * tt], [0, 1, 50 * ttt]])\n","        image = cv2.warpAffine(image, T, (width, height))\n","\n","        pred = self.model.predict(np.array([image]))[0]\n","        edges = self.grasp_to_bbox_pred(pred[0], pred[1],pred[2],pred[3],pred[4])\n","\n","        x = edges[:,0].reshape(1,-1)[0]\n","        y = edges[:,1].reshape(1,-1)[0]\n","        \n","        x = np.concatenate([x,x[:1]]) - 50\n","        y = np.concatenate([y,y[:1]]) - 50\n","        \n","\n","        fig, ax = plt.subplots(figsize = (8,8))\n","        ax.imshow(plot_im)\n","        ax.plot(x,y, 'ro-',color = 'green')\n","\n","        return pred,edges\n","\n","\n","    def get_model(self,lr = 0.001):\n","\n","        \"\"\"\n","        This function load trained model and return it.\n","\n","        Input: \n","            lr: learning rate but its not require here.\n","        Output:\n","            model: return trained model, type: tf.keras.Model\n","        \"\"\"\n","\n","        def resenet_load():\n","            RESNET50 = tf.keras.applications.resnet50.ResNet50(include_top=False, weights='imagenet',input_shape=(320,320,3), classes=2048)\n","            for i in range(len(RESNET50.layers)):\n","                RESNET50.layers[i].trainable = False\n","            return RESNET50\n","\n","        RESNET50 = resenet_load()\n","\n","        input = tf.keras.layers.Input((320,320,3,))\n","        resnet = RESNET50(input)\n","        x = tf.keras.layers.Flatten()(resnet)\n","        x = tf.keras.layers.Dense(1024,activation='relu')(x)\n","        x = tf.keras.layers.Dropout(0.2)(x)\n","        output = tf.keras.layers.Dense(5)(x)\n","        model = tf.keras.models.Model(input,output)\n","    \n","        model.compile(tf.keras.optimizers.Adam(lr), loss = 'mse')\n","        return model\n"],"execution_count":16,"outputs":[]},{"cell_type":"code","metadata":{"id":"_PqqBCfyj3Be","executionInfo":{"status":"ok","timestamp":1632035038079,"user_tz":-330,"elapsed":6248,"user":{"displayName":"Vishal Vanpariya","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjI-opVAF50hU4bs6Osmpwzh7ACFjjBuxgYjhKh7w=s64","userId":"03081609474255309688"}}},"source":["model = Final_Model()"],"execution_count":17,"outputs":[]},{"cell_type":"code","metadata":{"id":"45rZvX9hxerI","executionInfo":{"status":"ok","timestamp":1632038288897,"user_tz":-330,"elapsed":474,"user":{"displayName":"Vishal Vanpariya","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjI-opVAF50hU4bs6Osmpwzh7ACFjjBuxgYjhKh7w=s64","userId":"03081609474255309688"}}},"source":["# uploaded = files.upload()\n","\n","# for filename in uploaded.keys():\n","#     im = np.array(Image.open(BytesIO(uploaded[filename])))[:,:,:3]\n","#     (x, y, angle, h, w),edges = model.plot_predict(im)"],"execution_count":33,"outputs":[]},{"cell_type":"code","metadata":{"id":"twdc3v391tme","executionInfo":{"status":"ok","timestamp":1632034806105,"user_tz":-330,"elapsed":559,"user":{"displayName":"Vishal Vanpariya","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjI-opVAF50hU4bs6Osmpwzh7ACFjjBuxgYjhKh7w=s64","userId":"03081609474255309688"}}},"source":["import glob\n","images = glob.glob(\"/content/*.png\")"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"output_embedded_package_id":"1C91LoJ2xdjrwNWUYoGwkZ8N6cH0DDv6n"},"id":"CoBox9zgkDUE","executionInfo":{"status":"ok","timestamp":1632037851828,"user_tz":-330,"elapsed":98873,"user":{"displayName":"Vishal Vanpariya","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjI-opVAF50hU4bs6Osmpwzh7ACFjjBuxgYjhKh7w=s64","userId":"03081609474255309688"}},"outputId":"77e93abd-257e-4af2-b4b1-5f79af078166"},"source":["## Precting random 140 images\n","fig,ax = plt.subplots(20,7,figsize = (20,60))\n","k = 0\n","for i in range(20):\n","    for j in range(7):\n","\n","        im = cv2.imread(images[k])\n","        (x, y, angle, h, w),edges = model.predict(im)\n","        x = edges[:,0].reshape(1,-1)[0]\n","        y = edges[:,1].reshape(1,-1)[0]\n","        \n","        x = np.concatenate([x,x[:1]])\n","        y = np.concatenate([y,y[:1]])\n","        \n","        ax[i,j].imshow(im)\n","        ax[i,j].plot(x,y, 'ro-',color = 'green')\n","\n","        ax[i,j].set_xticklabels([])\n","        ax[i,j].set_yticklabels([])\n","        ax[i,j].set_aspect('equal')\n","\n","        k+=1\n","fig.subplots_adjust(wspace=0, hspace=0)"],"execution_count":31,"outputs":[{"output_type":"display_data","data":{"text/plain":"Output hidden; open in https://colab.research.google.com to view."},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"blZzaXkcDOvE"},"source":["## Refrences:\n","* https://www.researchgate.net/publication/310953119\n","* https://www.tensorflow.org/api_docs/python/tf/keras/applications/resnet\n","* https://github.com/tnikolla/robot-grasp-detection\n","* https://www.appliedaicourse.com/"]}]}