anomalies_detect.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:00:13.813919Z",
     "start_time": "2020-04-16T17:00:12.564311Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import deepmatcher as dm\n",
    "from explanation.deepmatcher_utils import wrapDm\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:00:13.836289Z",
     "start_time": "2020-04-16T17:00:13.815829Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "463"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv('datasets/DBLP-ACM/test.csv')\n",
    "testneg = test_df[test_df.label==0]\n",
    "len(testneg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T16:17:52.694259Z",
     "start_time": "2020-04-16T16:17:47.979224Z"
    }
   },
   "outputs": [],
   "source": [
    "dblpacm_model = dm.MatchingModel(attr_summarizer='rnn')\n",
    "dblpacm_model.load_state('models/dblp_acm_rnn.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:00:37.945243Z",
     "start_time": "2020-04-16T17:00:37.939736Z"
    }
   },
   "outputs": [],
   "source": [
    "f = lambda df,model,label:np.count_nonzero(np.argmax(wrapDm(df,model),axis=1)==label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T16:32:43.931691Z",
     "start_time": "2020-04-16T16:32:41.726588Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "460"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f(testneg,dblpacm_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T16:33:50.681440Z",
     "start_time": "2020-04-16T16:33:48.390688Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "392"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg_cp = testneg.copy()\n",
    "testneg_cp['ltable_title'] = testneg_cp['rtable_title']\n",
    "f(testneg_cp,dblpacm_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T16:34:03.739741Z",
     "start_time": "2020-04-16T16:34:01.599925Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "445"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg_cp = testneg.copy()\n",
    "testneg_cp['ltable_authors'] = testneg_cp['rtable_authors']\n",
    "f(testneg_cp,dblpacm_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T16:37:56.934341Z",
     "start_time": "2020-04-16T16:37:54.817176Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "333"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg_cp = testneg.copy()\n",
    "testneg_cp['ltable_title'] = testneg_cp['rtable_title']\n",
    "testneg_cp['ltable_authors'] = testneg_cp['rtable_authors']\n",
    "f(testneg_cp,dblpacm_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T16:40:46.710719Z",
     "start_time": "2020-04-16T16:40:44.514369Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "129"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg_cp = testneg.copy()\n",
    "testneg_cp['ltable_year'] = ['1998']*len(testneg_cp)\n",
    "testneg_cp['ltable_title'] = testneg_cp['rtable_title']\n",
    "testneg_cp['rtable_year'] = testneg_cp['ltable_year']\n",
    "f(testneg_cp,dblpacm_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T16:39:13.706513Z",
     "start_time": "2020-04-16T16:39:13.665747Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>ltable_title</th>\n",
       "      <th>ltable_authors</th>\n",
       "      <th>ltable_venue</th>\n",
       "      <th>ltable_year</th>\n",
       "      <th>rtable_title</th>\n",
       "      <th>rtable_authors</th>\n",
       "      <th>rtable_venue</th>\n",
       "      <th>rtable_year</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Database Principles Column - Introduction</td>\n",
       "      <td>Leonid Libkin</td>\n",
       "      <td>SIGMOD Record</td>\n",
       "      <td>1999</td>\n",
       "      <td>Lifestreams: a storage model for personal data</td>\n",
       "      <td>Eric Freeman, David Gelernter</td>\n",
       "      <td>ACM SIGMOD Record</td>\n",
       "      <td>1996</td>\n",
       "      <td>journals/sigmod/Libkin99a#381893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>The PointCast Network</td>\n",
       "      <td>Vibha Dayal, Satish Ramakrishnan</td>\n",
       "      <td>SIGMOD Conference</td>\n",
       "      <td>1998</td>\n",
       "      <td>Repositories and object oriented databases</td>\n",
       "      <td>Philip A. Bernstein</td>\n",
       "      <td>ACM SIGMOD Record</td>\n",
       "      <td>1998</td>\n",
       "      <td>conf/sigmod/RamakrishnanD98#273271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Chair's Message</td>\n",
       "      <td>Richard T. Snodgrass</td>\n",
       "      <td>SIGMOD Record</td>\n",
       "      <td>1998</td>\n",
       "      <td>Repositories and object oriented databases</td>\n",
       "      <td>Philip A. Bernstein</td>\n",
       "      <td>ACM SIGMOD Record</td>\n",
       "      <td>1998</td>\n",
       "      <td>journals/sigmod/Snodgrass98#273271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>Finding Generalized Projected Clusters In High...</td>\n",
       "      <td>Charu C. Aggarwal, Philip S. Yu</td>\n",
       "      <td>SIGMOD Conference</td>\n",
       "      <td>2000</td>\n",
       "      <td>DataBlitz storage manager: main-memory databas...</td>\n",
       "      <td>J. Baulier, P. Bohannon, S. Gogate, C. Gupta, ...</td>\n",
       "      <td>International Conference on Management of Data</td>\n",
       "      <td>1999</td>\n",
       "      <td>conf/sigmod/AggarwalY00#304239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>Ripple Joins for Online Aggregation</td>\n",
       "      <td>Peter J. Haas, Joseph M. Hellerstein</td>\n",
       "      <td>SIGMOD Conference</td>\n",
       "      <td>1999</td>\n",
       "      <td>DataBlitz storage manager: main-memory databas...</td>\n",
       "      <td>J. Baulier, P. Bohannon, S. Gogate, C. Gupta, ...</td>\n",
       "      <td>International Conference on Management of Data</td>\n",
       "      <td>1999</td>\n",
       "      <td>conf/sigmod/HaasH99#304239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>873</th>\n",
       "      <td>0</td>\n",
       "      <td>Update Propagation Strategies for Improving th...</td>\n",
       "      <td>Nick Roussopoulos, Alexandros Labrinidis</td>\n",
       "      <td>VLDB</td>\n",
       "      <td>2001</td>\n",
       "      <td>databases and visualization</td>\n",
       "      <td>Daniel A. Keim</td>\n",
       "      <td>International Conference on Management of Data</td>\n",
       "      <td>1996</td>\n",
       "      <td>conf/vldb/LabrinidisR01#280349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875</th>\n",
       "      <td>0</td>\n",
       "      <td>Qualitative Representation of Spatial Knowledg...</td>\n",
       "      <td>Dimitris Papadias, Timos K. Sellis</td>\n",
       "      <td>VLDB J.</td>\n",
       "      <td>1994</td>\n",
       "      <td>Analysis of predictive spatio-temporal queries</td>\n",
       "      <td>Yufei Tao, Jimeng Sun, Dimitris Papadias</td>\n",
       "      <td>ACM Transactions on Database Systems (TODS)</td>\n",
       "      <td>2003</td>\n",
       "      <td>journals/vldb/PapadiasS94#958943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>879</th>\n",
       "      <td>0</td>\n",
       "      <td>ACM TODS in this Internet Age</td>\n",
       "      <td>Richard T. Snodgrass</td>\n",
       "      <td>SIGMOD Record</td>\n",
       "      <td>2003</td>\n",
       "      <td>The TriGS active object-oriented database syst...</td>\n",
       "      <td>G. Kappel, W. Retschitzegger</td>\n",
       "      <td>ACM SIGMOD Record</td>\n",
       "      <td>1998</td>\n",
       "      <td>journals/sigmod/Snodgrass03#290601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>886</th>\n",
       "      <td>0</td>\n",
       "      <td>MineSet(tm): A System for High-End Data Mining...</td>\n",
       "      <td>?</td>\n",
       "      <td>VLDB</td>\n",
       "      <td>1996</td>\n",
       "      <td>Efficient and Effective Clustering Methods for...</td>\n",
       "      <td>Raymond T. Ng, Jiawei Han</td>\n",
       "      <td>Very Large Data Bases</td>\n",
       "      <td>1994</td>\n",
       "      <td>conf/vldb/MiningGroup96#672827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>0</td>\n",
       "      <td>Cost Estimation of User-Defined Methods in Obj...</td>\n",
       "      <td>Kinji Ono, Jihad Boulos</td>\n",
       "      <td>SIGMOD Record</td>\n",
       "      <td>1999</td>\n",
       "      <td>New standard for stored procedures in SQL</td>\n",
       "      <td>Andrew Eisenberg</td>\n",
       "      <td>ACM SIGMOD Record</td>\n",
       "      <td>1996</td>\n",
       "      <td>journals/sigmod/BoulosO99#245907</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>463 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                       ltable_title  \\\n",
       "1        0          Database Principles Column - Introduction   \n",
       "2        0                              The PointCast Network   \n",
       "3        0                                    Chair's Message   \n",
       "5        0  Finding Generalized Projected Clusters In High...   \n",
       "6        0                Ripple Joins for Online Aggregation   \n",
       "..     ...                                                ...   \n",
       "873      0  Update Propagation Strategies for Improving th...   \n",
       "875      0  Qualitative Representation of Spatial Knowledg...   \n",
       "879      0                      ACM TODS in this Internet Age   \n",
       "886      0  MineSet(tm): A System for High-End Data Mining...   \n",
       "889      0  Cost Estimation of User-Defined Methods in Obj...   \n",
       "\n",
       "                               ltable_authors       ltable_venue  ltable_year  \\\n",
       "1                               Leonid Libkin      SIGMOD Record         1999   \n",
       "2            Vibha Dayal, Satish Ramakrishnan  SIGMOD Conference         1998   \n",
       "3                        Richard T. Snodgrass      SIGMOD Record         1998   \n",
       "5             Charu C. Aggarwal, Philip S. Yu  SIGMOD Conference         2000   \n",
       "6        Peter J. Haas, Joseph M. Hellerstein  SIGMOD Conference         1999   \n",
       "..                                        ...                ...          ...   \n",
       "873  Nick Roussopoulos, Alexandros Labrinidis               VLDB         2001   \n",
       "875        Dimitris Papadias, Timos K. Sellis            VLDB J.         1994   \n",
       "879                      Richard T. Snodgrass      SIGMOD Record         2003   \n",
       "886                                         ?               VLDB         1996   \n",
       "889                   Kinji Ono, Jihad Boulos      SIGMOD Record         1999   \n",
       "\n",
       "                                          rtable_title  \\\n",
       "1       Lifestreams: a storage model for personal data   \n",
       "2           Repositories and object oriented databases   \n",
       "3           Repositories and object oriented databases   \n",
       "5    DataBlitz storage manager: main-memory databas...   \n",
       "6    DataBlitz storage manager: main-memory databas...   \n",
       "..                                                 ...   \n",
       "873                        databases and visualization   \n",
       "875     Analysis of predictive spatio-temporal queries   \n",
       "879  The TriGS active object-oriented database syst...   \n",
       "886  Efficient and Effective Clustering Methods for...   \n",
       "889          New standard for stored procedures in SQL   \n",
       "\n",
       "                                        rtable_authors  \\\n",
       "1                        Eric Freeman, David Gelernter   \n",
       "2                                  Philip A. Bernstein   \n",
       "3                                  Philip A. Bernstein   \n",
       "5    J. Baulier, P. Bohannon, S. Gogate, C. Gupta, ...   \n",
       "6    J. Baulier, P. Bohannon, S. Gogate, C. Gupta, ...   \n",
       "..                                                 ...   \n",
       "873                                     Daniel A. Keim   \n",
       "875           Yufei Tao, Jimeng Sun, Dimitris Papadias   \n",
       "879                       G. Kappel, W. Retschitzegger   \n",
       "886                          Raymond T. Ng, Jiawei Han   \n",
       "889                                   Andrew Eisenberg   \n",
       "\n",
       "                                       rtable_venue  rtable_year  \\\n",
       "1                                ACM SIGMOD Record          1996   \n",
       "2                                ACM SIGMOD Record          1998   \n",
       "3                                ACM SIGMOD Record          1998   \n",
       "5    International Conference on Management of Data         1999   \n",
       "6    International Conference on Management of Data         1999   \n",
       "..                                              ...          ...   \n",
       "873  International Conference on Management of Data         1996   \n",
       "875    ACM Transactions on Database Systems (TODS)          2003   \n",
       "879                              ACM SIGMOD Record          1998   \n",
       "886                           Very Large Data Bases         1994   \n",
       "889                              ACM SIGMOD Record          1996   \n",
       "\n",
       "                                     id  \n",
       "1      journals/sigmod/Libkin99a#381893  \n",
       "2    conf/sigmod/RamakrishnanD98#273271  \n",
       "3    journals/sigmod/Snodgrass98#273271  \n",
       "5        conf/sigmod/AggarwalY00#304239  \n",
       "6            conf/sigmod/HaasH99#304239  \n",
       "..                                  ...  \n",
       "873      conf/vldb/LabrinidisR01#280349  \n",
       "875    journals/vldb/PapadiasS94#958943  \n",
       "879  journals/sigmod/Snodgrass03#290601  \n",
       "886      conf/vldb/MiningGroup96#672827  \n",
       "889    journals/sigmod/BoulosO99#245907  \n",
       "\n",
       "[463 rows x 10 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:00:24.079049Z",
     "start_time": "2020-04-16T17:00:19.342166Z"
    }
   },
   "outputs": [],
   "source": [
    "walmart_model = dm.MatchingModel(attr_summarizer='rnn')\n",
    "walmart_model.load_state('models/walmart_amazon_rnn.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:00:29.747778Z",
     "start_time": "2020-04-16T17:00:29.718412Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv('datasets/walmart_amazon/augmented_test.csv')\n",
    "testneg = test[test.label==0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:00:57.031101Z",
     "start_time": "2020-04-16T17:00:41.257908Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(610, 626)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f(testneg,walmart_model,0),len(testneg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:02:49.141031Z",
     "start_time": "2020-04-16T17:02:44.761028Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "591"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg_cp = testneg.copy()\n",
    "testneg_cp['ltable_modelno'] = testneg_cp['rtable_modelno']\n",
    "f(testneg_cp,walmart_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:02:55.634026Z",
     "start_time": "2020-04-16T17:02:51.576637Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg_cp = testneg.copy()\n",
    "testneg_cp['ltable_brand'] = testneg_cp['rtable_brand']\n",
    "f(testneg_cp,walmart_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:03:01.775473Z",
     "start_time": "2020-04-16T17:02:57.243009Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "434"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg_cp = testneg.copy()\n",
    "testneg_cp['ltable_title'] = testneg_cp['rtable_title']\n",
    "f(testneg_cp,walmart_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-16T17:04:02.668292Z",
     "start_time": "2020-04-16T17:03:58.119583Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "615"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testneg_cp = testneg.copy()\n",
    "testneg_cp['ltable_price'] = testneg_cp['rtable_price']\n",
    "f(testneg_cp,walmart_model,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}