IJCAI2022 Experiments RF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SETUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests import get\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import datetime\n",
    "from IPython.display import HTML\n",
    "import pyodbc\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from pandas.io.json import json_normalize\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "casebase=list()\n",
    "#Function used retrieve relavant predictions for a given evaluation case. We call the prediction scores \"similarity\" for reusability reasons. \n",
    "#Prediction instances are filtered to ensure that they are relevant to the given case.\n",
    "def retrieve_c_att22(case):\n",
    "    global casebase\n",
    "    cases=casebase\n",
    "    cases=cases[cases[\"IndustrySubgroupCode\"].astype(float)==float(case[\"IndustrySubgroupCode\"])]\n",
    "    cases=cases[cases[\"MunicipalityNumber\"].astype(int)==int(case[\"MunicipalityNumber\"])]\n",
    "    results = cases.apply(pd.to_numeric, errors='coerce').fillna(cases).sort_values(by='similarity', ascending=False)\n",
    "    results['similarity'] = results['similarity'].astype(float)\n",
    "    results['NonCompliance'] = results['NonCompliance'].astype(int)\n",
    "    return results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "casebase=list()\n",
    "initCBCopy=casebase\n",
    "currentCBCopy=casebase\n",
    "\n",
    "#Fill casebase in case its empty\n",
    "dataZero=()\n",
    "def readData():\n",
    "    global dataZero\n",
    "    cnxn = pyodbc.connect('DRIVER={SQL Server Native Client 11.0};SERVER=localhost;DATABASE=IJCAI2022;Trusted_Connection=yes;')\n",
    "    cursor = cnxn.cursor()\n",
    "    dataZero=pd.read_sql(\"SELECT  * \"+\"FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDb]\"+\n",
    "                      \"left join (select NewID() as new, InspectionId as inspID from [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDb] group by InspectionId) as a on a.inspId=InspectionId\"+\n",
    "                      \" where InspectionDateId<20190601 ORDER BY new\",cnxn)\n",
    "    #Query selects items in random order (via the newID method), grouped by the inspectionIDs. \n",
    "    #We are doing crossvalidation with random split based on the indices of the randomly retrieved inspection ids. \n",
    "    #Thus, we are evaluating outcomes of inspections. This method also prevents data bleed/leakage.\n",
    "\n",
    "def FindBestParameters():\n",
    "    global casebase\n",
    "    global data00\n",
    "    # Specifying the ODBC driver, server name, database, etc. directly\n",
    "    #traindateid=\" where (InspectionDateId<\"+str(year)+\"0101 or InspectionDateId>=\"+str(year+1)+\"0101) and InspectionDateId<20190601\"\n",
    "    #testdateid=\" where InspectionDateId>=\"+str(year)+\"0101 and InspectionDateId<\"+str(year+1)+\"0101 and InspectionDateId<20190601\"\n",
    "    cnxn = pyodbc.connect('DRIVER={SQL Server Native Client 11.0};SERVER=localhost;DATABASE=IJCAI2022;Trusted_Connection=yes;')\n",
    "    cursor = cnxn.cursor()\n",
    "    data0=pd.read_sql(\"SELECT  * \"+\"FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDb]\",cnxn)\n",
    "    data=pd.DataFrame(data={'NonCompliance':data0['NonCompliance'],'IndustrySubgroupCode':data0['IndustrySubgroupCode'],'ControlPointText': data0[\"ControlPointText\"],\"InspectionDateId\":data0[\"InspectionDateId\"],\"Municipality\":data0[\"Municipality\"]})\n",
    "    #datatestout=data0.loc[round((len(data0)/8)*year):round((len(data0)/8)*(year+1))]\n",
    "    #print(data)\n",
    "    #cat_vars=['ControlPointText','IndustryMainAreaCode','IndustrySubgroupCode','OrganisationAge','OrganisationSize','Municipality']\n",
    "    cat_vars=['ControlPointText','IndustrySubgroupCode',\"Municipality\"]\n",
    "    for var in cat_vars:\n",
    "        cat_list='var'+'_'+var\n",
    "        cat_list = pd.get_dummies(data[var], prefix=var)\n",
    "        #print(cat_list)\n",
    "        data1=data.join(cat_list)\n",
    "        data=data1\n",
    "         #checklist.loc[checklist[\"AntallBrudd\"]==-1,\"AntallBrudd\"]=0\n",
    "    cat_vars=['ControlPointText','IndustrySubgroupCode',\"Municipality\"]\n",
    "    data_vars=data.columns.values.tolist()\n",
    "    to_keep=[i for i in data_vars if i not in cat_vars]\n",
    "    #print(to_keep)\n",
    "    data=data[to_keep]\n",
    "    cat_vars=['NonCompliance']\n",
    "    data_vars=data.columns.values.tolist()\n",
    "    to_keep=[i for i in data_vars if i not in cat_vars]\n",
    "    data2=data[to_keep]\n",
    "\n",
    "\n",
    "    data.loc[np.isnan(data['NonCompliance']),\"NonCompliance\"]=0\n",
    "    data.loc[data['NonCompliance']<0,\"NonCompliance\"]=0\n",
    "    data.loc[data['NonCompliance']>1,\"NonCompliance\"]=1\n",
    "\n",
    "    y=data[\"NonCompliance\"]\n",
    "    data=data.drop(columns=[\"NonCompliance\"])\n",
    "\n",
    "    data=data.drop(columns=[\"InspectionDateId\"])\n",
    "\n",
    "    mlp_gs = RandomForestClassifier()\n",
    "\n",
    "    parameter_space = {\n",
    "        'n_estimators': [2,10,20,50,100],\n",
    "        'max_samples': [0.5,1.0],\n",
    "        'min_samples_split':[2,10]\n",
    "    }\n",
    "    \n",
    "    clf = GridSearchCV(mlp_gs, parameter_space, n_jobs=2, cv=4)\n",
    "    clf.fit(data,y)\n",
    "\n",
    "    return clf\n",
    " \n",
    "    \n",
    "def fillCaseBase(index):\n",
    "    global casebase\n",
    "    global dataZero\n",
    "    # Specifying the ODBC driver, server name, database, etc. directly\n",
    "    data0=dataZero\n",
    "    data=pd.DataFrame(data={'NonCompliance':data0['NonCompliance'],'IndustrySubgroupCode':data0['IndustrySubgroupCode'],'ControlPointText': data0[\"ControlPointText\"],\"InspectionDateId\":data0[\"InspectionDateId\"],\"Municipality\":data0[\"Municipality\"]})\n",
    "    datatestout=data0.loc[round((len(data0)/8)*index):round((len(data0)/8)*(index+1))]\n",
    "    #Select independent variable and convert to one-hot vectors\n",
    "    cat_vars=['ControlPointText','IndustrySubgroupCode',\"Municipality\"]\n",
    "    for var in cat_vars:\n",
    "        cat_list='var'+'_'+var\n",
    "        cat_list = pd.get_dummies(data[var], prefix=var)\n",
    "        data1=data.join(cat_list)\n",
    "        data=data1\n",
    "    cat_vars=['ControlPointText','IndustrySubgroupCode',\"Municipality\"]\n",
    "    data_vars=data.columns.values.tolist()\n",
    "    to_keep=[i for i in data_vars if i not in cat_vars]\n",
    "    data=data[to_keep]\n",
    "\n",
    "\n",
    "    #Making sure that the target variable is binary\n",
    "    data.loc[np.isnan(data['NonCompliance']),\"NonCompliance\"]=0\n",
    "    data.loc[data['NonCompliance']<0,\"NonCompliance\"]=0\n",
    "    data.loc[data['NonCompliance']>1,\"NonCompliance\"]=1\n",
    "\n",
    "    #retrieving the test-fold out of the 8 crossvalidation folds (rest are training folds.)\n",
    "    datatest=data.loc[round((len(data)/8)*index):round((len(data)/8)*(index+1))]\n",
    "    datatesty=datatest[\"NonCompliance\"]\n",
    "    datatest=datatest.drop(columns=[\"NonCompliance\"])\n",
    "    #Retrieving the 7 training folds, leaving the test-fold out.\n",
    "    datacp=data.copy()\n",
    "    if index>0 and index <7:                   \n",
    "        datacp=datacp.loc[0:round((len(datacp)/8)*index)-1]\n",
    "        datacp=datacp.append(data.loc[round((len(data)/8)*(index+1)):len(data)])\n",
    "    elif index<1:\n",
    "        datacp=datacp.loc[round((len(datacp)/8))*(index+1):len(datacp)]\n",
    "    else:\n",
    "        datacp=datacp.loc[0:round((len(data)/8)*index)]\n",
    "    \n",
    "    data=datacp\n",
    "    #Creating the target variable, dropping unwanted columns from the training set and training the model.    \n",
    "    y=data[\"NonCompliance\"]\n",
    "    data=data.drop(columns=[\"NonCompliance\"])\n",
    "    data=data.drop(columns=[\"InspectionDateId\"])\n",
    "    datatest=datatest.drop(columns=[\"InspectionDateId\"])\n",
    "    clf = RandomForestClassifier(max_samples=0.5, min_samples_split=10,n_estimators=50)\n",
    "    t=clf.fit(data,y)\n",
    "    #generate prediction for the test data set. The test data set is used to create and evaluate checklists below.\n",
    "    tes=clf.predict_proba(datatest)\n",
    "    tes=(datatestout).assign(similarity=tes[:,1]) #We named the prediction score \"similarity\" for easy reuse in other scripts.\n",
    "    tes=tes.assign(NonCompliance=datatesty)\n",
    "    casebase=tes #Test data set is assigned to \"casebase\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find the best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time: 18633.6580452919\n",
      "{'max_samples': 0.5, 'min_samples_split': 10, 'n_estimators': 50}\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10, n_estimators=50)\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "clf=FindBestParameters()\n",
    "traintime=(time.time()-start_time)\n",
    "print(\"Training time: \"+ str(traintime))\n",
    "\n",
    "print(clf.best_params_)\n",
    "print(clf.best_estimator_)\n",
    "fh2=open(\"Log_\" + \"RFChecklistsBestParameters\"+ '.txt', 'w+')\n",
    "fh2.write(\"Training time: \"+ str(traintime))\n",
    "fh2.write(\"Best parameters:\"+ str(clf.best_params_))\n",
    "fh2.write(\"Best estimator:\"+ str(clf.best_estimator_))\n",
    "fh2.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EXPERIMENT, STATISTICS FOR GENERATED CHECKLISTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "138939\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10)\n",
      "Train time: 3221.3368487358093\n",
      "950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-9f8a06cba429>:56: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9402\n",
      "18474\n",
      "27802\n",
      "36929\n",
      "45999\n",
      "55258\n",
      "64426\n",
      "73762\n",
      "83036\n",
      "92396\n",
      "101655\n",
      "111012\n",
      "120358\n",
      "129621\n",
      "138871\n",
      "Current avg time:3750.791635274887\n",
      "Precision (gt): 0.2524728141399486\n",
      "Precision(val): 0.28477504637769424\n",
      "Precision: 0.28477504637769424\n",
      "Recall(val): 0.6687721054679276\n",
      "Recall: 0.6687721054679276\n",
      "Accuracy(val): 0.5170015019465504\n",
      "Accuracy: 0.5170015019465504\n",
      "Average number of control points per checklist(gt): 9.705013964622955\n",
      "Average number of control points per checklist: 14.171166378507781\n",
      "Average number of non-compliant control points per checklist(gt): 0.23988107251174265\n",
      "Average number of non-compliant control points per checklist(val): 4.096491175754277\n",
      "Average number of non-compliant control points per checklist: 4.096491175754277\n",
      "Percentage of control points in the intersection between the CBR generated and the original checklists: 0.6848422850600171\n",
      "Average similarity: 0.27870879527539444\n",
      "\n",
      "Train time: 3221.3368487358093\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138939\n",
      "277878\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10)\n",
      "Train time: 3856.0079085826874\n",
      "944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-9f8a06cba429>:56: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "148275\n",
      "157653\n",
      "166916\n",
      "176347\n",
      "185651\n",
      "195077\n",
      "204390\n",
      "213630\n",
      "222789\n",
      "232301\n",
      "241662\n",
      "250952\n",
      "260461\n",
      "269461\n",
      "Current avg time:4061.4484125375748\n",
      "Precision (gt): 0.25328777025356874\n",
      "Precision(val): 0.2844388423769513\n",
      "Precision: 0.2844388423769513\n",
      "Recall(val): 0.6708493564897535\n",
      "Recall: 0.6708493564897535\n",
      "Accuracy(val): 0.5205134145498695\n",
      "Accuracy: 0.5205134145498695\n",
      "Average number of control points per checklist(gt): 9.626423690205012\n",
      "Average number of control points per checklist: 14.200723569610076\n",
      "Average number of non-compliant control points per checklist(gt): 0.24062847261125586\n",
      "Average number of non-compliant control points per checklist(val): 4.097817398702641\n",
      "Average number of non-compliant control points per checklist: 4.097817398702641\n",
      "Percentage of control points in the intersection between the CBR generated and the original checklists: 0.6778826193621438\n",
      "Average similarity: 0.28434979273958844\n",
      "\n",
      "Train time: 3856.0079085826874\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "277878\n",
      "416816\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10)\n",
      "Train time: 3533.9720079898834\n",
      "975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-9f8a06cba429>:56: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "286779\n",
      "295678\n",
      "305029\n",
      "314469\n",
      "323734\n",
      "333186\n",
      "342468\n",
      "351386\n",
      "360284\n",
      "369642\n",
      "379019\n",
      "388320\n",
      "397664\n",
      "406836\n",
      "415926\n",
      "Current avg time:4034.105592330297\n",
      "Precision (gt): 0.24608725965789208\n",
      "Precision(val): 0.27187574459387326\n",
      "Precision: 0.27187574459387326\n",
      "Recall(val): 0.6686879606189056\n",
      "Recall: 0.6686879606189056\n",
      "Accuracy(val): 0.5139251523697452\n",
      "Accuracy: 0.5139251523697452\n",
      "Average number of control points per checklist(gt): 9.67919036909644\n",
      "Average number of control points per checklist: 14.153591744939806\n",
      "Average number of non-compliant control points per checklist(gt): 0.23299993615181025\n",
      "Average number of non-compliant control points per checklist(val): 3.907280340210538\n",
      "Average number of non-compliant control points per checklist: 3.907280340210538\n",
      "Percentage of control points in the intersection between the CBR generated and the original checklists: 0.6838681335115482\n",
      "Average similarity: 0.27735777878913315\n",
      "\n",
      "Train time: 3533.9720079898834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "416816\n",
      "555755\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10)\n",
      "Train time: 3137.1482152938843\n",
      "972\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-9f8a06cba429>:56: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "425937\n",
      "435274\n",
      "444401\n",
      "453957\n",
      "463672\n",
      "473010\n",
      "482002\n",
      "491263\n",
      "500515\n",
      "509906\n",
      "519076\n",
      "528398\n",
      "537840\n",
      "547065\n",
      "Current avg time:3919.8549568653107\n",
      "Precision (gt): 0.25509335738351757\n",
      "Precision(val): 0.2935643785203567\n",
      "Precision: 0.2935643785203567\n",
      "Recall(val): 0.681155679359142\n",
      "Recall: 0.681155679359142\n",
      "Accuracy(val): 0.5219878795273742\n",
      "Accuracy: 0.5219878795273742\n",
      "Average number of control points per checklist(gt): 9.677553205728818\n",
      "Average number of control points per checklist: 14.226609556953553\n",
      "Average number of non-compliant control points per checklist(gt): 0.24041123401651016\n",
      "Average number of non-compliant control points per checklist(val): 4.23785724327369\n",
      "Average number of non-compliant control points per checklist: 4.23785724327369\n",
      "Percentage of control points in the intersection between the CBR generated and the original checklists: 0.6802431153386586\n",
      "Average similarity: 0.28509493876398667\n",
      "\n",
      "Train time: 3137.1482152938843\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "555755\n",
      "694694\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10)\n",
      "Train time: 3090.356469631195\n",
      "987\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-9f8a06cba429>:56: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "564823\n",
      "574435\n",
      "583934\n",
      "592929\n",
      "602296\n",
      "611679\n",
      "621030\n",
      "630222\n",
      "639466\n",
      "648570\n",
      "657753\n",
      "667351\n",
      "676527\n",
      "685626\n",
      "Current avg time:3843.968590927124\n",
      "Precision (gt): 0.25337519698909855\n",
      "Precision(val): 0.28745805926082824\n",
      "Precision: 0.28745805926082824\n",
      "Recall(val): 0.6794347232367588\n",
      "Recall: 0.6794347232367588\n",
      "Accuracy(val): 0.5184253202753029\n",
      "Accuracy: 0.5184253202753029\n",
      "Average number of control points per checklist(gt): 9.695512820512821\n",
      "Average number of control points per checklist: 14.200320512820513\n",
      "Average number of non-compliant control points per checklist(gt): 0.23929879715637084\n",
      "Average number of non-compliant control points per checklist(val): 4.1492924761837156\n",
      "Average number of non-compliant control points per checklist: 4.1492924761837156\n",
      "Percentage of control points in the intersection between the CBR generated and the original checklists: 0.6827671820336305\n",
      "Average similarity: 0.2810288930625874\n",
      "\n",
      "Train time: 3090.356469631195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "694694\n",
      "833632\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10)\n",
      "Train time: 3179.7451939582825\n",
      "983\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-9f8a06cba429>:56: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "703753\n",
      "712801\n",
      "722349\n",
      "731522\n",
      "740577\n",
      "749984\n",
      "759472\n",
      "768609\n",
      "778012\n",
      "787452\n",
      "796955\n",
      "805819\n",
      "815221\n",
      "824673\n",
      "Current avg time:3810.0900650819144\n",
      "Precision (gt): 0.25222771563232343\n",
      "Precision(val): 0.2896636742494623\n",
      "Precision: 0.2896636742494623\n",
      "Recall(val): 0.6764638897491205\n",
      "Recall: 0.6764638897491205\n",
      "Accuracy(val): 0.5172320687856878\n",
      "Accuracy: 0.5172320687856878\n",
      "Average number of control points per checklist(gt): 9.733582487987187\n",
      "Average number of control points per checklist: 14.173918846769888\n",
      "Average number of non-compliant control points per checklist(gt): 0.24051185270652947\n",
      "Average number of non-compliant control points per checklist(val): 4.173438435234422\n",
      "Average number of non-compliant control points per checklist: 4.173438435234422\n",
      "Percentage of control points in the intersection between the CBR generated and the original checklists: 0.6867248636890132\n",
      "Average similarity: 0.28145134528670135\n",
      "\n",
      "Train time: 3179.7451939582825\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "833632\n",
      "972571\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10)\n",
      "Train time: 3425.900997400284\n",
      "965\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-9f8a06cba429>:56: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "843011\n",
      "852366\n",
      "861286\n",
      "870733\n",
      "880211\n",
      "889490\n",
      "898795\n",
      "907930\n",
      "917304\n",
      "926496\n",
      "935659\n",
      "945051\n",
      "954428\n",
      "963705\n",
      "Current avg time:3821.1916993345535\n",
      "Precision (gt): 0.25523670480330213\n",
      "Precision(val): 0.27937758464197243\n",
      "Precision: 0.27937758464197243\n",
      "Recall(val): 0.6746786130334028\n",
      "Recall: 0.6746786130334028\n",
      "Accuracy(val): 0.5198082544975724\n",
      "Accuracy: 0.5198082544975724\n",
      "Average number of control points per checklist(gt): 9.728804493179995\n",
      "Average number of control points per checklist: 14.236159400909335\n",
      "Average number of non-compliant control points per checklist(gt): 0.24216427127299125\n",
      "Average number of non-compliant control points per checklist(val): 4.033593417319791\n",
      "Average number of non-compliant control points per checklist: 4.033593417319791\n",
      "Percentage of control points in the intersection between the CBR generated and the original checklists: 0.6833868755753443\n",
      "Average similarity: 0.28293250244138624\n",
      "\n",
      "Train time: 3425.900997400284\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "C:\\Users\\flogard_e\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1720: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "972571\n",
      "1111510\n",
      "RandomForestClassifier(max_samples=0.5, min_samples_split=10)\n",
      "Train time: 10335.810318946838\n",
      "974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-9f8a06cba429>:56: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('display.max_colwidth', -1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "981586\n",
      "990693\n",
      "999982\n",
      "1009342\n",
      "1018848\n",
      "1028415\n",
      "1037882\n",
      "1047118\n",
      "1056411\n",
      "1065641\n",
      "1074493\n",
      "1083679\n",
      "1092817\n",
      "1102285\n",
      "1111323\n",
      "Current avg time:4703.863602668047\n",
      "Precision (gt): 0.25549876364836144\n",
      "Precision(val): 0.2816447579078633\n",
      "Precision: 0.2816447579078633\n",
      "Recall(val): 0.6780733288299267\n",
      "Recall: 0.6780733288299267\n",
      "Accuracy(val): 0.5196740058655847\n",
      "Accuracy: 0.5196740058655847\n",
      "Average number of control points per checklist(gt): 9.607494020728142\n",
      "Average number of control points per checklist: 14.140446452298697\n",
      "Average number of non-compliant control points per checklist(gt): 0.24327719111136833\n",
      "Average number of non-compliant control points per checklist(val): 4.02591654031217\n",
      "Average number of non-compliant control points per checklist: 4.02591654031217\n",
      "Percentage of control points in the intersection between the CBR generated and the original checklists: 0.6794335704419241\n",
      "Average similarity: 0.2809945752586866\n",
      "\n",
      "Train time: 10335.810318946838\n"
     ]
    }
   ],
   "source": [
    "\n",
    "currentCBCopy=initCBCopy\n",
    "readData()\n",
    "timetotal=0\n",
    "traintime=0\n",
    "traintimetot=0\n",
    "precctot=0\n",
    "accctot=0\n",
    "preccgttot=0\n",
    "recctot=0\n",
    "fh2=open(\"Log_\" + \"RFChecklists\" + '.txt', 'w+')\n",
    "#for-loop to perform 8-fold crossvalidation. ik is used as an index for the validation folds.\n",
    "for ik in range(0,8):\n",
    "    \n",
    "    start_time = time.time()\n",
    "    fillCaseBase(ik)\n",
    "    initCBCopy=casebase\n",
    "    currentCBCopy=initCBCopy\n",
    "\n",
    "    traintime=(time.time()-start_time)\n",
    "    traintimetot=traintimetot+(time.time()-start_time)\n",
    "    print(\"Train time: \"+str(traintime))\n",
    "    noncompliance=0 #true positive(for ground truth labels only)\n",
    "    controlpointsgtcount=0 \n",
    "    noncompliancengt=0 #true positive (for statistical estimates, see main paper)\n",
    "    controlpointscount=0 \n",
    "    truepositiveval=0\n",
    "    truepositive=0\n",
    "    falsepositivengt=0\n",
    "    recallval=0\n",
    "    accuracyval=0\n",
    "    precision=0\n",
    "    precisiongt=0\n",
    "    lengthprecgt=0\n",
    "    recall=0\n",
    "    accuracy=0\n",
    "    lengthprec=0.00001\n",
    "    lengthvalprec=0\n",
    "    lengthrec=0.00001\n",
    "    lengthvalrec=0\n",
    "    lengthacc=0.00001\n",
    "    lengthvalacc=0\n",
    "    truenegative=0\n",
    "    falsenegative=0\n",
    "    similarity=0\n",
    "    counter=0\n",
    "    Kcp=15\n",
    "    pd.set_option('display.max_rows', None)\n",
    "    pd.set_option('display.max_columns', None)\n",
    "    pd.set_option('display.width', None)\n",
    "    pd.set_option('display.max_colwidth', -1)\n",
    "    noncompliantInspection=0\n",
    "    inspections=casebase.drop_duplicates(subset = [\"InspectionId\"])\n",
    "    cases=casebase\n",
    "    cases[\"NonCompliance\"]=cases[\"NonCompliance\"].astype(int)\n",
    "    casestest=len(cases.drop_duplicates(subset = [\"ControlPointText\"]))\n",
    "    instanceMatches=list()\n",
    "    negatives=list()\n",
    "    inspectionstot=0\n",
    "    precchecklists=0\n",
    "    recchecklists=0\n",
    "    accchecklists=0\n",
    "    precchecklistsval=0\n",
    "    recchecklistsval=0\n",
    "    accchecklistsval=0\n",
    "    print(casestest)\n",
    "    #Iterate through all the past inspections in the data set. \n",
    "    #\"case\" contains a single inspection (which in practice consists of a set of instances in the data set).\n",
    "    for ind, case in inspections.iterrows():\n",
    "   \n",
    "        inspectionstot+=1\n",
    "        uniquechecklistlengthval=0\n",
    "        instanceMatches=list()\n",
    "        counter+=1\n",
    "        if counter>500:\n",
    "            counter=0\n",
    "            print(ind)\n",
    "            \n",
    "\n",
    "\n",
    "        #Retrieve the existing checklist. ExistingChecklist contains the checklist used in the current inspection.\n",
    "        existingChecklist=cases[cases['InspectionId']==case['InspectionId']]\n",
    "        #filter validation instances (cases) so that they match the industry code and municipality of the current inspection.\n",
    "        filteredCases=cases[cases[\"IndustrySubgroupCode\"].astype(float)==float(case[\"IndustrySubgroupCode\"])]\n",
    "        filteredCases=filteredCases[filteredCases[\"MunicipalityNumber\"].astype(int)==int(case[\"MunicipalityNumber\"])]\n",
    "        filteredCases[\"NonCompliance\"]=filteredCases[\"NonCompliance\"].astype(float)\n",
    "        #Temporary assigning the filtered cases. Another filter is applied further down to filter out positive cases.\n",
    "        negatives=filteredCases\n",
    "        if len(filteredCases)<=0:#Fail-safe mechanism. filteredCases should in practice never be 0.\n",
    "            continue\n",
    "        \n",
    "        \n",
    "        #Retrieve a constructed checklist\n",
    "        checklist=retrieve_c_att22(case)#Retrieve control points that matches each inspection (input case). checklist=CL from paper\n",
    "\n",
    "        #Drop any retrieved duplicates and remove select the top (Kcp) items with highest prediction score\n",
    "        uniqueChecklist=checklist.drop_duplicates(subset = [\"ControlPointText\"])#Find all unique control points by removing duplicates\n",
    "        uniqueChecklist=uniqueChecklist.head(Kcp)\n",
    "        if len(uniqueChecklist)>(0):#Safe check to make sure that somethingis on the checklist.\n",
    "            truepositive=0\n",
    "            truepositiveval=0\n",
    "            similarity+=uniqueChecklist[\"similarity\"].sum() #Sums prediction score (prediction score=similarity)\n",
    "            #Record statistics. sums up the ratio/fraction of non-compliance for each of the (unique) item in the checklist.\n",
    "            precpercl=0\n",
    "            lengthorgcl=0\n",
    "            #Iterate through all the items on a constructed checklist\n",
    "            for ind2, generatedChecklistControlpoint in uniqueChecklist.iterrows():#Find overlap between the existing and new generated checklist\n",
    "                #find the predicted negative records for the constructed checklist. uniqueChecklist contains all the predicted positives.\n",
    "                negatives=negatives[negatives[\"ControlPointText\"]!=generatedChecklistControlpoint[\"ControlPointText\"]]\n",
    "                #excp contains the items that can be found on both the existing checklist and the generated checklist\n",
    "                excp=existingChecklist[existingChecklist[\"ControlPointText\"]==generatedChecklistControlpoint[\"ControlPointText\"]]\n",
    "                #calculate ground truth precision for if excp contains an checklist item.\n",
    "                if len(excp)>0:\n",
    "                    precpercl+=((excp[\"NonCompliance\"].sum())/len(excp[\"NonCompliance\"])) #used to calculate average ground truth precision\n",
    "                    lengthorgcl+=1 #used to calculate average ground truth precision \n",
    "                    controlpointsgtcount+=1 #Used to find average number of items per checklist\n",
    "                controlpointscount+=1\n",
    "                #groups instances from the vaildation set by items and selects instances that matches the current item (generatedChecklistControlPoint)\n",
    "                instancesMatchingCurrentGenClItem=filteredCases.groupby([\"ControlPointText\"],as_index=False).mean()\n",
    "                instancesMatchingCurrentGenClItem=instancesMatchingCurrentGenClItem[instancesMatchingCurrentGenClItem[\"ControlPointText\"]==generatedChecklistControlpoint[\"ControlPointText\"]]\n",
    "                #Finds the total number of instances in the validation set with positive labels.\n",
    "                summ2=instancesMatchingCurrentGenClItem[\"NonCompliance\"].sum() \n",
    "                #Calculate the number of positive instances in the validation set (that matches the item), divided by the total number of instances.\n",
    "                #the expression summ2/len(instancesMatchingCurrentGenClItem[\"NonCompliance\"]) is a float-number between [0,1]\n",
    "                if len(instancesMatchingCurrentGenClItem[\"NonCompliance\"])>0:\n",
    "                    noncompliance+=(summ2/len(instancesMatchingCurrentGenClItem[\"NonCompliance\"]))\n",
    "                    truepositiveval+=(summ2/len(instancesMatchingCurrentGenClItem[\"NonCompliance\"]))\n",
    "                #test if there are at least 1 instance in the validation set that matches the current inspection and checklist item.\n",
    "                matchlen=len(instancesMatchingCurrentGenClItem[\"NonCompliance\"])\n",
    "                if matchlen>0:\n",
    "                    lengthvalprec+=1\n",
    "                    uniquechecklistlengthval+=1\n",
    "                \n",
    "                if matchlen==0:#expression is true if validation set record of checklist item does not exist. This expression never comes true for any method other than BCBR/CBCBR.\n",
    "                    instanceMatches=cases[cases[\"IndustrySubgroupCode\"].astype(float)==float(generatedChecklistControlpoint[\"IndustrySubgroupCode\"])]\n",
    "                    instanceMatches=instanceMatches[instanceMatches[\"MunicipalityNumber\"].astype(int)==int(generatedChecklistControlpoint[\"MunicipalityNumber\"])]\n",
    "                    instanceMatches=instanceMatches[instanceMatches[\"ControlPointText\"]==generatedChecklistControlpoint[\"ControlPointText\"]]\n",
    "                    instanceMatches[\"NonCompliance\"]=instanceMatches[\"NonCompliance\"].astype(float)\n",
    "\n",
    "                    matchesSum=instanceMatches[\"NonCompliance\"].sum()\n",
    "\n",
    "\n",
    "                    falsepositivengt+=(len(instanceMatches[\"NonCompliance\"])-matchesSum)\n",
    "                    if len(instanceMatches[\"NonCompliance\"])>0:\n",
    "                        noncompliancengt+=(matchesSum/len(instanceMatches[\"NonCompliance\"]))\n",
    "                        truepositive+=(matchesSum/len(instanceMatches[\"NonCompliance\"]))\n",
    "                        precision+=(matchesSum/len(instanceMatches[\"NonCompliance\"]))\n",
    "                        lengthprec+=1\n",
    "                #End of for-loop\n",
    "            if lengthorgcl>0:\n",
    "                precisiongt+=(precpercl/lengthorgcl)\n",
    "                lengthprecgt+=1      \n",
    "\n",
    "            #Calculate statistics on checklists level\n",
    "            uniquenegatives=len(negatives.drop_duplicates(subset = [\"ControlPointText\"]))\n",
    "            negativecpy=negatives.copy()\n",
    "            negativecpy[\"NonCompliance\"]=negativecpy[\"NonCompliance\"].astype(float)\n",
    "            groupedby=negativecpy.groupby([\"ControlPointText\"],as_index=False).count()\n",
    "            groupedbys=negativecpy.groupby([\"ControlPointText\"],as_index=False).sum()\n",
    "            groupedby[\"NonCompliance\"]=groupedbys[\"NonCompliance\"]/groupedby[\"NonCompliance\"]\n",
    "            uniquecalcnegatives=groupedby.drop_duplicates(subset = [\"ControlPointText\"])\n",
    "            noncompliancenegatives=uniquecalcnegatives[\"NonCompliance\"].sum()\n",
    "\n",
    "            falsenegativeelement=0\n",
    "            if uniquenegatives>0:\n",
    "                falsenegativeelement=noncompliancenegatives/uniquenegatives #To avoid selection bias effects\n",
    "            truenegativeelement=1-falsenegativeelement\n",
    "\n",
    "            truepositivesprchecklistval=(truepositiveval)\n",
    "            falsepositivesprchecklistval=(uniquechecklistlengthval-truepositivesprchecklistval)\n",
    "\n",
    "            truepositivesprchecklist=(truepositive+truepositiveval)\n",
    "            falsepositivesprchecklist=(len(uniqueChecklist)-(truepositive+truepositiveval))\n",
    "            truenegativesprchecklist=truenegativeelement*uniquenegatives\n",
    "            falsenegativesprchecklist=falsenegativeelement*uniquenegatives\n",
    "\n",
    "\n",
    "            precprchecklist=truepositivesprchecklist/len(uniqueChecklist)\n",
    "            precchecklists+=precprchecklist\n",
    "            precprchecklistval=0\n",
    "            if uniquechecklistlengthval>0:\n",
    "                precprchecklistval=truepositivesprchecklistval/uniquechecklistlengthval\n",
    "            precchecklistsval+=precprchecklistval\n",
    "\n",
    "            recprchecklist=0\n",
    "            if (truepositivesprchecklist+falsenegativesprchecklist)>0:\n",
    "                recprchecklist=truepositivesprchecklist/(truepositivesprchecklist+falsenegativesprchecklist)\n",
    "            recchecklists+=recprchecklist\n",
    "\n",
    "            recprchecklistval=0\n",
    "            if (truepositivesprchecklistval+falsenegativesprchecklist)>0:\n",
    "                recprchecklistval=truepositivesprchecklistval/(truepositivesprchecklistval+falsenegativesprchecklist)\n",
    "            recchecklistsval+=recprchecklistval\n",
    "\n",
    "            accprchecklist=0\n",
    "            if (len(uniqueChecklist)+uniquenegatives)>0:\n",
    "                accprchecklist=(truepositivesprchecklist+truenegativesprchecklist)/(len(uniqueChecklist)+uniquenegatives)\n",
    "            accchecklists+=accprchecklist\n",
    "\n",
    "            accprchecklistval=0\n",
    "            if (uniquechecklistlengthval+uniquenegatives)>0:\n",
    "                accprchecklistval=(truepositivesprchecklistval+truenegativesprchecklist)/(uniquechecklistlengthval+uniquenegatives)\n",
    "            accchecklistsval+=accprchecklistval\n",
    "\n",
    "            #|true positives for each checklist|=sum(true positive elements in checklist)\n",
    "            #|false positives for each checklist|=sum(false positive elements in checklist)\n",
    "            #|true negatives for each checklist|=|true unique negative elements not in checklist|\n",
    "            #|false negatives for each checklist|=|false unique negative elements not in checklist|\n",
    "        #End of inner for loop\n",
    "    #Recording statistics for the current cross-validation fold.\n",
    "    if inspectionstot>0:\n",
    "        precctot+=precchecklists/inspectionstot\n",
    "        accctot+=accchecklists/inspectionstot\n",
    "        preccgttot+=precisiongt/lengthprecgt\n",
    "        recctot+=recchecklists/inspectionstot\n",
    "    timetotal=timetotal+(time.time()-start_time)\n",
    "    print(\"Current avg time:\"+ str(timetotal/(ik+1)))\n",
    "    \n",
    "    print(\"Precision (gt): \"+str(precisiongt/lengthprecgt))\n",
    "    print(\"Precision: \"+str((precchecklists)/(inspectionstot)))\n",
    "    print(\"Recall: \"+str((recchecklists)/(inspectionstot)))\n",
    "    print(\"Accuracy: \"+str((accchecklists)/(inspectionstot)))\n",
    "    \n",
    "    #Additional information\n",
    "    print(\"Average number of items per checklist(gt): \"+str(controlpointsgtcount/inspectionstot))\n",
    "    print(\"Average number of items per checklist: \"+str((controlpointscount)/inspectionstot))\n",
    "    print(\"Average number of non-compliant items per checklist(gt): \"+str(precisiongt/inspectionstot))\n",
    "    print(\"Average number of non-compliant items per checklist: \"+str((noncompliance+noncompliancengt)/inspectionstot))\n",
    "    #number of common control points between CBR generated and original lists divided by the number of control points in the CBR generated list\n",
    "\n",
    "    print(\"Average similarity: \"+str(similarity/(lengthprecgt+lengthvalprec)))\n",
    "    print(\"\\nTrain time: \"+str(traintime))\n",
    "\n",
    "\n",
    "\n",
    "print(\"\\nAverage Traintime: \"+str(traintimetot/8))                                \n",
    "print(\"\\nAverage Accuracy: \"+str(accctot/8))\n",
    "print(\"\\nAverage Prec: \"+str(precctot/8))\n",
    "print(\"\\nAverage Rec: \"+str(recctot/8))\n",
    "print(\"\\nAverage Precgt: \"+str(preccgttot/8))\n",
    "fh2.write(\"\\nAverage Traintime: \"+str(traintimetot/8))                                \n",
    "fh2.write(\"\\nAverage Accuracy: \"+str(accctot/8))\n",
    "fh2.write(\"\\nAverage Prec: \"+str(precctot/8))\n",
    "fh2.write(\"\\nAverage Rec: \"+str(recctot/8))\n",
    "fh2.write(\"\\nAverage Precgt: \"+str(preccgttot/8))\n",
    "fh2.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}