DB_Reaxys (Fragment DB).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This continues from DB_Reaxys and supplements the database with active fragment/carrier fragment labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-22T22:58:45.407708Z",
     "start_time": "2021-12-22T22:58:44.826085Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r\n",
      "     active environment : impurityproject\r\n",
      "    active env location : /home/aa2133/miniconda3/envs/impurityproject\r\n",
      "            shell level : 2\r\n",
      "       user config file : /home/aa2133/.condarc\r\n",
      " populated config files : \r\n",
      "          conda version : 4.8.5\r\n",
      "    conda-build version : not installed\r\n",
      "         python version : 3.8.3.final.0\r\n",
      "       virtual packages : __glibc=2.23\r\n",
      "       base environment : /home/aa2133/miniconda3  (writable)\r\n",
      "           channel URLs : https://repo.anaconda.com/pkgs/main/linux-64\r\n",
      "                          https://repo.anaconda.com/pkgs/main/noarch\r\n",
      "                          https://repo.anaconda.com/pkgs/r/linux-64\r\n",
      "                          https://repo.anaconda.com/pkgs/r/noarch\r\n",
      "          package cache : /home/aa2133/miniconda3/pkgs\r\n",
      "                          /home/aa2133/.conda/pkgs\r\n",
      "       envs directories : /home/aa2133/miniconda3/envs\r\n",
      "                          /home/aa2133/.conda/envs\r\n",
      "               platform : linux-64\r\n",
      "             user-agent : conda/4.8.5 requests/2.24.0 CPython/3.8.3 Linux/4.4.0-87-generic ubuntu/16.04.6 glibc/2.23\r\n",
      "                UID:GID : 1020:10003\r\n",
      "             netrc file : None\r\n",
      "           offline mode : False\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "# Allow more than one output\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "!conda info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-22T22:58:52.143194Z",
     "start_time": "2021-12-22T22:58:48.604039Z"
    }
   },
   "outputs": [],
   "source": [
    "import shutil\n",
    "import itertools\n",
    "import copy\n",
    "from MainFunctions import writepickle, openpickle\n",
    "from collections import Counter, namedtuple\n",
    "from rdkit.Chem import BRICS  # For fragmenting\n",
    "from ttictoc import tic, toc\n",
    "from rxnmapper import RXNMapper  # Importing RXNMapper for unsupervised atom mapping\n",
    "import os  # Working with the OS\n",
    "from PIL import Image  # Working with images\n",
    "from IPython.display import SVG  # For SVG support\n",
    "from rdkit.Chem.Draw import rdMolDraw2D  # Drawing 2D molecules/reactions\n",
    "from rdkit.Chem import rdChemReactions  # Reaction processing\n",
    "from rdkit.Chem import Draw  # For drawing molecules/reactions\n",
    "from rdkit.Chem import AllChem  # Overall support\n",
    "from rdkit import Chem  # Importing RDKit\n",
    "from FindFunctionalGroups import identify_functional_groups as IFG\n",
    "from helpCompound import hc_smilesDict, hc_molDict\n",
    "import cairosvg\n",
    "import json\n",
    "from chempy import balance_stoichiometry\n",
    "import io\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import display, Image\n",
    "from rdkit.Chem.Draw import IPythonConsole\n",
    "from rdkit.Chem import RDConfig\n",
    "import cirpy\n",
    "from rdkit.Chem.Fingerprints import FingerprintMols\n",
    "from rdkit.Chem import PeriodicTable, GetPeriodicTable\n",
    "from rdkit.Chem import FunctionalGroups\n",
    "import sys\n",
    "sys.path.append('/home/aa2133/Impurity-Project')\n",
    "sys.path.append('/home/aa2133/Impurity-Project/getCandiRxns')\n",
    "try:\n",
    "    import pickle5 as pickle  # Only if pickle doesn't work\n",
    "except Exception:\n",
    "    import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-22T22:58:56.542863Z",
     "start_time": "2021-12-22T22:58:55.651394Z"
    }
   },
   "outputs": [],
   "source": [
    "# %%writefile FunctionsDB.py\n",
    "# %load ./FunctionsDB.py\n",
    "\n",
    "import shutil\n",
    "import itertools\n",
    "import copy\n",
    "from MainFunctions import writepickle, openpickle\n",
    "from collections import Counter, namedtuple\n",
    "\n",
    "from rdkit import Chem  # Importing RDKit\n",
    "from rdkit.Chem.Draw import rdMolDraw2D  # Drawing 2D molecules/reactions\n",
    "from rdkit.Chem import rdChemReactions  # Reaction processing\n",
    "from rdkit.Chem import Draw  # For drawing molecules/reactions\n",
    "from rdkit.Chem import AllChem  # Overall support\n",
    "from FindFunctionalGroups import identify_functional_groups as IFG\n",
    "\n",
    "import sqlite3\n",
    "import os\n",
    "import dask.delayed as delayed\n",
    "import dask.dataframe as dd\n",
    "import dask.array as da\n",
    "import dask.bag as dba\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "#%% Building basic substance database\n",
    "\n",
    "def info(molfile):\n",
    "    mol = Chem.MolFromMolFile(molfile)\n",
    "    if mol:\n",
    "        Chem.SanitizeMol(mol)\n",
    "        mol.UpdatePropertyCache(strict=False)\n",
    "        smils = Chem.MolToSmiles(mol)\n",
    "        return mol, smils\n",
    "    else:\n",
    "        return molfile\n",
    "\n",
    "\n",
    "def basic(ID, folder):\n",
    "    if str.isdecimal(ID):\n",
    "        molfileaddress = folder+os.sep+ID\n",
    "        try:\n",
    "            res = info(molfileaddress)\n",
    "        except Exception as e:\n",
    "            error = e\n",
    "            compaddrs = {'SubstanceID': int(\n",
    "                ID), 'MolFileAddress': molfileaddress, 'Error': error}\n",
    "        else:\n",
    "            if type(res) == tuple:\n",
    "                smiles = res[1]\n",
    "                compaddrs = {'SubstanceID': int(\n",
    "                    ID), 'MolFileAddress': molfileaddress, 'Smiles': smiles}\n",
    "            else:\n",
    "                error = 'Valence error'\n",
    "                compaddrs = {'SubstanceID': int(\n",
    "                    ID), 'MolFileAddress': molfileaddress, 'Error': error}\n",
    "    else:\n",
    "        compaddrs = {}\n",
    "    return compaddrs\n",
    "\n",
    "\n",
    "def basicgroup(molfilelist, folder):\n",
    "    return [basic(ID, folder) for ID in molfilelist]\n",
    "\n",
    "\n",
    "def substancedblist(folderName, partitions):\n",
    "    dem = os.listdir(folderName)\n",
    "    b = db.from_sequence(dem, npartitions=partitions)\n",
    "    dflist = b.map_partitions(basicgroup, folderName).compute()\n",
    "    return dflist\n",
    "\n",
    "\n",
    "#%% Fragment detection\n",
    "def getCarrierFrags0(smi,expand=1,resFormat='smarts',addHs=True):\n",
    "    \"\"\"\n",
    "    str (smiles), int -> list (str_smiles/smarts) \n",
    "    smi: str, smiles of a compound\n",
    "    size: Level to expand functional group by (eg./ 1 will expand to first degree neighbours, 2 will expand to second degree neighbors etc.)\n",
    "    resFormat: 'smiles' or 'smarts' \n",
    "    addHs: bool, if True H will be consider for generation of substructure, recommend True\n",
    "            otherwise terminal atoms of a molecule are not differentiated with other atoms\n",
    "    this function return list of strings of smarts representing carrier frags with miniSize of size\n",
    "    a carrier frag carries a functional group defined by using Ertl's method\n",
    "    find out the bonds to cut define a cutter to cut out the target fragments based on bonds to cut\n",
    "    1) get a list of function groups using IFG\n",
    "    2) if IFG list empty meaning no functional groups, directly return compound smiles\n",
    "    3) expand to nearest neighbors based on expand value\n",
    "    \"\"\"\n",
    "    mol = Chem.MolFromSmiles(smi)\n",
    "    Chem.SanitizeMol(mol)\n",
    "    mol.UpdatePropertyCache(strict=False)\n",
    "    if addHs:\n",
    "        mol = Chem.AddHs(mol)\n",
    "    # -- get the list of functional groups FG\n",
    "    # e.g., [IFG(atomIds=(1, 4, 7), atoms='NC=O', type='cNC(C)=O'), IFG(atomIds=(10,), atoms='O', type='cO')]\n",
    "    IFG_ls = IFG(mol)\n",
    "    # if IFG_ls is empty, directly return this compounds\n",
    "    if len(IFG_ls) == 0:\n",
    "        if resFormat == 'smiles':\n",
    "            return smi\n",
    "        elif resFormat == 'smarts':\n",
    "            return Chem.MolToSmarts(Chem.MolFromSmiles(smi))\n",
    "    # -- get atomIDs (FGs_atomIDs_expan) and terminalAtomIDs (FGs_terminal_atomIDs) for all frags\n",
    "    FGs_atomIDs = [_.atomIds for _ in IFG_ls]  # e.g., [(1, 4, 7), ...]\n",
    "    n_FGs = len(FGs_atomIDs)\n",
    "    # expan all FGs that < size # e.g., [[1, 4, 7, 8, 9], ...]\n",
    "    FGs_atomIDs_expan = [None]*n_FGs\n",
    "    # terminal atoms: atoms on which bonds to cut will be searched\n",
    "    # [[2, 3, 8], ...] or [[], [], ...], note [2, 3, 8] are terminal of comp not frag\n",
    "    FGs_terminal_atomIDs = [None]*n_FGs\n",
    "    for i in range(n_FGs):\n",
    "#         breakpoint()\n",
    "        # initialization before search and expand fragments\n",
    "        FG_atomIDs = list(FGs_atomIDs[i])  # e.g., [1, 4, 7]\n",
    "        FG_size = len(FG_atomIDs)\n",
    "        FG_terminal_atomIDs = []  # e.g., [2, 3, 8] or [], HNO3 or C=O -> [] Terminal IDs of fragments\n",
    "        for atomID in FG_atomIDs:\n",
    "            neis_IDs = [_.GetIdx() for _ in mol.GetAtomWithIdx(atomID).GetNeighbors()]\n",
    "            if len(set(neis_IDs) - set(FG_atomIDs)) != 0:\n",
    "                FG_terminal_atomIDs = FG_terminal_atomIDs + [atomID]\n",
    "        if len(FG_terminal_atomIDs) == 0:\n",
    "            # make sure all elements in FGs_atomIDs_expan are lists\n",
    "            FGs_atomIDs_expan[i] = FG_atomIDs\n",
    "            FGs_terminal_atomIDs[i] = FG_terminal_atomIDs\n",
    "            # case 3 still could be [] even if FG_size >= size, though less likely\n",
    "        else:\n",
    "            FG_expan_atomIDs = copy.deepcopy(FG_atomIDs)  # e.g., [1, 4, 7]\n",
    "             # max repeat size times, since repeat size should reach the size\n",
    "            for rep in range(expand):\n",
    "                FG_expan_atomIDs_old = copy.deepcopy(FG_expan_atomIDs)\n",
    "                # not all atoms need be searched for neis, only ones not in FG_expan_atomIDs for each epoch\n",
    "                for atomID in FG_terminal_atomIDs:\n",
    "                    neis_IDs = [_.GetIdx() for _ in mol.GetAtomWithIdx(atomID).GetNeighbors()]\n",
    "                    FG_expan_atomIDs = FG_expan_atomIDs + neis_IDs\n",
    "                FG_expan_atomIDs = list(set(FG_expan_atomIDs))\n",
    "                FG_terminal_atomIDs = list(set(FG_expan_atomIDs)-set(FG_expan_atomIDs_old))\n",
    "                FGs_atomIDs_expan[i] = FG_expan_atomIDs\n",
    "                FGs_terminal_atomIDs[i] = FG_terminal_atomIDs  # [] case 2\n",
    "                if len(FG_terminal_atomIDs) == 0:  # cannot expand due to small comp size\n",
    "                    break  # for expan of next FG\n",
    "    # it seems not possible that [[2, 3, 8], [], ...], if there one [] then all should be [],\n",
    "    # either [[]] or [[], [], ...] (i.e., small comp with single or multiple FGs)\n",
    "    # sum([len(_) for _ in FGs_terminal_atomIDs]) == 0:\n",
    "#     breakpoint()\n",
    "    if len(FGs_terminal_atomIDs[0]) == 0:\n",
    "        if resFormat == 'smiles':\n",
    "            return smi\n",
    "        elif resFormat == 'smarts':\n",
    "            return Chem.MolToSmarts(Chem.MolFromSmiles(smi))\n",
    "    else:\n",
    "        FGs_strs = []  # smiles or smarts\n",
    "        for FG_atomIDs in FGs_atomIDs_expan:\n",
    "            if resFormat == 'smiles':\n",
    "                FGs_str = Chem.MolFragmentToSmiles(mol, FG_atomIDs, canonical=True)\n",
    "                FGs_strs = FGs_strs + [FGs_str]\n",
    "            elif resFormat == 'smarts':\n",
    "                FGs_str = Chem.MolFragmentToSmarts(mol, FG_atomIDs, isomericSmarts=False)\n",
    "                FGs_strs = FGs_strs + [FGs_str]\n",
    "        return FGs_strs\n",
    "    \n",
    "    \n",
    "def getCarrierFrags(smi, size, resFormat='smarts', addHs=True):\n",
    "    \"\"\"\n",
    "    str (smiles), int -> list (str_smiles/smarts) \n",
    "    smi: str, smiles of a compound\n",
    "    size: size of carrier fragments, number atoms\n",
    "    resFormat: 'smiles' or 'smarts'\n",
    "    addHs: bool, if True H will be consider for generation of substructure, recommend True\n",
    "            otherwise terminal atoms of a molecule are not differentiated with other atoms\n",
    "    this function return list of strings of smarts representing carrier frags with miniSize of size\n",
    "    a carrier frag carries a functional group defined by using Ertl's method\n",
    "    find out the bonds to cut define a cutter to cut out the target fragments based on bonds to cut\n",
    "    1) get a list of function groups using IFG\n",
    "    2) if IFG list empty meaning no functional groups, directly return compound smiles\n",
    "    3) evaluate the size of current fragment,  \n",
    "       if frag_size >= size, directly return the comp smile\n",
    "       if frag_size < miniSizeFrag, expand the fragment by searching for neighors, only use terminal atoms to search\n",
    "       repeat expansion till frag_size >= size\n",
    "    \"\"\"\n",
    "    mol = Chem.MolFromSmiles(smi)\n",
    "    Chem.SanitizeMol(mol)\n",
    "    mol.UpdatePropertyCache(strict=False)\n",
    "    if addHs:\n",
    "        mol = Chem.AddHs(mol)\n",
    "    # -- get the list of functional groups FG\n",
    "    # e.g., [IFG(atomIds=(1, 4, 7), atoms='NC=O', type='cNC(C)=O'), IFG(atomIds=(10,), atoms='O', type='cO')]\n",
    "    IFG_ls = IFG(mol)\n",
    "    # if IFG_ls is empty, directly return this compounds\n",
    "    if len(IFG_ls) == 0:\n",
    "        if resFormat == 'smiles':\n",
    "            return smi\n",
    "        elif resFormat == 'smarts':\n",
    "            return Chem.MolToSmarts(Chem.MolFromSmiles(smi))\n",
    "    # -- get atomIDs (FGs_atomIDs_expan) and terminalAtomIDs (FGs_terminal_atomIDs) for all frags\n",
    "    FGs_atomIDs = [_.atomIds for _ in IFG_ls]  # e.g., [(1, 4, 7), ...]\n",
    "    n_FGs = len(FGs_atomIDs)\n",
    "    # expan all FGs that < size # e.g., [[1, 4, 7, 8, 9], ...]\n",
    "    FGs_atomIDs_expan = [None]*n_FGs\n",
    "    # terminal atoms: atoms on which bonds to cut will be searched\n",
    "    # [[2, 3, 8], ...] or [[], [], ...], note [2, 3, 8] are terminal of comp not frag\n",
    "    FGs_terminal_atomIDs = [None]*n_FGs\n",
    "    for i in range(n_FGs):\n",
    "        # initialization before search and expand fragments\n",
    "        FG_atomIDs = list(FGs_atomIDs[i])  # e.g., [1, 4, 7]\n",
    "        FG_size = len(FG_atomIDs)\n",
    "        # find initial terminal atoms\n",
    "        # terminal atoms: atoms with neis that not belong to FG_atomIDs\n",
    "        # note terminal atoms here are terminal of fragments not comp\n",
    "        # C=O has no fragment terminal, but both C and O are comp terminals\n",
    "        # there are 3 cases that no fragment terminal ie, FG_terminal_atomIDs = [] one case FG_terminal_atomIDs not empty\n",
    "        # case 1: FG_size < size, and FG_terminal_atomIDs = [], ie., comp too small, no chane to expand e.g., C=O, HCl, HNO3\n",
    "        # case 2: FG_size < size, and FG_terminal_atomIDs = [2,3,8],\n",
    "        #         got chane to expand, but after expand still cannot reach size, e.g., OCC=O\n",
    "        #         eventually FG_terminal_atomIDs = []\n",
    "        # case 3: FG_size >= size, and FG_terminal_atomIDs = [], cos the whole big comp is a functional group\n",
    "        # for all above cases the original comp will be return\n",
    "        # two case FG_terminal_atomIDs is not empty, cutting will performed based on FG_terminal_atomIDs\n",
    "        # case 4: FG_size >= size, and FG_terminal_atomIDs = [2,3,8], just cut using FG_terminal_atomIDs\n",
    "        # case 5: FG_size < size, and FG_terminal_atomIDs = [2,3,8], after expand, size reached and FG_terminal_atomIDs changed\n",
    "        FG_terminal_atomIDs = []  # e.g., [2, 3, 8] or [], HNO3 or C=O -> []\n",
    "        for atomID in FG_atomIDs:\n",
    "            neis_IDs = [_.GetIdx() for _ in mol.GetAtomWithIdx(atomID).GetNeighbors()]\n",
    "            if len(set(neis_IDs) - set(FG_atomIDs)) != 0:\n",
    "                FG_terminal_atomIDs = FG_terminal_atomIDs + [atomID]\n",
    "        # perform search / expand fragments\n",
    "        # [],  case 1 cannot expand, although FG_size < size\n",
    "        if (FG_size >= size) | (len(FG_terminal_atomIDs) == 0):\n",
    "            # make sure all elements in FGs_atomIDs_expan are lists\n",
    "            FGs_atomIDs_expan[i] = FG_atomIDs\n",
    "            FGs_terminal_atomIDs[i] = FG_terminal_atomIDs\n",
    "            # case 3 still could be [] even if FG_size >= size, though less likely\n",
    "        else:\n",
    "            # init FG to be expand\n",
    "            FG_expan_atomIDs = copy.deepcopy(FG_atomIDs)  # e.g., [1, 4, 7]\n",
    "            # max repeat size times, since repeat size should reach the size\n",
    "            for rep in range(size):\n",
    "                FG_expan_atomIDs_old = copy.deepcopy(FG_expan_atomIDs)\n",
    "                # not all atoms need be searched for neis, only ones not in FG_expan_atomIDs for each epoch\n",
    "                for atomID in FG_terminal_atomIDs:\n",
    "                    neis_IDs = [_.GetIdx() for _ in mol.GetAtomWithIdx(atomID).GetNeighbors()]\n",
    "                    FG_expan_atomIDs = FG_expan_atomIDs + neis_IDs\n",
    "                FG_expan_atomIDs = list(set(FG_expan_atomIDs))\n",
    "                FG_terminal_atomIDs = list(set(FG_expan_atomIDs)-set(FG_expan_atomIDs_old))\n",
    "                if len(FG_terminal_atomIDs) == 0:  # cannot expand due to small comp size\n",
    "                    FGs_atomIDs_expan[i] = FG_expan_atomIDs\n",
    "                    FGs_terminal_atomIDs[i] = FG_terminal_atomIDs  # [] case 2\n",
    "                    break  # for expan of next FG\n",
    "                if len(FG_expan_atomIDs) >= size:  # reach the desired size\n",
    "                    FGs_atomIDs_expan[i] = FG_expan_atomIDs\n",
    "                    FGs_terminal_atomIDs[i] = FG_terminal_atomIDs\n",
    "                    break\n",
    "    # it seems not possible that [[2, 3, 8], [], ...], if there one [] then all should be [],\n",
    "    # either [[]] or [[], [], ...] (i.e., small comp with single or multiple FGs)\n",
    "    # sum([len(_) for _ in FGs_terminal_atomIDs]) == 0:\n",
    "    if len(FGs_terminal_atomIDs[0]) == 0:\n",
    "        if resFormat == 'smiles':\n",
    "            return smi\n",
    "        elif resFormat == 'smarts':\n",
    "            return Chem.MolToSmarts(Chem.MolFromSmiles(smi))\n",
    "    else:\n",
    "        FGs_strs = []  # smiles or smarts\n",
    "        for FG_atomIDs in FGs_atomIDs_expan:\n",
    "            if resFormat == 'smiles':\n",
    "                FGs_str = Chem.MolFragmentToSmiles(mol, FG_atomIDs, canonical=True)\n",
    "                FGs_strs = FGs_strs + [FGs_str]\n",
    "            elif resFormat == 'smarts':\n",
    "                FGs_str = Chem.MolFragmentToSmarts(mol, FG_atomIDs, isomericSmarts=False)\n",
    "                FGs_strs = FGs_strs + [FGs_str]\n",
    "        return FGs_strs\n",
    "    \n",
    "#%% Adding fragment smarts column accounting for mixtures\n",
    "def getfrags(series,expand=1): #natoms changed to expand\n",
    "    smiles=series['Smiles']\n",
    "    if smiles=='Error':\n",
    "        return 'Error','Error'\n",
    "    if series['>1 Compound']==True: #This compound is a mixture. Need to split and apply getcarrierfrags to each smiles\n",
    "        fragsmarts=getmixturefrags(smiles,expand=expand)\n",
    "        fragsmiles=getmixturefrags(smiles,expand=expand,resFormat='smiles')\n",
    "    else:\n",
    "        try:\n",
    "            fragsmarts=getCarrierFrags0(smiles,expand=expand)\n",
    "            fragsmiles=getCarrierFrags0(smiles,expand=expand,resFormat='smiles')\n",
    "        except Exception:\n",
    "            return 'Error','Error'\n",
    "    if type(fragsmarts)!=list:\n",
    "        fragsmarts=[fragsmarts]\n",
    "        fragsmiles=[fragsmiles]\n",
    "    \n",
    "#     fragsmiles=[Chem.MolToSmiles(Chem.MolFromSmarts(fragsmart)) for fragsmart in fragsmarts] #Does not capture aromaticity\n",
    "    return fragsmiles,fragsmarts\n",
    "        \n",
    "    \n",
    "def getfragpartition(partition,natoms):\n",
    "    return partition.apply(getfrags,natoms=natoms,axis=1)\n",
    "\n",
    "\n",
    "#%% Adding fragment smiles column\n",
    "def getfragsmiles(fragsmarts):\n",
    "    if type(fragsmarts)==list:\n",
    "        fragsmiles=[Chem.MolToSmiles(Chem.MolFromSmarts(fragsmart)) for fragsmart in fragsmarts]    \n",
    "    elif fragsmarts=='Error':\n",
    "        return 'Error'\n",
    "    else:\n",
    "         fragsmiles=Chem.MolToSmiles(Chem.MolFromSmarts(fragsmarts))\n",
    "    return fragsmiles\n",
    "\n",
    "def getsmiles(series):\n",
    "    fragsmarts=series['FragmentSmarts']\n",
    "    return getfragsmiles(fragsmarts)\n",
    "\n",
    "def getsmilespartition(partition):\n",
    "    return partition.apply(getsmiles,axis=1)\n",
    "\n",
    "#%% Creating mixture column (True if mixture, False if not mixture, Error if smiles not present)\n",
    "\n",
    "def mixtures(smiles):\n",
    "    if smiles=='Error':\n",
    "        return 'Error'\n",
    "    elif len(smiles.split('.'))>1:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def findMixtures(series):\n",
    "    smiles=series['Smiles']\n",
    "    return mixtures(smiles)\n",
    "    \n",
    "def findMixturespartition(partition):\n",
    "       return partition.apply(findMixtures,axis=1)\n",
    "\n",
    "#%% Changing fragment entries in mixture rows\n",
    "\n",
    "def getmixturefrags(mixsmiles,expand=1,resFormat='smarts', addHs=True): \n",
    "    try:\n",
    "        res=[]\n",
    "        for smiles in mixsmiles.split('.'):\n",
    "            reslist=getCarrierFrags0(smiles,expand=expand,resFormat=resFormat,addHs=addHs)\n",
    "            if type(reslist)!=list:\n",
    "                res+=[reslist]\n",
    "            else:\n",
    "                res+=reslist\n",
    "    except Exception:\n",
    "        return 'Error'\n",
    "    else:\n",
    "        return res    \n",
    "\n",
    "def getMixturefrags(series,expand=1):\n",
    "    mixsmiles=series['Smiles'] # add .values[0] if column is a multiindex, otherwise droplevel = 1 to remove list\n",
    "    return getmixturefrags(mixsmiles,expand=expand)  \n",
    "\n",
    "def getMixturefragspartition(partition,natoms):\n",
    "    return partition.apply(getMixturefrags,expand=expand,axis=1)\n",
    "\n",
    "# def collapsepartition(partition): #Doesn't work yet. Once exploded it is extremely time-consuming to collapse the dataframe\n",
    "#     temp=partition.groupby(['Smiles']).agg([list])\n",
    "#     return temp\n",
    "\n",
    "    \n",
    "#%% Joining columns to a dataframe\n",
    "\n",
    "def joindf(seriesdf,DB,explodeDB=None):\n",
    "    if seriesdf.index.name!=DB.index.name or seriesdf.index.names!=DB.index.names:\n",
    "        if DB.index.name or DB.index.names:\n",
    "            DB.reset_index(inplace=True)\n",
    "        if seriesdf.index.name or seriesdf.index.names:\n",
    "            seriesdf.reset_index(inplace=True)        \n",
    "    DB=DB.join(seriesdf)\n",
    "    if explodeDB:\n",
    "        DB=DB.explode(explodeDB)\n",
    "    return DB\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def buildfragdb(sdb=None,sdbd=None,writesdbd=False, \\\n",
    "                sdbdc=None,fragseries=None,natoms=None, \\\n",
    "                writefragseries=False, fdbm=None, fdb=None, dfdb=None, \\\n",
    "                fragsmiles=False, mixtures=False, mixturefrags=False, \\\n",
    "                index=None, writefdb=False,writedfdb=False):\n",
    "    \n",
    "    # Note: cluster and client must be initiated for this function to work. Substance database, either dask or pandas\n",
    "    # should also be loaded. It is advised to avoid loading dask dataframes due to high memory usage. It is recommended to\n",
    "    #persist and create dask dataframes outside this function, otherwise overheads may be added. Never reset dask dataframe\n",
    "    #index or else it will reindex all partitions...always start with pandas, reindex and then convert to dask when large computations\n",
    "    #need to be done.\n",
    "    \n",
    "    # sdb = substance database, sdbd = substance database dask, sdbdc = substance database dask cleaned \n",
    "    #(remove smile errors), # fragseries = series containing fragment information, natoms =  size of fragment, \n",
    "    # writefragseries = True if write to file false otherwise, fdbm = fragment database master, pandas version of sdbdc,\n",
    "    # fdb = final fragment database exploded and unindexed pfdb = final pandas fragment dataframe, multiindexed, \n",
    "    # dfdb = final dask fragment dataframe\n",
    "   \n",
    "    # Step 1: Create dask dataframe from pandas substance database (output: sdbd)\n",
    "    \n",
    "    if sdb and not type(sdbd)==dd.core.DataFrame:\n",
    "        sdbd = dd.from_pandas(sdb, npartitions=16)\n",
    "        sdbd = client.persist(sdbd)\n",
    "        print('Dask substance dataframe created and persisted')\n",
    "        if writesdbd:\n",
    "            sdbd.to_parquet(writesdbd)\n",
    "            print('Dask substance dataframe written to file: ' + writesdb)\n",
    "        if not natoms and not fragseries:\n",
    "            print(\"Please specify size of fragments that should be retrieved\")\n",
    "            return sdbd\n",
    "        \n",
    "   # Step 2: Clean dask data frame and select only substance ID and smiles column, removing errors (output: sdbdc).\n",
    "\n",
    "    if sdbd and not type(sdbdc)==dd.core.DataFrame:\n",
    "        sdbd = client.persist(sdbd)\n",
    "        sdbdc = sdbd.reset_index()[['SubstanceID', 'Smiles','>1 Compound']]\n",
    "        sdbdc= sdbdc[sdbdc.Smiles!='Error']\n",
    "        sdbdc=client.persist(sdbdc)\n",
    "        print('Cleaned dask substance dataframe created and persisted')\n",
    "        if not natoms and not type(fragseries)==pd.core.series.Series:\n",
    "            print(\"Please specify size of fragments that should be retrieved\")\n",
    "            return sdbdc      \n",
    "        \n",
    "    # Step 3: Scrape 16 million compounds, and extract series of active fragments for each (output: fragseries)\n",
    "    \n",
    "    if not type(fragseries)==pd.core.series.Series and not type(fdb)==pd.core.frame.DataFrame:\n",
    "        if not type(sdbdc)==dd.core.DataFrame:\n",
    "            return \"Please include a cleaned dask substance dataframe for fragment retrieval\"\n",
    "        if not natoms:\n",
    "            return \"Please specify size of fragments retrieved\"\n",
    "        sdbdc=client.persist(sdbdc)\n",
    "#         if natoms==0:\n",
    "#             name='ActiveFragmentSmarts'\n",
    "#         else:\n",
    "#            filename='CarrierFragments'+'(n='+str(natoms)+')'\n",
    "        name='FragmentSmarts'\n",
    "        fragseries=sdbdc.map_partitions(getfragpartition,natoms=natoms,meta=(name,'O')).compute()\n",
    "        print('fragseries retrieved')\n",
    "    \n",
    "    if writefragseries:\n",
    "        if not type(fragseries)==pd.core.series.Series:\n",
    "            return \"Supply fragment series to write to file\"\n",
    "        writepickle(fragseries,writefragseries)\n",
    "        print('fragseries writted to file: '+ writefragseries)\n",
    "      \n",
    "     # Step 4: Prepare cleaned pandas dataframe for fragment series attachment (output: fdbm)  \n",
    "        \n",
    "    if not type(fdbm)==pd.core.frame.DataFrame and not type(fdb)==pd.core.frame.DataFrame:\n",
    "        if not type(sdbdc)==dd.core.DataFrame:\n",
    "            return \"Please include a cleaned dask substance dataframe to which fragment information can be attached\"\n",
    "        fdbm=sdbdc.compute()\n",
    "#         fdbm.reset_index(inplace=True)\n",
    "#         fdbm.drop('index',axis=1,inplace=True)\n",
    "        \n",
    "    # Step 5: Attaching fragment series, generating an exploded fragment database (output: fdb)\n",
    "    \n",
    "    if type(fdbm)==pd.core.frame.DataFrame:\n",
    "        if not type(fragseries)==pd.core.series.Series:\n",
    "            return \"Please include the fragment series that should be attached\"\n",
    "        fragdf=pd.DataFrame(fragseries)\n",
    "        fdb=joindf(fragdf,fdbm,explodeDB=fragdf.columns[0])\n",
    "        print(\"Unindexed fragment database completed\")\n",
    "    \n",
    "    # Step 6: Adding additional columns, formatting, indexing and writing fragment database to file\n",
    "\n",
    "    if fragsmiles:\n",
    "        if not type(fdb)==pd.core.frame.DataFrame:\n",
    "            return \"Please supply fragment database to analyze\"\n",
    "        if not type(dfdb)==dd.core.DataFrame:\n",
    "            if fdb.index.name or fdb.index.names:\n",
    "                fdb.reset_index(inplace=True)\n",
    "            dfdb=dd.from_pandas(fdb,npartitions=181)\n",
    "            dfdb=client.persist(dfdb)\n",
    "        fragseries=dfdb.map_partitions(getsmilespartition,meta=('FragmentSmiles','O')).compute()\n",
    "        if writefragseries:\n",
    "            if not type(fragseries)==pd.core.series.Series:\n",
    "                return \"Supply fragment series to write to file\"\n",
    "            writepickle(fragseries,writefragseries)\n",
    "            print('fragseries writted to file: '+ writefragseries)    \n",
    "        smilesdf=pd.DataFrame(fragseries)\n",
    "        fdb=joindf(smilesdf,fdb)\n",
    "        \n",
    "        \n",
    "    if mixtures:\n",
    "        if not type(fdb)==pd.core.frame.DataFrame:\n",
    "            return \"Please supply fragment database to analyze\"\n",
    "        if not type(dfdb)==dd.core.DataFrame:\n",
    "            if fdb.index.name or fdb.index.names:\n",
    "                fdb.reset_index(inplace=True)\n",
    "            dfdb=dd.from_pandas(fdb,npartitions=181)\n",
    "            dfdb=client.persist(dfdb)\n",
    "        fragseries=dfdb.map_partitions(findMixturespartition,meta=('>1 Compound','boolean'))\n",
    "        if writefragseries:\n",
    "            if not type(fragseries)==pd.core.series.Series:\n",
    "                return \"Supply fragment series to write to file\"\n",
    "            writepickle(fragseries,writefragseries)\n",
    "            print('fragseries writted to file: '+ writefragseries) \n",
    "        mixturedf=pd.DataFrame(fragseries)\n",
    "        fdb=joindf(mixturedf,fdb)\n",
    "    \n",
    "    if mixturefrags:\n",
    "        #NOTE: If exploded, fragment database should be aggregated by Smiles/Smarts\n",
    "        if not type(fdb)==pd.core.frame.DataFrame or '> 1 Compound' not in fdb.columns:\n",
    "            return \"Please supply fragment database to analyze, with mixture indication (specify mixtures=True)\"\n",
    "        if not natoms:\n",
    "            return \"Please specify size of fragments retrieved\"\n",
    "        if not type(dfdb)==dd.core.DataFrame:\n",
    "            dfdb=dd.from_pandas(fdb,npartitions=16)\n",
    "            dfdb=client.persist(dfdb)\n",
    "        fragseries=dfdb.map_partitions(getMixturefragspartition,natoms=natoms,meta=('FragmentSmarts','O'))\n",
    "        if writefragseries:\n",
    "            if not type(fragseries)==pd.core.series.Series:\n",
    "                return \"Supply fragment series to write to file\"\n",
    "            writepickle(fragseries,writefragseries)\n",
    "            print('fragseries writted to file: '+ writefragseries) \n",
    "        mixturesmarts=pd.DataFrame(fragseries)\n",
    "        fdb=joindf(mixturesmarts,fdb)\n",
    "        \n",
    "        \n",
    "    \n",
    "# Deprecated..Dask dataframes take up too much memory when read from file\n",
    "\n",
    "#     if dfdb:\n",
    "#         if fdb.index.name:\n",
    "#             fdb.reset_index(inplace=True)\n",
    "#         fdb.set_index('SubstanceID',inplace=True)\n",
    "#         dfdb=dd.from_pandas(fdb,npartitions=64)\n",
    "# #         dfdb=dfdb.reset_index()\n",
    "# #         dfdb=dfdb.set_index(dfdb.columns[dfdb.columns.str.contains('ragment')][0])\n",
    "#         print('Dask fragment database created')\n",
    "#     if writedfdb:\n",
    "#         dfdb.to_parquet(writedfdb)\n",
    "#         print('Dask fragment database written to file: ' + writedfdb)\n",
    "\n",
    "        \n",
    "    #%% Indexing dataframe\n",
    "    \n",
    "    if index:\n",
    "        if not type(fdb)==pd.core.frame.DataFrame:\n",
    "             return \"Please supply fragment database to index\"\n",
    "        if fdb.index.name==index or fdb.index.names==index:\n",
    "            print(\"Database is already indexed.\") \n",
    "        elif fdb.index.name or fdb.index.names:\n",
    "            fdb.reset_index(inplace=True)\n",
    "            fdb.set_index([index],inplace=True)\n",
    "        else:\n",
    "            fdb.set_index([index],inplace=True)\n",
    "        print('Pandas fragment database indexed')\n",
    "    if writefdb:\n",
    "        writepickle(fdb,writefdb)\n",
    "        print('Pandas fragment database written to file: ' + writefdb)\n",
    "    \n",
    "    return fdb\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing fragment database"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is just for Reaxys. For pistachio, curl/json requests need to be processed for each reaction ID (has not been implemented yet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:25:49.357982Z",
     "start_time": "2021-12-03T02:25:34.682703Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb = openpickle(\"/home/aa2133/Impurity-Project/Reaxys_Data/SubstanceDB.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:25:54.578352Z",
     "start_time": "2021-12-03T02:25:54.543297Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>MolFileAddress</th>\n",
       "      <th>Error</th>\n",
       "      <th>&gt;1 Compound</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SubstanceID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6890270</th>\n",
       "      <td>CC(C)(C)[C@@H]1CC[C@@](C)(F)[C@H](Br)C1</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/6890270</td>\n",
       "      <td>No</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27615522</th>\n",
       "      <td>Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/27615522</td>\n",
       "      <td>No</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29164535</th>\n",
       "      <td>[Gd+3].[Na+].[O-]B([O-])[O-].[Tb+3]</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/29164535</td>\n",
       "      <td>No</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11230426</th>\n",
       "      <td>Nc1cc(F)c(N2CC(F)C2)c(F)c1</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/11230426</td>\n",
       "      <td>No</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9019791</th>\n",
       "      <td>CCCCOc1ccc(-c2nc(-c3ccc(Cl)cc3)[nH]c2-c2ccncc2...</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/9019791</td>\n",
       "      <td>No</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26761831</th>\n",
       "      <td>CCOC(=O)C(CC)(CC)P(=O)(OCC)OCC</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/26761831</td>\n",
       "      <td>No</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3409746</th>\n",
       "      <td>N#CC(COC(=O)c1ccccc1)OC(=O)c1ccccc1</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/3409746</td>\n",
       "      <td>No</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6138726</th>\n",
       "      <td>CN(O)C(CCC(C)(C)O)C1=CCCCC1</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/6138726</td>\n",
       "      <td>No</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22727365</th>\n",
       "      <td>CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O...</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/22727365</td>\n",
       "      <td>No</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/11404182</td>\n",
       "      <td>No</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16129451 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        Smiles  \\\n",
       "SubstanceID                                                      \n",
       "6890270                CC(C)(C)[C@@H]1CC[C@@](C)(F)[C@H](Br)C1   \n",
       "27615522           Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F   \n",
       "29164535                   [Gd+3].[Na+].[O-]B([O-])[O-].[Tb+3]   \n",
       "11230426                            Nc1cc(F)c(N2CC(F)C2)c(F)c1   \n",
       "9019791      CCCCOc1ccc(-c2nc(-c3ccc(Cl)cc3)[nH]c2-c2ccncc2...   \n",
       "...                                                        ...   \n",
       "26761831                        CCOC(=O)C(CC)(CC)P(=O)(OCC)OCC   \n",
       "3409746                    N#CC(COC(=O)c1ccccc1)OC(=O)c1ccccc1   \n",
       "6138726                            CN(O)C(CCC(C)(C)O)C1=CCCCC1   \n",
       "22727365     CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O...   \n",
       "11404182                   COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1   \n",
       "\n",
       "                                                MolFileAddress Error  \\\n",
       "SubstanceID                                                            \n",
       "6890270        /home/projects/graph/data/batch_6500000/6890270    No   \n",
       "27615522      /home/projects/graph/data/batch_6500000/27615522    No   \n",
       "29164535      /home/projects/graph/data/batch_6500000/29164535    No   \n",
       "11230426      /home/projects/graph/data/batch_6500000/11230426    No   \n",
       "9019791        /home/projects/graph/data/batch_6500000/9019791    No   \n",
       "...                                                        ...   ...   \n",
       "26761831     /home/projects/graph/data/batch_12600000/26761831    No   \n",
       "3409746       /home/projects/graph/data/batch_12600000/3409746    No   \n",
       "6138726       /home/projects/graph/data/batch_12600000/6138726    No   \n",
       "22727365     /home/projects/graph/data/batch_12600000/22727365    No   \n",
       "11404182     /home/projects/graph/data/batch_12600000/11404182    No   \n",
       "\n",
       "            >1 Compound  \n",
       "SubstanceID              \n",
       "6890270           False  \n",
       "27615522          False  \n",
       "29164535           True  \n",
       "11230426          False  \n",
       "9019791           False  \n",
       "...                 ...  \n",
       "26761831          False  \n",
       "3409746           False  \n",
       "6138726           False  \n",
       "22727365           True  \n",
       "11404182          False  \n",
       "\n",
       "[16129451 rows x 4 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "substancedb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-05T12:48:20.466316Z",
     "start_time": "2021-05-05T12:48:16.591238Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb.reset_index(inplace=True)\n",
    "substancedb.set_index('SubstanceID',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T06:29:48.657397Z",
     "start_time": "2021-05-07T06:29:45.992225Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-07 07:29:47,420\tINFO services.py:1171 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'node_ip_address': '172.24.144.187',\n",
       " 'raylet_ip_address': '172.24.144.187',\n",
       " 'redis_address': '172.24.144.187:6379',\n",
       " 'object_store_address': '/tmp/ray/session_2021-05-07_07-29-45_995432_3147/sockets/plasma_store',\n",
       " 'raylet_socket_name': '/tmp/ray/session_2021-05-07_07-29-45_995432_3147/sockets/raylet',\n",
       " 'webui_url': '127.0.0.1:8265',\n",
       " 'session_dir': '/tmp/ray/session_2021-05-07_07-29-45_995432_3147',\n",
       " 'metrics_export_port': 48993,\n",
       " 'node_id': '81e091672a7c14fb5a9abf3fe95843e810ddef02'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UserWarning: The pandas version installed 1.2.4 does not match the supported pandas version in Modin 1.2.3. This may cause undesired side effects!\n"
     ]
    }
   ],
   "source": [
    "import ray\n",
    "ray.init(num_cpus=16,log_to_driver=False)\n",
    "# ray.init(address=<address>)\n",
    "# import modin.pandas as mpd\n",
    "#{'node_ip_address': '172.24.144.187',\n",
    "#  'raylet_ip_address': '172.24.144.187',\n",
    "#  'redis_address': '172.24.144.187:6379',\n",
    "#  'object_store_address': '/tmp/ray/session_2021-03-17_08-06-30_976889_20598/sockets/plasma_store',\n",
    "#  'raylet_socket_name': '/tmp/ray/session_2021-03-17_08-06-30_976889_20598/sockets/raylet',\n",
    "#  'webui_url': '127.0.0.1:8265',\n",
    "#  'session_dir': '/tmp/ray/session_2021-03-17_08-06-30_976889_20598',\n",
    "#  'metrics_export_port': 62625,\n",
    "#  'node_id': '614e0285af16d747aad7f68221d8e1670dca25a1'}\n",
    "import modin.pandas as mpd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T09:08:12.383893Z",
     "start_time": "2021-05-06T09:08:09.420915Z"
    }
   },
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-22T22:59:12.262015Z",
     "start_time": "2021-12-22T22:59:12.254639Z"
    }
   },
   "outputs": [],
   "source": [
    "import ray\n",
    "def initray(restart=True,num_cpus=16,log_to_driver=False):\n",
    "    if restart:\n",
    "        ray.shutdown()\n",
    "    ray.init(num_cpus=num_cpus,log_to_driver=log_to_driver)\n",
    "    import modin.pandas as mpd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-05T19:12:39.697418Z",
     "start_time": "2021-05-05T19:12:11.606528Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbr=mpd.DataFrame(substancedb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-14T17:44:42.014619Z",
     "start_time": "2021-05-14T17:44:41.997775Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cOC', 'CC#N']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getCarrierFrags0('COc2ccc1cccc(CC#N)c1c2',resFormat='smiles')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-05T12:51:09.704370Z",
     "start_time": "2021-05-05T12:51:09.638703Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('c1ccncc1.c1nc[nH]n1')\n",
    "IFG(Chem.MolFromSmiles('c1ccncc1'))\n",
    "IFG(Chem.MolFromSmiles('c1nc[nH]n1'))\n",
    "getmixturefrags('c1ccncc1.c1nc[nH]n1',expand=1)\n",
    "getmixturefrags('c1ccncc1.c1nc[nH]n1',expand=2)\n",
    "Chem.MolFromSmarts('[#6]:[#7]:[#7](:[#6])-[H]')\n",
    "Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7](:[#7]:1)-[H])-[H])-[H]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-05T17:32:05.951043Z",
     "start_time": "2021-05-05T17:32:05.901250Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags0('C1=CC=C(C=C1)O',expand=1,resFormat='smiles')\n",
    "getCarrierFrags0('CCO',expand=1,resFormat='smiles')\n",
    "getCarrierFrags0('C1=CC=C(C=C1)O',expand=1,resFormat='smarts')\n",
    "getCarrierFrags0('CCO',expand=1,resFormat='smarts')\n",
    "#Smarts can be used to display fragment, smiles indicates aromaticity but is not chemically valid (as it is a fragment)\n",
    "from MainFunctions import mol_with_atom_index\n",
    "mol_with_atom_index(Chem.AddHs(Chem.MolFromSmiles('C1=CC=C(C=C1)O')))\n",
    "Chem.MolFragmentToSmarts(Chem.AddHs(Chem.MolFromSmiles('C1=CC=C(C=C1)O')),[6,12,3,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-05T17:18:56.282483Z",
     "start_time": "2021-05-05T17:18:56.259403Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('cOC',sanitize=False)\n",
    "# Chem.SanitizeMol(m,sanitizeOps=Chem.SanitizeFlags.SANITIZE_ALL^Chem.SanitizeFlags.SANITIZE_KEKULIZE)\n",
    "# display(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-05T12:57:56.739700Z",
     "start_time": "2021-05-05T12:57:56.696060Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb.iloc[1000]\n",
    "IFG(Chem.MolFromSmiles('COc1cccc2c1OCC(c1ccccc1)C2'))\n",
    "getfrags(substancedb.iloc[1000],expand=2)\n",
    "getfrags(substancedb.iloc[1000])\n",
    "Chem.MolFromSmarts('[#6]-[#8]-[#6]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-05T19:12:02.222308Z",
     "start_time": "2021-05-05T19:11:55.528140Z"
    }
   },
   "outputs": [],
   "source": [
    "initray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T09:09:04.824648Z",
     "start_time": "2021-05-06T09:09:04.813113Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# substancedb.loc[4122952].Smiles='[H-].[Na+]' #Correcting sodium hydride smiles\n",
    "# 16934069\tO.[Hg]\t/home/projects/graph/data/batch_600000/16934069\tNo\tTrue\n",
    "# 23018147\tO.[Hg]\t/home/projects/graph/data/batch_2900000/23018147\tNo\tTrue\n",
    "# 16689882\tO.[Hg]\t/home/projects/graph/data/batch_1300000/16689882\tNo\tTrue\n",
    "# 16798713\tO.[Hg]\t/home/projects/graph/data/batch_2000000/16798713\tNo\tTrue\n",
    "# 16532899\tO.[Hg]\t/home/projects/graph/data/batch_5600000/16532899\tNo\tTrue\n",
    "# 8128139\tO.[Hg]\t/home/projects/graph/data/batch_16000000/8128139\tNo\tTrue\n",
    "# 16502414\tO.[Hg]\t/home/projects/graph/data/batch_7600000/16502414\tNo\tTrue\n",
    "# 23514369\tO.[Hg]\t/home/projects/graph/data/batch_6400000/23514369\tNo\tTrue\n",
    "# 16981445\t\n",
    "# for index,row in substancedb.loc[substancedb.Smiles=='O.[Hg]'].iterrows():\n",
    "#     substancedb.loc[index].Smiles='O=[Hg]'\n",
    "# for index,row in substancedb.loc[substancedb.Smiles=='O=[Hg]'].iterrows():\n",
    "#     substancedb.loc[index]['>1 Compound']=False\n",
    "\n",
    "# substancedb.loc[substancedb.Smiles=='[H].[NaH]']\n",
    "# substancedb.loc[15489681].Smiles='[2H-].[Na+]'\n",
    "# substancedb.loc[16497959].Smiles='[H-].[Na+]'\n",
    "# substancedb.loc[16497075].Smiles='[Na]'\n",
    "# substancedb.loc[substancedb.Smiles=='[Na-][H]']\n",
    "# substancedb.loc[28525100].Smiles='[I-].[K+]'\n",
    "# substancedb.loc[8127929].Smiles='[I-].[K+]'\n",
    "# substancedb.loc[20571218].Smiles='[129I-].[K+]'\n",
    "# substancedb.loc[19231183].Smiles='[125I-].[K+]'\n",
    "# substancedb.loc[16769113].Smiles='[I-].[42K+]'\n",
    "# substancedb.loc[16628792].Smiles='[I-].[K+]'\n",
    "# substancedb.loc[16400527].Smiles='[127I-].[K+]'\n",
    "# substancedb.loc[substancedb.Smiles=='I.[KH]']\n",
    "# substancedb.loc[8127930]\n",
    "# substancedb.loc[31519620].Smiles='[123I-].[Na+]'\n",
    "# substancedb.loc[14344914].Smiles='[125I-].[Na+]'\n",
    "# substancedb.loc[14445085].Smiles='[131I-].[Na+]'\n",
    "# substancedb.loc[16857217].Smiles='[129I-].[Na+]'\n",
    "# substancedb.loc[14341279].Smiles='[123I-].[Na+]'\n",
    "# substancedb.loc[8127930].Smiles='[I-].[Na+]'\n",
    "# substancedb.loc[16855705].Smiles='[138I-].[Na+]'\n",
    "# substancedb.loc[16769306].Smiles='[I-].[22Na+]'\n",
    "# substancedb.loc[16497432].Smiles='[I-].[Na+]'\n",
    "# substancedb.loc[27847271].Smiles='[I-].[Na+]'\n",
    "# substancedb.loc[23094471].Smiles='[125I-].[Na+]'\n",
    "# substancedb.loc[14750800].Smiles='[124I-].[Na+]'\n",
    "# substancedb.loc[14364494].Smiles='[127I-].[Na+]'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-05T21:31:53.062610Z",
     "start_time": "2021-05-05T19:13:18.786324Z"
    }
   },
   "outputs": [],
   "source": [
    "fragdat=substancedbr.apply(getfrags,axis=1,result_type='reduce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T05:10:51.277233Z",
     "start_time": "2021-05-06T05:05:10.209411Z"
    }
   },
   "outputs": [],
   "source": [
    "fragdat=pd.Series(data=fragdat.values,index=fragdat.index) #Optional convert modin back to pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T05:13:38.513518Z",
     "start_time": "2021-05-06T05:11:52.272669Z"
    }
   },
   "outputs": [],
   "source": [
    "writepickle(fragdat,'/home/aa2133/Impurity-Project/Reaxys_Data/','fragdat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:45:10.634319Z",
     "start_time": "2021-12-02T23:43:22.433904Z"
    }
   },
   "outputs": [],
   "source": [
    "fragdat=openpickle('/home/aa2133/Impurity-Project/Reaxys_Data/fragdat.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:45:22.890884Z",
     "start_time": "2021-12-02T23:45:10.638803Z"
    }
   },
   "outputs": [],
   "source": [
    "fragdatdf=pd.DataFrame(data=fragdat.tolist(),index=fragdat.index,columns=['FragmentSmiles','FragmentSmarts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:45:22.924109Z",
     "start_time": "2021-12-02T23:45:22.895301Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th>FragmentSmarts</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SubstanceID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6890270</th>\n",
       "      <td>[CF, CBr]</td>\n",
       "      <td>[[#9]-[#6], [#35]-[#6]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27615522</th>\n",
       "      <td>[cnc, cF, [H]C1([H])OccO1, cF]</td>\n",
       "      <td>[[#6]:[#7]:[#6], [#6]-[#9], [#6]1:[#6]-[#8]-[#...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29164535</th>\n",
       "      <td>[[Gd+3], [Na+], [O-]B([O-])[O-], [Tb+3]]</td>\n",
       "      <td>[[Gd+3], [Na+], [#8-]-[#5](-[#8-])-[#8-], [Tb+3]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11230426</th>\n",
       "      <td>[[H]N([H])c, cF, cN(C)C, CF, cF]</td>\n",
       "      <td>[[#7](-[#6])(-[H])-[H], [#6]-[#9], [#6]-[#7](-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9019791</th>\n",
       "      <td>[cOC, cnc, cCl, [H]n(c)c, cnc, cOC]</td>\n",
       "      <td>[[#6]-[#8]-[#6], [#6]:[#7]:[#6], [#6]-[#17], [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26761831</th>\n",
       "      <td>[COC(C)=O, COP(C)(=O)OC]</td>\n",
       "      <td>[[#6]-[#8]-[#6](=[#8])-[#6], [#6]-[#15](=[#8])...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3409746</th>\n",
       "      <td>[CC#N, cC(=O)OC, cC(=O)OC]</td>\n",
       "      <td>[[#7]#[#6]-[#6], [#6]-[#8]-[#6](=[#8])-[#6], [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6138726</th>\n",
       "      <td>[[H]ON(C)C, [H]OC, [H]C(C)=C(C)C]</td>\n",
       "      <td>[[#6]-[#7](-[#8]-[H])-[#6], [#6]-[#8]-[H], [#6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22727365</th>\n",
       "      <td>[cn(c)C(=O)[O-], [Li+]]</td>\n",
       "      <td>[[#6]:[#7](:[#6])-[#6](=[#8])-[#8-], [Li+]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>[cOC, CCl, cOC, CF, CF, CF]</td>\n",
       "      <td>[[#6]-[#8]-[#6], [#6]-[#17], [#6]-[#8]-[#6], [...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16129451 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       FragmentSmiles  \\\n",
       "SubstanceID                                             \n",
       "6890270                                     [CF, CBr]   \n",
       "27615522               [cnc, cF, [H]C1([H])OccO1, cF]   \n",
       "29164535     [[Gd+3], [Na+], [O-]B([O-])[O-], [Tb+3]]   \n",
       "11230426             [[H]N([H])c, cF, cN(C)C, CF, cF]   \n",
       "9019791           [cOC, cnc, cCl, [H]n(c)c, cnc, cOC]   \n",
       "...                                               ...   \n",
       "26761831                     [COC(C)=O, COP(C)(=O)OC]   \n",
       "3409746                    [CC#N, cC(=O)OC, cC(=O)OC]   \n",
       "6138726             [[H]ON(C)C, [H]OC, [H]C(C)=C(C)C]   \n",
       "22727365                      [cn(c)C(=O)[O-], [Li+]]   \n",
       "11404182                  [cOC, CCl, cOC, CF, CF, CF]   \n",
       "\n",
       "                                                FragmentSmarts  \n",
       "SubstanceID                                                     \n",
       "6890270                                [[#9]-[#6], [#35]-[#6]]  \n",
       "27615522     [[#6]:[#7]:[#6], [#6]-[#9], [#6]1:[#6]-[#8]-[#...  \n",
       "29164535     [[Gd+3], [Na+], [#8-]-[#5](-[#8-])-[#8-], [Tb+3]]  \n",
       "11230426     [[#7](-[#6])(-[H])-[H], [#6]-[#9], [#6]-[#7](-...  \n",
       "9019791      [[#6]-[#8]-[#6], [#6]:[#7]:[#6], [#6]-[#17], [...  \n",
       "...                                                        ...  \n",
       "26761831     [[#6]-[#8]-[#6](=[#8])-[#6], [#6]-[#15](=[#8])...  \n",
       "3409746      [[#7]#[#6]-[#6], [#6]-[#8]-[#6](=[#8])-[#6], [...  \n",
       "6138726      [[#6]-[#7](-[#8]-[H])-[#6], [#6]-[#8]-[H], [#6...  \n",
       "22727365           [[#6]:[#7](:[#6])-[#6](=[#8])-[#8-], [Li+]]  \n",
       "11404182     [[#6]-[#8]-[#6], [#6]-[#17], [#6]-[#8]-[#6], [...  \n",
       "\n",
       "[16129451 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fragdatdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:45:27.141223Z",
     "start_time": "2021-12-02T23:45:22.925921Z"
    }
   },
   "outputs": [],
   "source": [
    "fragdatdf=fragdatdf.loc[fragdatdf.FragmentSmiles!='Error']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T06:42:18.517493Z",
     "start_time": "2021-05-07T06:42:03.330995Z"
    }
   },
   "outputs": [],
   "source": [
    "# fragdatdf\n",
    "# fragdatdf['Smiles']=substancedb.loc[substancedb.index.isin(fragdatdf.index)]['Smiles']\n",
    "# fragdatdf\n",
    "# fragdatdf['>1 Compound']=substancedb[substancedb.index.isin(fragdatdf.index)]['>1 Compound']\n",
    "# fragdatdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T10:07:04.391980Z",
     "start_time": "2021-05-06T10:06:58.332869Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index              118786200\n",
       "FragmentSmiles    1664796272\n",
       "FragmentSmarts    1664796272\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fragdatdf.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-08T18:42:07.682860Z",
     "start_time": "2021-06-08T18:42:07.676440Z"
    }
   },
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "import os\n",
    "\n",
    "db=sqlite3.connect(\"/home/aa2133/Impurity-Project/Reaxys_Data/SQL/Reaxys_Data.db\")\n",
    "cursor = db.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T08:05:17.444267Z",
     "start_time": "2021-05-07T08:05:17.437447Z"
    }
   },
   "outputs": [],
   "source": [
    "db.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2021-05-06T09:31:01.730Z"
    }
   },
   "outputs": [],
   "source": [
    "# pd.read_sql_query('''SELECT Smiles,\">1 Compound\"\n",
    "# FROM SubstanceDB\n",
    "# WHERE SubstanceID IN (6890270,6138726,11404182,3409746);''',db)\n",
    "\n",
    "# pd.read_sql_query('''SELECT SubstanceID,Smiles,\">1 Compound\"\n",
    "# FROM SubstanceDB\n",
    "# WHERE SubstanceID IN {};'''.format(plist),db)\n",
    "\n",
    "# # plist=(6890270,6138726,11404182,3409746)\n",
    "# plist=tempidx[:100000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:46:05.261192Z",
     "start_time": "2021-12-02T23:45:27.145168Z"
    }
   },
   "outputs": [],
   "source": [
    "temp=fragdatdf[['FragmentSmiles']].explode('FragmentSmiles')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:47:10.897461Z",
     "start_time": "2021-12-02T23:46:05.266789Z"
    }
   },
   "outputs": [],
   "source": [
    "temp['count']=temp.groupby([temp.index,'FragmentSmiles'])['FragmentSmiles'].transform('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T18:10:12.816802Z",
     "start_time": "2021-05-06T18:10:12.763246Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SubstanceID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6890270</th>\n",
       "      <td>CF</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6890270</th>\n",
       "      <td>CBr</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27615522</th>\n",
       "      <td>cnc</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27615522</th>\n",
       "      <td>cF</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27615522</th>\n",
       "      <td>[H]C1([H])OccO1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>CCl</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>cOC</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>CF</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>CF</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>CF</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68240841 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              FragmentSmiles  count\n",
       "SubstanceID                        \n",
       "6890270                   CF      1\n",
       "6890270                  CBr      1\n",
       "27615522                 cnc      1\n",
       "27615522                  cF      2\n",
       "27615522     [H]C1([H])OccO1      1\n",
       "...                      ...    ...\n",
       "11404182                 CCl      1\n",
       "11404182                 cOC      2\n",
       "11404182                  CF      3\n",
       "11404182                  CF      3\n",
       "11404182                  CF      3\n",
       "\n",
       "[68240841 rows x 2 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:47:47.024526Z",
     "start_time": "2021-12-02T23:47:10.902030Z"
    }
   },
   "outputs": [],
   "source": [
    "temp.reset_index(inplace=True)\n",
    "temp.drop_duplicates(inplace=True)\n",
    "# temp.set_index('SubstanceID',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:55:34.262836Z",
     "start_time": "2021-12-02T23:55:33.169683Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SubstanceID</th>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6890270</td>\n",
       "      <td>CF</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6890270</td>\n",
       "      <td>CBr</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27615522</td>\n",
       "      <td>cnc</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27615522</td>\n",
       "      <td>cF</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27615522</td>\n",
       "      <td>[H]C1([H])OccO1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68240833</th>\n",
       "      <td>22727365</td>\n",
       "      <td>cn(c)C(=O)[O-]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68240834</th>\n",
       "      <td>22727365</td>\n",
       "      <td>[Li+]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68240835</th>\n",
       "      <td>11404182</td>\n",
       "      <td>cOC</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68240836</th>\n",
       "      <td>11404182</td>\n",
       "      <td>CCl</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68240838</th>\n",
       "      <td>11404182</td>\n",
       "      <td>CF</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50181190 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          SubstanceID   FragmentSmiles  count\n",
       "0             6890270               CF      1\n",
       "1             6890270              CBr      1\n",
       "2            27615522              cnc      1\n",
       "3            27615522               cF      2\n",
       "4            27615522  [H]C1([H])OccO1      1\n",
       "...               ...              ...    ...\n",
       "68240833     22727365   cn(c)C(=O)[O-]      1\n",
       "68240834     22727365            [Li+]      1\n",
       "68240835     11404182              cOC      2\n",
       "68240836     11404182              CCl      1\n",
       "68240838     11404182               CF      3\n",
       "\n",
       "[50181190 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp\n",
    "temp.set_index('FragmentSmiles',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:55:51.320730Z",
     "start_time": "2021-12-02T23:55:45.235903Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index          3308625317\n",
       "SubstanceID     401449520\n",
       "count           401449520\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-02T23:55:58.885738Z",
     "start_time": "2021-12-02T23:55:58.864568Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SubstanceID</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CF</th>\n",
       "      <td>6890270</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CBr</th>\n",
       "      <td>6890270</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnc</th>\n",
       "      <td>27615522</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cF</th>\n",
       "      <td>27615522</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C1([H])OccO1</th>\n",
       "      <td>27615522</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cn(c)C(=O)[O-]</th>\n",
       "      <td>22727365</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[Li+]</th>\n",
       "      <td>22727365</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cOC</th>\n",
       "      <td>11404182</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCl</th>\n",
       "      <td>11404182</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CF</th>\n",
       "      <td>11404182</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50181190 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 SubstanceID  count\n",
       "FragmentSmiles                     \n",
       "CF                   6890270      1\n",
       "CBr                  6890270      1\n",
       "cnc                 27615522      1\n",
       "cF                  27615522      2\n",
       "[H]C1([H])OccO1     27615522      1\n",
       "...                      ...    ...\n",
       "cn(c)C(=O)[O-]      22727365      1\n",
       "[Li+]               22727365      1\n",
       "cOC                 11404182      2\n",
       "CCl                 11404182      1\n",
       "CF                  11404182      3\n",
       "\n",
       "[50181190 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:23:21.671734Z",
     "start_time": "2021-12-03T02:23:08.917393Z"
    }
   },
   "outputs": [],
   "source": [
    "# temp.to_pickle('/home/aa2133/Impurity-Project/Reaxys_Data/fragdb.pickle')\n",
    "temp=pd.read_pickle('/home/aa2133/Impurity-Project/Reaxys_Data/fragdb.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Constructing an aromatic fragment database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T04:06:25.622934Z",
     "start_time": "2021-12-23T04:06:18.424372Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb=pd.read_pickle('/home/aa2133/Impurity-Project/Reaxys_Data/SubstanceSmiles.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T04:06:47.490298Z",
     "start_time": "2021-12-23T04:06:44.990914Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-23 04:06:46,358\tINFO services.py:1171 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n",
      "UserWarning: The pandas version installed 1.2.4 does not match the supported pandas version in Modin 1.2.3. This may cause undesired side effects!\n"
     ]
    }
   ],
   "source": [
    "initray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-28T23:25:15.699917Z",
     "start_time": "2021-12-28T23:25:15.566686Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "def molfromsmiles(SMILES):\n",
    "    '''\n",
    "    Converts a smiles string into a mol object for RDKit use. Also graphically\n",
    "    represents molecule using matplotlib.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    SMILES : String\n",
    "        Smiles string of molecule\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    mol: RDKit mol object\n",
    "\n",
    "    '''\n",
    "\n",
    "    mol=Chem.MolFromSmiles(SMILES)\n",
    "    Chem.SanitizeMol(mol)\n",
    "    mol.UpdatePropertyCache(strict=False)\n",
    "    return mol\n",
    "\n",
    "\n",
    "\n",
    "def findfragsub(parent,patt,fragment=False,addHs=True): # [HH] is not a valid SMARTS but is a valid SMILES\n",
    "    '''\n",
    "    Given a parent and pattern (patt) smiles, calculates if there is a substructure match or not. \n",
    "    \n",
    "    Fragments are always passed in as the pattern, but the parent can either be another fragment (fragment = True) \n",
    "    or a molecule (fragment = False).\n",
    "    The addHs option is mainly to account for hydrogens but may not work if a fragment is passed in due to \n",
    "    valence/sanitization errors. In these cases, False is returned.\n",
    "\n",
    "    '''\n",
    "    try:\n",
    "        if fragment:\n",
    "            parentmol=Chem.MolFromSmarts(parent)\n",
    "        else:\n",
    "            parentmol=molfromsmiles(parent)\n",
    "            if addHs:\n",
    "                parentmol=Chem.AddHs(parentmol)\n",
    "        pattmol=Chem.MolFromSmarts(patt) #Pattern\n",
    "        return parentmol.GetSubstructMatches(pattmol)\n",
    "#         return parentmol.HasSubstructMatch(pattmol)\n",
    "    except Exception:\n",
    "        return False       \n",
    "\n",
    "def ffsrow(row,patt,colname='FragSmiles',reverse=False):\n",
    "    '''\n",
    "    Given a fragment dataframe, applies findfragsub across each row (parallelisation)\n",
    "    colname is either 'FragSmiles' if main species is a fragment or 'Smiles' if main species is whole molecule\n",
    "    '''\n",
    "    if colname=='FragSmiles':\n",
    "        addHs=False\n",
    "        fragment=True\n",
    "        parent=row.name\n",
    "    elif colname=='Smiles':\n",
    "        fragment=False\n",
    "        addHs=True\n",
    "        parent=row['Smiles']\n",
    "    if parent=='Error':\n",
    "        return False\n",
    "    if not reverse:\n",
    "        return findfragsub(parent,patt,fragment=fragment,addHs=addHs)\n",
    "    else:\n",
    "        return findfragsub(patt,parent,fragment=fragment,addHs=addHs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T04:09:29.344157Z",
     "start_time": "2021-12-23T04:09:29.305172Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>&gt;1 Compound</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SubstanceID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6890270</th>\n",
       "      <td>CC(C)(C)[C@@H]1CC[C@@](C)(F)[C@H](Br)C1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27615522</th>\n",
       "      <td>Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29164535</th>\n",
       "      <td>[Gd+3].[Na+].[O-]B([O-])[O-].[Tb+3]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11230426</th>\n",
       "      <td>Nc1cc(F)c(N2CC(F)C2)c(F)c1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9019791</th>\n",
       "      <td>CCCCOc1ccc(-c2nc(-c3ccc(Cl)cc3)[nH]c2-c2ccncc2...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26761831</th>\n",
       "      <td>CCOC(=O)C(CC)(CC)P(=O)(OCC)OCC</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3409746</th>\n",
       "      <td>N#CC(COC(=O)c1ccccc1)OC(=O)c1ccccc1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6138726</th>\n",
       "      <td>CN(O)C(CCC(C)(C)O)C1=CCCCC1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22727365</th>\n",
       "      <td>CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16129451 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        Smiles  >1 Compound\n",
       "SubstanceID                                                                \n",
       "6890270                CC(C)(C)[C@@H]1CC[C@@](C)(F)[C@H](Br)C1        False\n",
       "27615522           Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F        False\n",
       "29164535                   [Gd+3].[Na+].[O-]B([O-])[O-].[Tb+3]         True\n",
       "11230426                            Nc1cc(F)c(N2CC(F)C2)c(F)c1        False\n",
       "9019791      CCCCOc1ccc(-c2nc(-c3ccc(Cl)cc3)[nH]c2-c2ccncc2...        False\n",
       "...                                                        ...          ...\n",
       "26761831                        CCOC(=O)C(CC)(CC)P(=O)(OCC)OCC        False\n",
       "3409746                    N#CC(COC(=O)c1ccccc1)OC(=O)c1ccccc1        False\n",
       "6138726                            CN(O)C(CCC(C)(C)O)C1=CCCCC1        False\n",
       "22727365     CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O...         True\n",
       "11404182                   COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1        False\n",
       "\n",
       "[16129451 rows x 2 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "substancedb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T04:16:01.097318Z",
     "start_time": "2021-12-23T04:16:01.092665Z"
    }
   },
   "outputs": [],
   "source": [
    "import modin.pandas as mpd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T04:16:11.994227Z",
     "start_time": "2021-12-23T04:16:03.240485Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UserWarning: Distributing <class 'pandas.core.frame.DataFrame'> object. This may take some time.\n"
     ]
    }
   ],
   "source": [
    "substancedbdis=mpd.DataFrame(substancedb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T04:24:49.150565Z",
     "start_time": "2021-12-23T04:16:25.437632Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UserWarning: User-defined function verification is still under development in Modin. The function provided is not verified.\n"
     ]
    }
   ],
   "source": [
    "hassubstruct=substancedbdis.apply(ffsrow,patt='c[H]',colname='Smiles',axis=1,result_type='reduce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T12:30:59.689490Z",
     "start_time": "2021-12-23T12:30:57.793215Z"
    }
   },
   "outputs": [],
   "source": [
    "hassubstruct=pd.Series(data=hassubstruct.values,index=hassubstruct.index)\n",
    "match=list(hassubstruct[hassubstruct.values==True].index)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T12:31:03.816708Z",
     "start_time": "2021-12-23T12:31:03.805873Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11628811"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T12:56:59.684569Z",
     "start_time": "2021-12-23T12:56:54.966598Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>&gt;1 Compound</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SubstanceID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27615522</th>\n",
       "      <td>Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11230426</th>\n",
       "      <td>Nc1cc(F)c(N2CC(F)C2)c(F)c1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9019791</th>\n",
       "      <td>CCCCOc1ccc(-c2nc(-c3ccc(Cl)cc3)[nH]c2-c2ccncc2...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4584084</th>\n",
       "      <td>CC(C)(C)OC(=O)C=C1C[N+]([O-])=C(c2ccccc2F)c2cc...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>918811</th>\n",
       "      <td>Cc1cc([N+](=O)[O-])cc([N+](=O)[O-])c1NN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9199630</th>\n",
       "      <td>N#C[C@@H]1[C@@H]2C=C[C@@H](O2)[C@H]1S(=O)(=O)c...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7027543</th>\n",
       "      <td>CCN1Cc2ccccc2C(C#N)C1=O</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19632868</th>\n",
       "      <td>COc1cc(F)c(-c2c(Cl)nc(C)n2-c2ccc(C)cc2)c(F)c1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3409746</th>\n",
       "      <td>N#CC(COC(=O)c1ccccc1)OC(=O)c1ccccc1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11404182</th>\n",
       "      <td>COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10762713 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        Smiles  >1 Compound\n",
       "SubstanceID                                                                \n",
       "27615522           Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F        False\n",
       "11230426                            Nc1cc(F)c(N2CC(F)C2)c(F)c1        False\n",
       "9019791      CCCCOc1ccc(-c2nc(-c3ccc(Cl)cc3)[nH]c2-c2ccncc2...        False\n",
       "4584084      CC(C)(C)OC(=O)C=C1C[N+]([O-])=C(c2ccccc2F)c2cc...        False\n",
       "918811                 Cc1cc([N+](=O)[O-])cc([N+](=O)[O-])c1NN        False\n",
       "...                                                        ...          ...\n",
       "9199630      N#C[C@@H]1[C@@H]2C=C[C@@H](O2)[C@H]1S(=O)(=O)c...        False\n",
       "7027543                                CCN1Cc2ccccc2C(C#N)C1=O        False\n",
       "19632868         COc1cc(F)c(-c2c(Cl)nc(C)n2-c2ccc(C)cc2)c(F)c1        False\n",
       "3409746                    N#CC(COC(=O)c1ccccc1)OC(=O)c1ccccc1        False\n",
       "11404182                   COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1        False\n",
       "\n",
       "[10762713 rows x 2 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "substancedb.loc[(substancedb.index.isin(match)) & ~(substancedb['>1 Compound'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-23T04:13:09.466294Z",
     "start_time": "2021-12-23T04:13:04.275982Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SubstanceID\n",
       "6890270     False\n",
       "27615522     True\n",
       "29164535    False\n",
       "11230426     True\n",
       "9019791      True\n",
       "            ...  \n",
       "30970884     True\n",
       "15681494     True\n",
       "6481720     False\n",
       "6341250      True\n",
       "6931723      True\n",
       "Length: 10000, dtype: bool"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "substancedb.iloc[:10000].apply(ffsrow,patt='c[H]',colname='Smiles',axis=1,result_type='reduce')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualizing frags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T18:31:08.490769Z",
     "start_time": "2021-05-06T18:31:08.473159Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[H]Oc', '[H]N(c)C(C)=O']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['[#6]-[#8]-[H]', '[#6]-[#6](=[#8])-[#7](-[#6])-[H]']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getCarrierFrags0('CC(=O)NC1=CC=C(C=C1)O',resFormat='smiles')\n",
    "getCarrierFrags0('CC(=O)NC1=CC=C(C=C1)O',resFormat='smarts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T04:13:28.407982Z",
     "start_time": "2021-05-07T04:13:28.363710Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "rdkit.Chem.rdmolops.SanitizeFlags.SANITIZE_NONE"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAUW0lEQVR4nO3deVCU9/0H8PciGFEicnjgmRiJEMUIqESN4rFO0IhpptW2qdBJD4ytZRcjQaOGeA6K0WftGMf0Gn46Y0OdaQeJ4gAeJIKoCIIEDKBB8eQSo4ggu78/npVVQ+TY47vH+zX544HwLG/G5c1zfh6FTqcDERF1l5PoAEREto01SkRkFNYoEZFRWKNEREZxFh2ACABw9SoKC1FTg9paKBTw9oaXF4KCMGiQ6GREHWCNklA3b2L7dqSmoqSknf+rUCAgAO+8A5UKXl4WD0fUKQpe8ERiPHyIDRsgSbh/v+MvdndHXBxiY+HMP/xkdVijJEJdHebNQ27uU590dcXIkfD2hlaL6mpUVKCl5akvmDMH//0v+vSxZFKiDrFGyeJaWhAaipwcw2emTUNsLGbPRu/ehk82NCAtDZs3o7DQ8MmwMBw6BIXCcmmJOsIz9WRx69YZOtTZGX/7G7KyEB7+VIcCcHfHL3+Jc+cQH2/4ZFoaNBrLRSXqBG6NkmWdP4/gYLS26j/cvx+/+lXHa23ciLVr9cu9eqG0FCNGmCshURdxa5QsS5IMHbp4cac6FMDHH2PaNP1yUxN27TJLNqJu4dYoWVBNDYYNQ1MTADg54fJlDB/e2XVzcjBlin7Z0xNVVXB1NUtIoi7i1ihZUEaGvkMBzJ7dhQ4FMHkyXntNv1xXh+xsE2cj6i7WKFnQqVOG5bCwLq8+f75h+ckT/URCsUbJgp68UDQwsMurBwW1/1JEQrFGyYKuXzcs+/l1eXV/f8PytWsmyENkCqxRsqD6esOyu3uXV+/Xr/2XIhKKNUqW0tqKe/f0y87Oz15s3xlPNu+dO6ZJRWQ01ihZikJhuIlTq0U3rrRru+AU4IwSsh6sUbIUJyf07atf1mo7NdjpGXfvGpY9PEyTishorFGyICMPbtbVtf9SREKxRsmCnrze/ttvu7x6cbFhmffUk9VgjZIFhYQYlgsKurz6k6u88YYJ8hCZAmuULOjJGj18uMurHzpkWGaNktXgaBKyoDt3MHSo/uSSQoGKCrz8cmfXzc7G1Kn6ZR8fVFbCxcUsIYm6iFujZEH9+uG99/TLOh3WrOnsijqdYd4ogA8+YIeS9eDWKFnWhQsIDMSjR/oPDxzAz3/e8VoaDdRq/bKbG8rK+OBlsh7cGiXzuHLF0JVPGjsWK1caPnzvPfz73x281F//ipgYw4effdZ+h375JW7c6EZSIiOxRsnUGhuxZQvGjMHu3e1/wSefGM41NTfj17/GggXIyYFW+9SXPXqEY8cwYwaiow23PL3zDv74x3Zes7wckZHw9cWnnxpGmhJZBHfqyXR0Ouzfj7g4VFVBocDSpT/5tI+GBrz7Lo4de+qTAwZgzBj4+ECrxY0bKCp66np7AAsXYu9evPBCOy9YWQm1Gv/7HwC88gq2bcPPfmaSn4moYzoik8jL0735pg7QAbqgIF1WVgdf39SkW7pU5+SkX+X5//Xpo1u3Ttfa2sFrHj2qGzdOv8rMmbrz5031wxE9B7dGyWg1NdiwAbt2obUV3t5YswbLlqFHj06tW1gIjQapqbh9u/0vGDYM776LlSvh49OpF9RqsW8fVqxAdTWcnPCb32DbNgwY0NmfhajrWKNkhJYWfP45PvkEd+/CxQVLl2LDBsP8kc7T6ZCfj/Pncfs2bt+GQoGBAzFgAIKDMXZsd4LV12PLFuzYgeZmeHggLg4xMejZszsvRdQR1ih118GDiIlBRQUAKJXYufOp6fTW4OJFLF+uv/dp9Ghs345580RnIjvEGqWuu3gRMTH6uzmtv54yMqBS6SehKJXQaAxPGCUyBV7wRF1RXw+VCmPH4vBheHhAknDhglV3KAClEgUFkCS4uyMjA+PHQ6VCQ4PoWGQ/uDVKnfPoEf75T6xZg+pqODvjd7/Dxo3o3190rK6orcX69fpTYV5eWLu2C6fCiH4aa5Q64ehRqNUoKgKAWbOwYwfGjROdqbvOnYNaja+/BoCgIEgSpk0TnYlsG3fq6bnKy7FoEWbPRlERRo1CcjIyM224QwEEBSErCykpeOklnDuH6dMRHo7vvxcdi2wYt0bpJ9y/j8REbNmCpib06YMVK7ByJXr1Eh3LdB48wM6d2LgR9+7B1RXR0VizBm5uomOR7WGN0o/odNi7F3FxuHkTCgUWL8bWrXY7UenaNaxahX37oNNhyBBs3oyICMMTTIk6gTVKTztzBioVcnIAYOJEaDSYPFl0JvM7fRoqFU6dAoBJk6DRcLo+dR6PjdJj164hMhIhIcjJwZAhSEpCbq5DdCiASZOQnY2kJAwahNOnMWUKIiNx86boWGQbuDVK7R0lXL0aL74oOpYI8hHhhAQ8fKg/IrxqVfszpYgeY406vIMHER2tP1U9fz527uzC85HsVXk5Pv4Y//kPAIwahc2bsXCh6ExkvVijDiw/H2o1srIAIDAQkoTp00VnsibPXC0rSQgIEJ2JrBGPjTqk2lqoVJg4EVlZ8PKCJOHMGXbos2bNwrlz2LMH/fvj6FEEBWHJElRXi45FVodbow5GHm0XH4+GBv1ou/Xr4e4uOpZ1q6vDunX4/HM8egQPD8TH489/hrOz6FhkLVijjoSzjoxRWorly/Vzrfz8sH075s4VnYmsAnfqHcN332H+fMyZg2+/xauvIjUV6ens0K7x88OhQ0hJwSuvoLQU8+YhPFw/bpUcG2vU3tXXY+VKBATgq6/Qrx8SElBUhLffFh3LZoWHo6QEkoS+fZGaCn9/qFS4e1d0LBKJO/X2S34qUWwsbt/mU4lM78YNfPop/v53aLXw8cGnn+L3v+fYPcfEGrVTx45BrUZhIQDMnIkdO/D666Iz2aO8PKjV+OYbAAgOhiThzTdFZyJL40693bl6FZGRmDULhYUYNgxJSTh6lB1qLsHByMpCcjJGjEBeHqZPx6JFqKwUHYssilujduTJ0Xa9eyM21t5G21mzxkZs3YqtW/HgAXr3xl/+wrF7joM1ahd0Ohw4gBUrcOUKFAr84hfYtg3Dh4uO5XiqqvDxx/qxe0OHYtMmjt1zBKxR23f2LFQqZGcDwIQJ0GgwZYroTI4tNxcqFXJzASAkBBoNQkJEZyIz4rFRW3b9OpYsQUgIsrMxeDD27EFuLjtUPPlfJCkJAwfq/0UiI3HrluhYZC7cGrVNzc3YvRtr1+KHH9CzJz74ABs3OuhoO2t27x62bdOP3XNzw4cfcuyeXWKN2qCDB6FS4fJlAJg/HxoNRo4UnYl+WlkZVq/Wj93z9cWmTRy7Z2dYozalpAQxMThyBAD8/bFjB956S3Qm6pyMDMTE4MIFAJg9G5KEsWNFZyLT4LFRG1FXB5UKAQE4cgSenpAkFBWxQ22JUqkfu+ftjcxMBAZiyRLU1IiORSbArVGr19KCf/0Lq1ejpgYuLnj/fWzaBG9v0bGou+Sxe7t2obUVnp745BMsW8a7SG0aa9S6ZWRArUZxMQAolZAkjBkjOhOZAo/P2BHu1FursjKEh2POHBQXw9cXyclIT2eH2g9/f6SlISUFI0eipARhYQgPx6VLomNRd3Br1PrcuYOEBEgSr5JxCD++dm3DBvTtKzoWdQFr1JrIo+0++gi3bulH2yUmYuBA0bHI/K5fx7p1+rF7gwcjPh5/+AOcuLNoG1ijVuPECajVKCgAgNBQSBLGjxediSzrmft6JQlTp4rORB3jnzsrUFWFyEjMnImCAgwdiqQkHDvGDnVEEybgm2+QnIzhw3H2LKZNw6JFuHJFdCzqALdGhXpmulpsLOLi4OoqOhaJJr8xOPPQRrBGBZFH28XGorJSP9ouMREjRoiORdbk6lWsXo29ewFg2DBs3IjISNGZqB2sURHy8qBS4eRJgE+eoI4cPw61GufPA8CMGZAkPsvA2vDYqGXduIElSzBpEk6ehI8P9uzB6dPsUHqeGTOQl4fPP4e3N44fR3Dw/RUrangXqTVhjVpKczM0Gvj54Ysv0KMHoqNRWoqoKF7UQh3r0QNLl6KsDHFx6NHj/06d8vX13bJlS3Nzs+hkBHCn3kIOHoRarb9HZf58SBJeeUV0JrJJj0pLF8TEHE5LA+Dv7799+/awsDDRoRwdN4XMrLQUc+diwQJcugQ/Pxw6hIMH2aHUbc5+focOH05PTx8zZkxJScncuXPnzJlTLE9dIEFYo2bTNtouLc0w2m7uXNGxyB4olcr8/HxJktzd3TMyMgIDA1UqVUNDg+hcDoo79abX0tKSv3fvpBUrUF8PZ2f86U+Ij4enp+hcZIdqa2vXr1+/a9eu1tZWLy+vtWvXLlu2rAfH7lkWa9TEMjMzY2JiysvLSz08hssD0AICRIciO5efn69Wq7OysgAEBgZKkjR9+nTRoRwId+pNpqysbMGCBUqlsqioaOjQobf+8Q9kZLBDyQICAwNPnDiRkpLy8ssv5+fnh4aGhoeHX5af1kUWoCOj3bt3Lz4+/oUXXgDQp0+f+Pj4pqYm0aHIETU2NiYkJLz44osAXF1d4+Li7t69KzqU/WONGqW1tTUpKWngwIEAFApFRETEjRs3RIciR1dVVRUREaFQKAAMHjw4KSlJq9WKDmXPWKPdl5ub+8Ybb8gb9ZMmTcrJyRGdiMjg9OnTkydPlt+fEydOzM7OFp3IbrFGu+PJv/ZDhgzhX3uyTlqtNikpadCgQfLe0sKFC69cuSI6lB1ijXbN/fv3ExIS3Nzc2o49/fDDD6JDET2PfOy+V69ebcfuHzx4IDqUXWGNdkFKSspLL70k7yXNnz//8uXLohMRdVZ5efnChQvld++wYcOSkpJEJ7IfrNFOycvLmzZtmvwWDAoKysrKEp2IqDsyMzPHjRsnv5Nnzpx5/vx50YnsAWu0A9XV1dHR0fJtIV5eXpIkPXr0SHQoou6TLy/p378/ACcnp4iIiNu3b4sOZdtYoz+publZkqS+ffsCcHFxiY6OvnPnjuhQRKZRV1cXFxfXs2dPAB4eHgkJCQ8fPhQdylaxRtuXnp7u7+8v7/solcri4mLRiYhMr7S0dN68efL7fPTo0V999ZXoRDaJNfosvrHI0XCjwUisUQPu5pDDkg9hubu78xBWN7BGdTqdrqWlZc+ePTzoTg6upqaGJ1S7gTXKS0CInsLL+7rKoWu0rKys7YLk4cOH84Jkoja82aTzHLRGeXscUYd463MnOVyNPjOsgaPtiJ6Pg3g65Fg1ytFhRN3DsZDP4Sg1ykG2REaS7yLlntyP2X+N8rEKRCbER+b8mJ3XqPyQr7azjZcuXRKdiMgePHmVy6hRo5KTk0UnEslua/TcuXNtz5iVn5soOhGRvcnIyAh4/Ozb2bNnFxYWik4khh3WKO/EILIY+Q5Ab29vAM7OzlFRUQ54B6Bd1SjvCyYSora2Njo62tnZGYCnp6ckSS0tLaJDWY791Gh6evprr73WNqXmwoULohMROZaSkpKwsDD5d9DPz+/QoUOiE1mIPdToxYsX3377bfkf79VXX01NTRWdiMhxpaSkjBw5su28bnl5uehEZmfbNVpfX9822q5fv34cbUdkDR4+fPjMkyMaGhpEhzIjW61R+UrgAQMGtI22u3XrluhQRGRw/fr1qKgoJycnAD4+Pnv27GltbRUdyixsskaPHj36+uuvy3sNM2bMKCgoEJ2IiNp39uzZqVOnyr+twcHBX3/9tehEpmdjNXrlypWIiAg+a5vIhmi12uTk5BEjRsh3kS5cuPD7778XHcqUFDqdDrbg/v37iYmJW7ZsaWpq6t27d2xs7MqVK+VJd0Rk/RobG7du3bp169YHDx7Iv8JxcXGurq6ic5mC6B7vmPynbPjw4Xj8p6yyslJ0KCLqjqtXr7YNCRo6dKh9DAmy9ho9c+bMlClT5MafMGHCyZMnRSciImMdP358/Pjx8u91aGhofn6+6ERGsd4avXbtWttpvsGDB9vxaT4iByRfbDNw4EA8vtjm5s2bokN1kzXWqHzRmTzarmfPntHR0RxtR2SX5Eu/5bF7bm5uHY7dW7x48auP1dbWdvK7JCUlta21f/9+UwR/itXV6DO3QFRUVIhORETm9d1337WN3fP19X3O2L22sW0AOn+puCRJbWvt3r3bRKkNnMx7Aqsr5BtyFyxYcOnSJX9//7S0tIMHD7ZVKhHZK7k609PTx4wZU1ZWtmjRojlz5ly4cEF0rs6yihqtq6tTqVQBAQFHjhyRx8MUFRW99dZbonMRkeUolcr8/Pzt27f369cvIyMjKCho+fLljY2NonN1THCNtrS0fPHFF6NHj965c6eTk1NUVNTFixdVKpU8LZSIHIqLi0tMTExFRUV0dLRWq01LS3NxcREdqmPOAr93RkaGWq0uLi4GoFQqd+zYMXbsWIF5iMgaeHp6ajSa999/v6mpiTX6k8rKypYvX56amgrA19d306ZNbQeYiYgAtF1Yav0sXaN37txJSEiQJOnhw4dubm4ffvjhqlWr5MsdiIhskeVqVKvV7tu376OPPrp165Z8tW1iYqJ88S0Rke2yUI2eOHFCrVYXFBQACA0NlSTJhrbYiYiew+xn6quqqiIjI2fOnFlQUCBPIjh27Bg7lIjshhm3Ru15LhYR0WNmqVGdTnfgwIHY2NjKykp5tF1iYqI8tJWIyCSUSqX8SOcOVVdXmzWJWWr0t7/97d69ewFMnDhRo9FMnjzZHN+FiBxZUVGR6Ah6Zjk2GhkZKT/B6tSpU+xQIrJvZtkaVSqVFRUVPAxKRObz2WefyeM0O5SZmfnll1+aL4m5TjGxQ4nIrBYvXiw/Yr1DjY2NZq1Rq5jwRERku1ijRERGYY0SERmFNUpEZBTWKBGRUVijRERGYY0SERmFNUpEZBTWKBGRUVijRERGYY0SERlF5AOWiYi6JCoqKiwsTF52c3Pr5FpTp07dvHmzvBwSEmLyVAqdTmfyFyUichzcqSciMgprlIjIKKxRIiKjsEaJiIzy/2fK58okpVzTAAAAl3pUWHRyZGtpdFBLTCByZGtpdCAyMDIwLjA5LjEAAHice79v7T0GIOABYkYGCGAGYiYgbmBkFNAAiTNK8gIpx5L8XBDOTPYrzVWUgqmWYdDmEFAAacGmigOhik0gAWQYMzZlbAhl3AyMApKcQLZTfl6Kf1FKahGKXYxM+GRFGMRhPAbmh27L7IG+2MeAAPYgAii+HyYuBgAsFiYrpdUEAwAAAIN6VFh0TU9MIHJka2l0IDIwMjAuMDkuMQAAeJzjUgCBIBfvzBIFODBy4eJSUDAGMhQUDLAiS0tLhTAjAwMDLpB6Qz0jS0ugsIKugZ6RqQGIpWCgB5Q1UPBQwGUEMuJC0gFmmRqgmOJPtCm6eNziTKwphmCfG0I4RuCQAHN8FRRc/Vy4AFw0L/v3AO/hAAAAQXpUWHRTTUlMRVMgcmRraXQgMjAyMC4wOS4xAAB4nIv2iPV3VqjRMNQzsrQ0MNHRNdAzMtWxNtAx0ANSuqjCmjUA/V0KHNk0Eq0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<rdkit.Chem.rdchem.Mol at 0x7f5d18264a00>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m=Chem.MolFromSmarts(Chem.MolToSmarts(Chem.MolFromSmiles('[H]Oc',sanitize=False)))\n",
    "Chem.SanitizeMol(m)\n",
    "m.UpdatePropertyCache(strict=False)\n",
    "m\n",
    "m.GetAtoms()[2].GetIsAromatic()\n",
    "m.GetAtoms()[2].GetAtomicNum()\n",
    "n=Chem.MolFromSmiles('[H]Oc',sanitize=False)\n",
    "n.GetAtoms()[2].GetIsAromatic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T04:34:20.168299Z",
     "start_time": "2021-05-07T04:34:20.139726Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAWbElEQVR4nO3deVRU590H8O+wKSCbiopRo6JUJHVBUKMSok4aY6A5KTHJiYWkfasJbcLiEogxEkysC4kOWlPJWz3FeHoaa5sKLrQg0RASWSRoQBTR1w2BAIIiIstw3z8uDoOiDjDwzPL9HP94Zrx38sOMX5+7PL+rkCQJRETUXRaiCyAiMm6MUSKiHmGMEhH1CGOUiKhHrEQXQNSZixdRUIDqalRVwdISgwbB1RXe3hgyRHRlRPdijJIhuXoVmzfj4EEUF3fyuwoFpkxBYCBCQzFsWJ8XR9Q5BW94IoPQ2Ig1a7BtGxoaHr2xnR1iYrByJRSK3q+M6BEYo2QAbtzAiy/i6687vGlnhzFjMHgw1GpUVuL8ebS0dNggOBg7d8Laui8rJbofY5REa2zEnDnIzW1/Z+5crFiBuXNha9v+Zk0NDh3CunUoKmp/81e/wj//2XelEnWGV+pJtFWr2jPUxga7dyM9HQsXdshQAC4uWLwYJ0/i3Xfb3/zXv/D5531XKlFnOBslobKyMGsWWlsBQKHAV1/hhRcevdeqVVi/vm1sb49z5+Dm1otFEj0UZ6Mk1ObNbRkKYMkSnTIUwNq18PVtG9fXIyGhV2oj0g1noyTOtWsYPRrNzQBgbY0rVzB0qK77HjkCpbJtPGwYLl2CjU2vFEn0KJyNkjgpKW0ZCmDhwi5kKIB58+Du3jYuL+9whYqobzFGSZzjx9vHCxZ0bV+FAs891/7y++/1UxJR1zFGSZysrPbx1Kld3l17l+xsPdRD1C2MURLn2rX28YQJXd594sT2cWmpHuoh6hbGKAkiSaitbRtbWMDBocuf4OzcPq6p0U9VRF3HGCVB6uvbF3fa28Oi619FJ6f2sSaRifocY5QE0c5Nza2jXaJWt4+t2KuMhGGMkiB2dujXr218+3aHTNTRzZvtYxcX/VRF1HWMURJHc1SufZ5Ud9evt4+1z5MS9S3GKIkzalT7+PTpLu+uvcvjj+uhHqJuYYySODNmtI/z87u8+w8/dP5RRH2LMUriaGff4cNd27e1FYcOtb988kn9lETUdWxNQuL89BNGjUJjIwBYWeHy5S70u0tLwzPPtI1Hj0ZJCSwte6VIokfhbJTEGTIEixa1jVtasGaNrju2tnbY+A9/YIaSQJyNklC5uZgxo71t84EDWLjw0XutX49Vq9rGzs44fx4DB/ZikUQPxdko9YkLF7BuXSfv+/ggMrJtLEkICsK///2wz5EkbNqE999vf2fr1k4ydMMGhISgoqInJRPpSiLqVbduSTExUv/+EiAdPNjJBg0N0pQpEtD+KyhIys6WWls7bNbcLKWlSbNnd9jylVc6+cD6emngQAmQnJ2lzZulpqZe+bmI7uJBPfUaScK+fVixApcvQ6HASy/hk0863Cuqcf06AgPx3Xcd3nRzg5cXhg2DWo1r13Dq1L39R379a+za1fkDls+dw7JlOHAAAMaPx7p17SdhifROdI6TicrOlmbNapsz+vhImZmP2L6+XgoNlaysOkw2H/TLwUFav/7e6er9UlMlL6+2XZRK6ccf9fXDEWljjJK+lZZKS5dKFhYSIA0fLiUkSGq1rvueOSOFhEiDBz8wQEeNkiIjpYoKXT+wqUlKSGj7QGtraelSqbKyez8W0YPwoJ70p6kJf/4zPvgAdXWwscFbb+Hjj7vTSLS1Fbm5KChAeTkqK2FpiaFD4eqK6dM7tGrW3fXriI3F9u1QqzFwINaswdtv8x4p0hfGKOlJcjLCw/F//wcAAQHYuhVjxoiuqaOiIkRG4j//AQBPT2zZgmefFV0TmQLGKPVYfj4iInDsGGAM8ZScjIgIXLgAAAEBiI/H2LGiayLjxvtGqQeuX0d4OHx8cOwYBg6ESoUffzToDAUQGIiiIqhUcHDAgQPw9ER4OOrqRJdFRoyzUeqW5mZ89hk+/BC1tbC2RmgoYmONrOnntWuIjcVf/oLWVgwfjpgY/O533XmWCZk9xih1XVoaIiJQWAgASiVUKnh5ia6pu3JzER7edsuqjw/i4zFrluiayMjw317qiuJiBAbimWdQWAgPDyQnIzXViDMUgI8Pvv0We/di1Cjk5mLOHLz8Mi5fFl0WGRPORkk3tbXYsAEqFRob4eyM6GhERLQ/TMkE1NcjLg4bN+LOHdjZYeVKREejf3/RZZERYIzSo7S2Ys8erFyJn36ChQUWL0ZcHIYOFV1W77hyBe+/jy++AICRI/HxxwgJEV0TGTrGKD3U0aOIiMDJkwDg7w+VClOmiK6p9339NSIj237qp5+GSoXJk0XXRIaL50bpAa5eRUgI5s3DyZMYMQKJifj6a7PIUABz5yIvD4mJGDIER4/C2xshIfjpJ9FlkYHibJTuc/s2Nm3qcJYwKgq2tqLLEkE+I7xlC5qa2s4IR0bCxkZ0WWRYGKOk5f7WdnFxfHYxiouxbBkOHgQADw9s3oznnxddExkQxijdlZuLiAhkZgLAtGmIj8fs2aJrMiRpaQgPx+nTgPHfLUt6xXOjBJSV4c03MWMGMjPh5oaEBGRnM0PvpVQiPx8qFZyckJaGqVMRHo4bN0SXReJxNmre5NZ2a9bg5s221nYffQRHR9FlGbbqaqxd29Z2b9AgfPAB2+6ZOcaoGWOvo5744QdEROCbbwBg6lSoVHjqKdE1kRiMUbN05gwiI5GSAgATJmDLFixYILom42T4XVap9/HcqJmRW9v9/OdISWlvbccM7bbAQBQWYsOGtrZ7Xl6IjmbbPXPD2ajZaGnBrl14/31UVcHKCr/9Ldatw+DBossyFaWleO897NkDScJjj+GPf0RwMBQK0WVRX2CMmocjRxARgYICAJg/HyoVnnhCdE2mKCcH4eH4/nsA8PVFfDyefFJ0TdTreFBv6s6dw8svQ6lEQQHGj8fevUhLY4b2Fl9fZGYiMRHDhiEnB7NnIyQE5eWiy6Lexdmo6bp1C598gg0b0NgIe3usWIH33jOp1naGTLvtnvyHz7Z7posxaork1nbvvouKCtNvbWfISkqwahX+8Q8AcHfH+vVYtEh0TaR/jFGTk5WF8HBkZQHA9OmIj8fMmaJrMm/p6YiMxKlTADBvHrZswaRJomsifeK5URMit7Z78klkZbW1tjt+nBkq3rx5OHECCQlwdUV6OqZORUgIKitFl0V6w9moSbh9G9u24eOPcesW7OzwzjtYvRoDBoguizqqqcHGjW1t91xcEBXFtnumgTFq/JKT8c47uHQJAAICsG0bRo8WXBI9xNmzWLYMhw4BwM9+hs2bsXCh6JqoRxijxiwvD+Hh+PZbAPD2Rnw85swRXRPpJi0NYWEoKgIApRLx8Zg4UXRN1E08N2qc5NZ206fj228xeDBUKmRnM0ONiVKJkyehUsHREWlpmDKFbfeMF2ejxqa5GZ991tbaztoaoaFsbWfcqqrw0Udsu2fUGKNGJTkZkZE4fx4AlEps3QpPT9E1kT7k5SEiAhkZAODtDZUKfn6iayJdMUaNxJkzWLYMhw8DvC5hupKTERaGixcBXi00Jjw3avBqatpa2x0+DBcXqFQoKGCGmia57d6aNbC1ldvuSbGxDbdviy6LHoExarhaWlr+9Kc//e9rr2HrVgB4+22UlCA8HFZWokujXmNnh9hYnDuH4GA0NFz+73/He3js3r2bR42GjAf1Bio1NTUyMrKwsNDZyenC00+7rFvH51CanYyM//noo12pqQD8/PxUKpW3t7fomqgzEhmYc+fOLbrbwGLcuHF79+4VXREJo1arExMThw4dCsDCwiI4OLisrEx0UXQvzkYNSH19fVxc3MaNG+/cuWNvb79ixYro6Oj+7K5m9uQvxoYNGxobG+UvxnvvvdePPQ8Nh+gcJ0mSpNbW1sTExGHDhgFQKBScdND9iouLeZhimBij4mVnZz9591ETvr6+3333neiKyHClpaU9cffhBfPnzz916pToiogxKtTVq1eDg4MVCgWAxx57LDExsbW1VXRRZOiam5sTEhIGDx4MwMrKaunSpZWVlaKLMmuMUTFu3769YcOGAQMGALC1tY2Kirp586boosiYVFdXh4WFWVlZARg4cKBKpWpubhZdlJlijAqQlJQ0+u7qlICAgAsXLoiuiIxVUVHRggUL5O/ShAkTDh8+LLoic8QY7VN5eXl+d9dKT5069dixY6IrIlOQlJQ0duxYzT/MJSUloisyL4zRPlJVVRUWFmZpaQlg0KBBKpWqpaVFdFFkOhobG1UqlaOjIwBra+uwsLAbN26ILspcMEZ7XVNTk0qlcnJy0ny/a2trRRdFpunatWtLly61sLAA4ObmlpCQoFarRRdl+hijvSs1NXXi3a7mSqWysLBQdEVk+nJzc2fPni1/66ZNm5aRkSG6IhPHGO0tZ86cef755+WvsoeHx4EDB0RXRGaktbV17969jz/+uLygY9GiRRcvXhRdlMlijOrf9evXo6KibGxsADg7O8tr+EQXReaovr4+JibG1tYWgJ2dXUxMzO3bt0UXZYIYo/okN5IYMmQI7jaSqKioEF0UmbsrV65oVnmMGDGCqzz0jjGqN+np6ZMmTZKP4ufOnZufny+6IqJ2R48enTx5svz99Pf3/+GHH0RXZDoYo3pw+fLl4OBg+Qs6cuTIxMRE0RURdeL+tnvl5eWiizIFjNEeuXXrVkxMjNzLTj731NDQILooooepqamJioqS++zJ5+7v3LkjuijjxhjtJvlK6KhRozRXQi9duiS6KCJdnT17NiAgQD6EGj9+PNvu9QRjtDtycnJmzZolfwV9fHwyMzNFV0TUHampqV53H06jVCoLCgpEV2SUGKNdU1paqlklMnz4cK4SIWMnr7JzdnaWV9mx7V43MEZ1Ja9ZdnBwAGBjYxMWFsbWdmQy5LZ7cs8Hue0eez7ojjGqk6SkpDFjxmg66Jw/f150RUT6d/r06WeffVb+nnt6eqakpIiuyDgwRh+BXywyN5w0dBVj9IF4mENmi6ewuoQx2ommpibNs2540p3MFi+o6ogxei/eAkKkLTs7m7f3PRxjtF1xcTFvSCa6HxebPBxjVJI6Lo8bMGBATEwMl8cR3UN76bO9vT2XPmuYe4yyWQNRl7ARz/3MOkaPHj06ZcoUtg4j6iq2hdRmpjHKRrZEPcQm5RpmF6N8rAKRHvGROZIkKSRJgmFTq9U5OTknTpyoqqqqrq5ubW0dMmTIsGHD/Pz8PD09df8cSZL27du3cuXKS5cuKRSKl156KS4uTn7mFxH1xNmzZ5cvX37w4EEAHh4emzdv1jzPsVOFhYXyxgD8/f1nzJihy39F7qIij93c3DSnaMUTneMPc/bs2eDgYBcXlwcV7+npuWnTJl2mk3zkLFFv0/1x4n/96181f4vXr1+v4+ffuHFDs5evr6+eqtYDA43RO3fuhIaGWllZ6fIvgYeHR15e3oM+6tq1a5qVGG5ublyJQdR75Amjk5MTAGtr67CwsNra2vs3Y4z2upqaGn9//3uysn///tOmTVu4cGFgYKCPj498clPDwcHh/gmmvC7Y0dFR83/0xo0bQn4iIrNSVVWl6UcxaNCg+/tRMEZ7V0tLi5+fn3ZETpgw4e9//3tdXZ32ZnV1dXv27Bk9erRmM29vb+2r7UlJSWPHjpV/KyAgoKSkpM9/FCKzduLEiTlz5mj+emrfkW1iMWrxqCPmvrZx48aMjAzNy9DQ0JMnT77yyisDBgzQ3mzAgAGLFy8uKCgICgoC4Ofnd/jwYfkGJgBqtXr16tUXLlyYMGHCoUOHkpOT3d3d+/KnICJvb++MjIykpKTRo0fb2NjIt0aZJJ1OPvaZsrKy2NhYzcs33nhj+/btmnC8n729/d/+9rcdO3a89dZb8i0XMktLy61btxYWFi5dulTHE6xE1BsCAwPnz59fWVn5kL/Ixs6wIubzzz9vamqSx25ubiqV6pF/9HIzxPvf9/f3v/8EKxH1PTs7O9O+s9CADuolSdq5c6fm5apVq+TrfUREhsyAYrSkpOTKlSvy2Nra+tVXXxVbDxGRLgwoRrOysjTj6dOny83niYgMnAHFaH5+vmY8efJkgZUQEenOgC4xVVZWasYeHh4CKyGiPnP8+PHt27frsuWdO3d6u5juMaAYra2t1Yx5cYnITOzfv3///v2iq+gRAzqo116iID/ZlYjI8BnQbFT7/vnm5maBlRBRnxk5cuTw4cN12VKtVufm5vZ2Pd1gQDGqfSBfV1cnsBIi6jO///3vo6Ojddny5s2bhnm6z4AO6p2dnTXjqqoqgZUQEenOgGJUfgq27McffxRYCRGR7gwoRqdPn64Z5+XlCayEiEh3BhSjvr6+mkYkZ8+e5YSUiIyCAcXowIEDNY9LAqDd2JWIyGAZUIwCCA0N1Yy3b99eVFQksBgiIl0YVowGBQWNHDlSHjc2Nr7++uvaS5uIiAyQYcVov379du3apTlDmpOTM3/+/OLi4ofsolar9+3bl5KS0icFEhHdy7BiFIBSqVy+fLnmZV5e3hNPPBEaGnrkyBFNY3wADQ0NmZmZ0dHR7u7uixYtWr9+vYhiiYgMaRWTxqZNm+zs7NauXSu/bG5u3rFjx44dOywtLV1dXZ2dnevq6ioqKlpaWjS7ZGRkXLhwQfMoUCKiPmNws1EACoUiNjZ237599zzOU61Wl5eXnzlzprS0VDtDAcycObOxsbFvyyQiAgxzNioLCgr65S9/uXv37q+++uro0aP19fX3bGBhYTFu3LjnnnsuKCjonkfbE5Ehc3R01EySXFxcdNzLwsJCs5fmWrQhUEiSJLqGR2tqaiouLq6qqqqoqJAkydHRcdCgQV5eXvc8vF5bc3NzcXFxWVmZpaXliBEjxo0bp/2Q0aampvr6ehsbG3t7ewBXr14tKSkZOnSou7u7dqMpIqJHk0xOenp6SEjIPQnr5ub26aefqtVqeZsvvvgCgFKpvH79+qJFiyws2k5ufPnll2KLJyKjY7gH9d324YcffvPNN7a2tpMnTx45cmRdXd2pU6fKysqWL19eW1uruXIF4PTp04GBgZmZmQqFwtbWtqGhYdKkSQIrJyJjZBwH9V1y7Nixixcvvvjii46OjvI7tbW1L7zwgpytZWVlTk5Oe/bsCQ4OBmBvb79t27bXXnvNxsbmypUrI0aM0MxMiYh0YYKR4e/v//rrr2syFICzs/Pq1asBNDQ0aD/GGcCOHTt+85vf9OvXT6FQjBo1ihlKRF1lLqkxfvx4eVBeXq79Pi/xE1EPmeC5UQCSJOXm5qanpxcXF8vPI9HcVcrbS4lIv0wwRouKit54443s7GzRhRCRWTC1GC0tLfXz86uurnZxcQkJCVEqla6urq6urtXV1drd9YmI9MXUYjQ+Pr66utrGxiYrK0tzPhTAQ27UJyLqCVO7xJSfnw9gypQp2hlKRNR7TC1G5aWcNTU12vfDSpL06aefiiuKiEyZqcXoL37xCwDnzp1bsmRJbm5uUVHRl19+6efnt2nTJu019URE+mJqMfrmm2/Onz8fwM6dO319fSdOnPjqq6/m5OTExcU99dRToqsjIhNkapeY+vXrl5KSsmvXrv3795eXlzs4OMycOXPJkiXu7u7jx4+fOXPm5MmTAXh5eUVFRQFwcnISXTIRGTcTXFNPRNSXTO2gnoiojzFGiYh6hDFKRNQj/w9TS5D6msHs3gAAAJd6VFh0cmRraXRQS0wgcmRraXQgMjAyMC4wOS4xAAB4nHu/b+09BiDgAWJGBghgBmImIG5gZBTQAIkzSvICKceS/FwQzkz2K81VlIKplmHQ5hBQAGnBpooDoYpNIAFkGDM2ZWwIZdwMjAKSnEC2U35ein9RSmoRil2MTPhkRRjEYTwG5oduy+yBvtjHgAD2IAIovh8mLgYALBYmK6XVBAMAAACDelRYdE1PTCByZGtpdCAyMDIwLjA5LjEAAHic41IAgSAX78wSBTgwcuHiUlAwBjIUFAywIktLS4UwIwMDAy6QekM9I0tLoLCCroGekakBiKVgoAeUNVDwUMBlBDLiQtIBZpkaoJjiT7Qpunjc4kysKYZgnxtCOEbgkABzfBUUXP1cuABcNC/79wDv4QAAAEF6VFh0U01JTEVTIHJka2l0IDIwMjAuMDkuMQAAeJyL9oj1d1ao0TDUM7K0NDDR0TXQMzLVsTbQMdADUrqowpo1AP1dChzZNBKtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<rdkit.Chem.rdchem.Mol at 0x7f5d18264a00>"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n.GetAtoms()[0].GetIdx()\n",
    "m.GetAtoms()[0].GetIdx()\n",
    "m.GetAtoms()[2].SetProp(\"_displayLabel\",\"C<sub>ar</sub>\")\n",
    "# mol.GetAtomWithIdx(3).SetProp(\"_displayLabel\",\"R<sub>1</sub>\")\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T04:17:57.955028Z",
     "start_time": "2021-05-07T04:17:57.929546Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAUW0lEQVR4nO3deVCU9/0H8PciGFEicnjgmRiJEMUIqESN4rFO0IhpptW2qdBJD4ytZRcjQaOGeA6K0WftGMf0Gn46Y0OdaQeJ4gAeJIKoCIIEDKBB8eQSo4ggu78/npVVQ+TY47vH+zX544HwLG/G5c1zfh6FTqcDERF1l5PoAEREto01SkRkFNYoEZFRWKNEREZxFh2ACABw9SoKC1FTg9paKBTw9oaXF4KCMGiQ6GREHWCNklA3b2L7dqSmoqSknf+rUCAgAO+8A5UKXl4WD0fUKQpe8ERiPHyIDRsgSbh/v+MvdndHXBxiY+HMP/xkdVijJEJdHebNQ27uU590dcXIkfD2hlaL6mpUVKCl5akvmDMH//0v+vSxZFKiDrFGyeJaWhAaipwcw2emTUNsLGbPRu/ehk82NCAtDZs3o7DQ8MmwMBw6BIXCcmmJOsIz9WRx69YZOtTZGX/7G7KyEB7+VIcCcHfHL3+Jc+cQH2/4ZFoaNBrLRSXqBG6NkmWdP4/gYLS26j/cvx+/+lXHa23ciLVr9cu9eqG0FCNGmCshURdxa5QsS5IMHbp4cac6FMDHH2PaNP1yUxN27TJLNqJu4dYoWVBNDYYNQ1MTADg54fJlDB/e2XVzcjBlin7Z0xNVVXB1NUtIoi7i1ihZUEaGvkMBzJ7dhQ4FMHkyXntNv1xXh+xsE2cj6i7WKFnQqVOG5bCwLq8+f75h+ckT/URCsUbJgp68UDQwsMurBwW1/1JEQrFGyYKuXzcs+/l1eXV/f8PytWsmyENkCqxRsqD6esOyu3uXV+/Xr/2XIhKKNUqW0tqKe/f0y87Oz15s3xlPNu+dO6ZJRWQ01ihZikJhuIlTq0U3rrRru+AU4IwSsh6sUbIUJyf07atf1mo7NdjpGXfvGpY9PEyTishorFGyICMPbtbVtf9SREKxRsmCnrze/ttvu7x6cbFhmffUk9VgjZIFhYQYlgsKurz6k6u88YYJ8hCZAmuULOjJGj18uMurHzpkWGaNktXgaBKyoDt3MHSo/uSSQoGKCrz8cmfXzc7G1Kn6ZR8fVFbCxcUsIYm6iFujZEH9+uG99/TLOh3WrOnsijqdYd4ogA8+YIeS9eDWKFnWhQsIDMSjR/oPDxzAz3/e8VoaDdRq/bKbG8rK+OBlsh7cGiXzuHLF0JVPGjsWK1caPnzvPfz73x281F//ipgYw4effdZ+h375JW7c6EZSIiOxRsnUGhuxZQvGjMHu3e1/wSefGM41NTfj17/GggXIyYFW+9SXPXqEY8cwYwaiow23PL3zDv74x3Zes7wckZHw9cWnnxpGmhJZBHfqyXR0Ouzfj7g4VFVBocDSpT/5tI+GBrz7Lo4de+qTAwZgzBj4+ECrxY0bKCp66np7AAsXYu9evPBCOy9YWQm1Gv/7HwC88gq2bcPPfmaSn4moYzoik8jL0735pg7QAbqgIF1WVgdf39SkW7pU5+SkX+X5//Xpo1u3Ttfa2sFrHj2qGzdOv8rMmbrz5031wxE9B7dGyWg1NdiwAbt2obUV3t5YswbLlqFHj06tW1gIjQapqbh9u/0vGDYM776LlSvh49OpF9RqsW8fVqxAdTWcnPCb32DbNgwY0NmfhajrWKNkhJYWfP45PvkEd+/CxQVLl2LDBsP8kc7T6ZCfj/Pncfs2bt+GQoGBAzFgAIKDMXZsd4LV12PLFuzYgeZmeHggLg4xMejZszsvRdQR1ih118GDiIlBRQUAKJXYufOp6fTW4OJFLF+uv/dp9Ghs345580RnIjvEGqWuu3gRMTH6uzmtv54yMqBS6SehKJXQaAxPGCUyBV7wRF1RXw+VCmPH4vBheHhAknDhglV3KAClEgUFkCS4uyMjA+PHQ6VCQ4PoWGQ/uDVKnfPoEf75T6xZg+pqODvjd7/Dxo3o3190rK6orcX69fpTYV5eWLu2C6fCiH4aa5Q64ehRqNUoKgKAWbOwYwfGjROdqbvOnYNaja+/BoCgIEgSpk0TnYlsG3fq6bnKy7FoEWbPRlERRo1CcjIyM224QwEEBSErCykpeOklnDuH6dMRHo7vvxcdi2wYt0bpJ9y/j8REbNmCpib06YMVK7ByJXr1Eh3LdB48wM6d2LgR9+7B1RXR0VizBm5uomOR7WGN0o/odNi7F3FxuHkTCgUWL8bWrXY7UenaNaxahX37oNNhyBBs3oyICMMTTIk6gTVKTztzBioVcnIAYOJEaDSYPFl0JvM7fRoqFU6dAoBJk6DRcLo+dR6PjdJj164hMhIhIcjJwZAhSEpCbq5DdCiASZOQnY2kJAwahNOnMWUKIiNx86boWGQbuDVK7R0lXL0aL74oOpYI8hHhhAQ8fKg/IrxqVfszpYgeY406vIMHER2tP1U9fz527uzC85HsVXk5Pv4Y//kPAIwahc2bsXCh6ExkvVijDiw/H2o1srIAIDAQkoTp00VnsibPXC0rSQgIEJ2JrBGPjTqk2lqoVJg4EVlZ8PKCJOHMGXbos2bNwrlz2LMH/fvj6FEEBWHJElRXi45FVodbow5GHm0XH4+GBv1ou/Xr4e4uOpZ1q6vDunX4/HM8egQPD8TH489/hrOz6FhkLVijjoSzjoxRWorly/Vzrfz8sH075s4VnYmsAnfqHcN332H+fMyZg2+/xauvIjUV6ens0K7x88OhQ0hJwSuvoLQU8+YhPFw/bpUcG2vU3tXXY+VKBATgq6/Qrx8SElBUhLffFh3LZoWHo6QEkoS+fZGaCn9/qFS4e1d0LBKJO/X2S34qUWwsbt/mU4lM78YNfPop/v53aLXw8cGnn+L3v+fYPcfEGrVTx45BrUZhIQDMnIkdO/D666Iz2aO8PKjV+OYbAAgOhiThzTdFZyJL40693bl6FZGRmDULhYUYNgxJSTh6lB1qLsHByMpCcjJGjEBeHqZPx6JFqKwUHYssilujduTJ0Xa9eyM21t5G21mzxkZs3YqtW/HgAXr3xl/+wrF7joM1ahd0Ohw4gBUrcOUKFAr84hfYtg3Dh4uO5XiqqvDxx/qxe0OHYtMmjt1zBKxR23f2LFQqZGcDwIQJ0GgwZYroTI4tNxcqFXJzASAkBBoNQkJEZyIz4rFRW3b9OpYsQUgIsrMxeDD27EFuLjtUPPlfJCkJAwfq/0UiI3HrluhYZC7cGrVNzc3YvRtr1+KHH9CzJz74ABs3OuhoO2t27x62bdOP3XNzw4cfcuyeXWKN2qCDB6FS4fJlAJg/HxoNRo4UnYl+WlkZVq/Wj93z9cWmTRy7Z2dYozalpAQxMThyBAD8/bFjB956S3Qm6pyMDMTE4MIFAJg9G5KEsWNFZyLT4LFRG1FXB5UKAQE4cgSenpAkFBWxQ22JUqkfu+ftjcxMBAZiyRLU1IiORSbArVGr19KCf/0Lq1ejpgYuLnj/fWzaBG9v0bGou+Sxe7t2obUVnp745BMsW8a7SG0aa9S6ZWRArUZxMQAolZAkjBkjOhOZAo/P2BHu1FursjKEh2POHBQXw9cXyclIT2eH2g9/f6SlISUFI0eipARhYQgPx6VLomNRd3Br1PrcuYOEBEgSr5JxCD++dm3DBvTtKzoWdQFr1JrIo+0++gi3bulH2yUmYuBA0bHI/K5fx7p1+rF7gwcjPh5/+AOcuLNoG1ijVuPECajVKCgAgNBQSBLGjxediSzrmft6JQlTp4rORB3jnzsrUFWFyEjMnImCAgwdiqQkHDvGDnVEEybgm2+QnIzhw3H2LKZNw6JFuHJFdCzqALdGhXpmulpsLOLi4OoqOhaJJr8xOPPQRrBGBZFH28XGorJSP9ouMREjRoiORdbk6lWsXo29ewFg2DBs3IjISNGZqB2sURHy8qBS4eRJgE+eoI4cPw61GufPA8CMGZAkPsvA2vDYqGXduIElSzBpEk6ehI8P9uzB6dPsUHqeGTOQl4fPP4e3N44fR3Dw/RUrangXqTVhjVpKczM0Gvj54Ysv0KMHoqNRWoqoKF7UQh3r0QNLl6KsDHFx6NHj/06d8vX13bJlS3Nzs+hkBHCn3kIOHoRarb9HZf58SBJeeUV0JrJJj0pLF8TEHE5LA+Dv7799+/awsDDRoRwdN4XMrLQUc+diwQJcugQ/Pxw6hIMH2aHUbc5+focOH05PTx8zZkxJScncuXPnzJlTLE9dIEFYo2bTNtouLc0w2m7uXNGxyB4olcr8/HxJktzd3TMyMgIDA1UqVUNDg+hcDoo79abX0tKSv3fvpBUrUF8PZ2f86U+Ij4enp+hcZIdqa2vXr1+/a9eu1tZWLy+vtWvXLlu2rAfH7lkWa9TEMjMzY2JiysvLSz08hssD0AICRIciO5efn69Wq7OysgAEBgZKkjR9+nTRoRwId+pNpqysbMGCBUqlsqioaOjQobf+8Q9kZLBDyQICAwNPnDiRkpLy8ssv5+fnh4aGhoeHX5af1kUWoCOj3bt3Lz4+/oUXXgDQp0+f+Pj4pqYm0aHIETU2NiYkJLz44osAXF1d4+Li7t69KzqU/WONGqW1tTUpKWngwIEAFApFRETEjRs3RIciR1dVVRUREaFQKAAMHjw4KSlJq9WKDmXPWKPdl5ub+8Ybb8gb9ZMmTcrJyRGdiMjg9OnTkydPlt+fEydOzM7OFp3IbrFGu+PJv/ZDhgzhX3uyTlqtNikpadCgQfLe0sKFC69cuSI6lB1ijXbN/fv3ExIS3Nzc2o49/fDDD6JDET2PfOy+V69ebcfuHzx4IDqUXWGNdkFKSspLL70k7yXNnz//8uXLohMRdVZ5efnChQvld++wYcOSkpJEJ7IfrNFOycvLmzZtmvwWDAoKysrKEp2IqDsyMzPHjRsnv5Nnzpx5/vx50YnsAWu0A9XV1dHR0fJtIV5eXpIkPXr0SHQoou6TLy/p378/ACcnp4iIiNu3b4sOZdtYoz+publZkqS+ffsCcHFxiY6OvnPnjuhQRKZRV1cXFxfXs2dPAB4eHgkJCQ8fPhQdylaxRtuXnp7u7+8v7/solcri4mLRiYhMr7S0dN68efL7fPTo0V999ZXoRDaJNfosvrHI0XCjwUisUQPu5pDDkg9hubu78xBWN7BGdTqdrqWlZc+ePTzoTg6upqaGJ1S7gTXKS0CInsLL+7rKoWu0rKys7YLk4cOH84Jkoja82aTzHLRGeXscUYd463MnOVyNPjOsgaPtiJ6Pg3g65Fg1ytFhRN3DsZDP4Sg1ykG2REaS7yLlntyP2X+N8rEKRCbER+b8mJ3XqPyQr7azjZcuXRKdiMgePHmVy6hRo5KTk0UnEslua/TcuXNtz5iVn5soOhGRvcnIyAh4/Ozb2bNnFxYWik4khh3WKO/EILIY+Q5Ab29vAM7OzlFRUQ54B6Bd1SjvCyYSora2Njo62tnZGYCnp6ckSS0tLaJDWY791Gh6evprr73WNqXmwoULohMROZaSkpKwsDD5d9DPz+/QoUOiE1mIPdToxYsX3377bfkf79VXX01NTRWdiMhxpaSkjBw5su28bnl5uehEZmfbNVpfX9822q5fv34cbUdkDR4+fPjMkyMaGhpEhzIjW61R+UrgAQMGtI22u3XrluhQRGRw/fr1qKgoJycnAD4+Pnv27GltbRUdyixsskaPHj36+uuvy3sNM2bMKCgoEJ2IiNp39uzZqVOnyr+twcHBX3/9tehEpmdjNXrlypWIiAg+a5vIhmi12uTk5BEjRsh3kS5cuPD7778XHcqUFDqdDrbg/v37iYmJW7ZsaWpq6t27d2xs7MqVK+VJd0Rk/RobG7du3bp169YHDx7Iv8JxcXGurq6ic5mC6B7vmPynbPjw4Xj8p6yyslJ0KCLqjqtXr7YNCRo6dKh9DAmy9ho9c+bMlClT5MafMGHCyZMnRSciImMdP358/Pjx8u91aGhofn6+6ERGsd4avXbtWttpvsGDB9vxaT4iByRfbDNw4EA8vtjm5s2bokN1kzXWqHzRmTzarmfPntHR0RxtR2SX5Eu/5bF7bm5uHY7dW7x48auP1dbWdvK7JCUlta21f/9+UwR/itXV6DO3QFRUVIhORETm9d1337WN3fP19X3O2L22sW0AOn+puCRJbWvt3r3bRKkNnMx7Aqsr5BtyFyxYcOnSJX9//7S0tIMHD7ZVKhHZK7k609PTx4wZU1ZWtmjRojlz5ly4cEF0rs6yihqtq6tTqVQBAQFHjhyRx8MUFRW99dZbonMRkeUolcr8/Pzt27f369cvIyMjKCho+fLljY2NonN1THCNtrS0fPHFF6NHj965c6eTk1NUVNTFixdVKpU8LZSIHIqLi0tMTExFRUV0dLRWq01LS3NxcREdqmPOAr93RkaGWq0uLi4GoFQqd+zYMXbsWIF5iMgaeHp6ajSa999/v6mpiTX6k8rKypYvX56amgrA19d306ZNbQeYiYgAtF1Yav0sXaN37txJSEiQJOnhw4dubm4ffvjhqlWr5MsdiIhskeVqVKvV7tu376OPPrp165Z8tW1iYqJ88S0Rke2yUI2eOHFCrVYXFBQACA0NlSTJhrbYiYiew+xn6quqqiIjI2fOnFlQUCBPIjh27Bg7lIjshhm3Ru15LhYR0WNmqVGdTnfgwIHY2NjKykp5tF1iYqI8tJWIyCSUSqX8SOcOVVdXmzWJWWr0t7/97d69ewFMnDhRo9FMnjzZHN+FiBxZUVGR6Ah6Zjk2GhkZKT/B6tSpU+xQIrJvZtkaVSqVFRUVPAxKRObz2WefyeM0O5SZmfnll1+aL4m5TjGxQ4nIrBYvXiw/Yr1DjY2NZq1Rq5jwRERku1ijRERGYY0SERmFNUpEZBTWKBGRUVijRERGYY0SERmFNUpEZBTWKBGRUVijRERGYY0SERlF5AOWiYi6JCoqKiwsTF52c3Pr5FpTp07dvHmzvBwSEmLyVAqdTmfyFyUichzcqSciMgprlIjIKKxRIiKjsEaJiIzy/2fK58okpVzTAAAAl3pUWHRyZGtpdFBLTCByZGtpdCAyMDIwLjA5LjEAAHice79v7T0GIOABYkYGCGAGYiYgbmBkFNAAiTNK8gIpx5L8XBDOTPYrzVWUgqmWYdDmEFAAacGmigOhik0gAWQYMzZlbAhl3AyMApKcQLZTfl6Kf1FKahGKXYxM+GRFGMRhPAbmh27L7IG+2MeAAPYgAii+HyYuBgAsFiYrpdUEAwAAAIN6VFh0TU9MIHJka2l0IDIwMjAuMDkuMQAAeJzjUgCBIBfvzBIFODBy4eJSUDAGMhQUDLAiS0tLhTAjAwMDLpB6Qz0jS0ugsIKugZ6RqQGIpWCgB5Q1UPBQwGUEMuJC0gFmmRqgmOJPtCm6eNziTKwphmCfG0I4RuCQAHN8FRRc/Vy4AFw0L/v3AO/hAAAAQXpUWHRTTUlMRVMgcmRraXQgMjAyMC4wOS4xAAB4nIv2iPV3VqjRMNQzsrQ0MNHRNdAzMtWxNtAx0ANSuqjCmjUA/V0KHNk0Eq0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<rdkit.Chem.rdchem.Mol at 0x7f5d17b4a940>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<rdkit.Chem.rdAbbreviations._vectN5RDKit13Abbreviations22AbbreviationDefinitionE at 0x7f5d17b44ac0>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from rdkit.Chem import rdAbbreviations\n",
    "abbrevs = rdAbbreviations.GetDefaultAbbreviations()\n",
    "nm = rdAbbreviations.CondenseMolAbbreviations(m,abbrevs)\n",
    "nm\n",
    "abbrevs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T18:42:04.375819Z",
     "start_time": "2021-05-06T18:42:04.365127Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Chem.Fragments.fr_Ar_OH(Chem.AddHs(Chem.MolFromSmiles('CC(=O)NC1=CC=C(C=C1)O')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T14:39:50.195208Z",
     "start_time": "2021-05-06T14:39:43.635009Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SubstanceID</th>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>28349593</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>13263142</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>19189832</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>21333123</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>791</th>\n",
       "      <td>9808753</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180454</th>\n",
       "      <td>14525820</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180664</th>\n",
       "      <td>7549637</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180740</th>\n",
       "      <td>10725846</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180793</th>\n",
       "      <td>10784965</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50181077</th>\n",
       "      <td>29514296</td>\n",
       "      <td>COC(=O)N(C)C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>281355 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          SubstanceID FragmentSmiles  count\n",
       "153          28349593   COC(=O)N(C)C      1\n",
       "385          13263142   COC(=O)N(C)C      1\n",
       "452          19189832   COC(=O)N(C)C      1\n",
       "515          21333123   COC(=O)N(C)C      1\n",
       "791           9808753   COC(=O)N(C)C      1\n",
       "...               ...            ...    ...\n",
       "50180454     14525820   COC(=O)N(C)C      1\n",
       "50180664      7549637   COC(=O)N(C)C      1\n",
       "50180740     10725846   COC(=O)N(C)C      1\n",
       "50180793     10784965   COC(=O)N(C)C      1\n",
       "50181077     29514296   COC(=O)N(C)C      1\n",
       "\n",
       "[281355 rows x 3 columns]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp.loc[temp.FragmentSmiles=='COC(=O)N(C)C']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functional group popularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T12:14:56.662757Z",
     "start_time": "2021-05-06T12:14:19.845528Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th>Frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CF</td>\n",
       "      <td>1034442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CBr</td>\n",
       "      <td>207543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cnc</td>\n",
       "      <td>2820013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cF</td>\n",
       "      <td>864157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[H]C1([H])OccO1</td>\n",
       "      <td>95034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180462</th>\n",
       "      <td>[H]C(=C([H])C1=C([H])C([H])=C([H])C1=P(c)(c)c)...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180467</th>\n",
       "      <td>[H]N(C)C(=O)O/N=C1/CN(C)C(C)(C)N1[H]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180509</th>\n",
       "      <td>[H]C([H])=C([S@@](c)=O)[S@](c)=O</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180824</th>\n",
       "      <td>C/C1=C(\\[Sn](C)(C)C)[Si](C)(C)C2[Si](C)(C)/C([...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50180867</th>\n",
       "      <td>[H]O/N=C(\\[H])C1=N/C(=N\\N([H])c)C(C)=C1C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>573153 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             FragmentSmiles  Frequency\n",
       "0                                                        CF    1034442\n",
       "1                                                       CBr     207543\n",
       "2                                                       cnc    2820013\n",
       "3                                                        cF     864157\n",
       "4                                           [H]C1([H])OccO1      95034\n",
       "...                                                     ...        ...\n",
       "50180462  [H]C(=C([H])C1=C([H])C([H])=C([H])C1=P(c)(c)c)...          1\n",
       "50180467               [H]N(C)C(=O)O/N=C1/CN(C)C(C)(C)N1[H]          1\n",
       "50180509                   [H]C([H])=C([S@@](c)=O)[S@](c)=O          1\n",
       "50180824  C/C1=C(\\[Sn](C)(C)C)[Si](C)(C)C2[Si](C)(C)/C([...          1\n",
       "50180867           [H]O/N=C(\\[H])C1=N/C(=N\\N([H])c)C(C)=C1C          1\n",
       "\n",
       "[573153 rows x 2 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freq=temp[['SubstanceID','FragmentSmiles']].groupby(['FragmentSmiles'])['SubstanceID'].transform('count')\n",
    "freqtable=temp[['FragmentSmiles']]\n",
    "freqtable['Frequency']=freq\n",
    "freqtable.drop_duplicates(inplace=True)\n",
    "freqtable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T12:50:54.006356Z",
     "start_time": "2021-05-06T12:50:53.975484Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "# freqtable.set_index('FragmentSmiles',inplace=True)\n",
    "freqtable['Popularity'] = freqtable['Frequency'].div(freqtable['Frequency'].sum(axis=0), axis=0).multiply(100).round()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T04:41:46.119787Z",
     "start_time": "2021-05-07T04:41:46.095483Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAO8ElEQVR4nO3da1CU1x0G8GfZVRBRWwNqrFoNokC8EhjGmKoRY7RCPsTatI6rmcxkSYYKNloXb2wSkik0iVliKKIfMqjRCU47GZgkzYCaeMUi4oCgImoJl1LFG1dZ2T39sBtMOiksnPflXeD5fVrHfc//r8M+nPNezuqEECAiot7y0roBIqL+jTFKRCSFMUpEJIUxSkQkhTFKRCTFoHUDRGhra6uoqACg0+lmzZrV7ftLSkqcd5hMmzZt2LBhqvdH1CUdb3gizZ07dy4iIgKAXq/v6Ojo9v0Gg8FutwMoLCwMDw9XvT+iLnFRT0QkhTFKRCSFMUpEJIUxSkQkhTFKRCSFMUpEJIUxSkQkhTFKRCSFTzGRZykpKdG6BaKeYYySB7Hb7bNnz9a6C6Ke4aKeiEgKZ6PkQby8vJKTk7t9244dOxwORx/0Q+QObk1C2uPWJNSvcVFPRCSFMUpEJIUxSkQkhTFKRCSFMUpEJIUxSkQkhTFKRCSFMUpEJIUxSkQkhQ+DkvYCAwMPHToEwMvLrd/rBw8edD4MGhgYqG5nRG7gw6BERFK4qCePFhcXFxcXp3UXRF3hbJQ8mk6nA8CfUvJknI0SEUlhjBIRSWGMEhFJYYwSEUlhjBIRSWGMEhFJYYwSEUlhjBIRSWGMEhFJYYwSEUlhjBIRSWGMEhFJYYwSEUlhjBIRSWGMEhFJYYwSEUlhjBIRSWGMEhFJYYwSEUlhjBIRSWGMEhFJYYwSEUlhjBIRSWGMEhFJYYwSEUlhjBIRSTFo3QBRV8LDb2rdAlE3dEIIrXsg+r90OgDgDyl5Mi7qiYikMEaJiKQwRomIpDBGiYikMEaJiKQwRomIpDBGiYikMEaJiKQwRomIpDBGiYikMEaJiKQwRomIpDBGiYikMEaJiKQwRomIpDBGiYikMEaJiKQwRomIpDBGiYikMEaJiKQwRomIpDBGiYikMEaJiKQwRomIpDBGiYikMEaJiKQYtG6AqCuXLmndAVF3dEIIrXsgQlUV7HYAGDoUEyZ09c72dtTWAoBej1/+si96I+oaY5Q8QkAAGhoAQK/HP/+JsLD/+85z5xARAQD+/rh1q4/aI+oCz42SZ7HbERvrmpkS9QuMUfI4584hM1PrJojcxhglT7RlC+rqtG6CyD2MUfIsy5cDQGMjNm3SuhUi9zBGybMkJWHkSAA4dAhffql1N0RuYIySZwkIeDQPTUjAgweadkPkBsYoeZw//QlTpgBAZSX+/GetuyHqDmOUPI6PD957z/U6JYUPMpGnY4xSX3PngY+VK7F4MQDYbIiLc+sQ9wcnUhZjlPrO3btITMSaNW69edcuDBkCAMeO4cABd0v8/veIjeXTTdSnGKPUFzo6kJ6OoCCkpiI7G5WV3R8SGoo//tH1+o03XI+Kdq2yEn/7G/bsQUgI/vpXdHRI9UzkJsYoqe7YMTz1FP7wB9y+jWefRVERpk5168CkJEyaBAANDdixo/v3T52K8+cRFYXbtxEXh7lzcfSoVOdE7mCMkoqqq7F2LRYvRkkJJk5EVhaOHsWsWe4ePnw4PvzQ9XrPHhQVdX/IzJnIz0dODgIDcfEiY5T6AmOUVNHSgjffxLRp2L8fw4fDYkFFBdau7fE4L76IFSsAwOFAQoK7V5BiYlBWhp07kZjY44pEPcWN8khhQmD/fpjNqK+HToff/AYffICJE7s5qnOjvMpKBAb+6K+uXcOMGa778A8fxuTJUhvlCQGdrsdHEXWBs1FSUmEh5s/HunWor0dEBE6eRHZ29xnatcBAmM2u15s3o72990Pl5yM8HCdPSvVD9D8Yo6SMujqsXYvISJw5g/HjkZmJggI8/bQyg2/ZgmnTAODGDeze3ftxUlNx/jwWLMCaNaipUaY3IsYoyWprQ2oqgoOxfz98fGA24/JlmEzwUu6Hy9sbaWmu1wcP9n6czz/H9u3w8cGnnyI4GMnJfGafFMAYJSm5uXjySSQmoqkJ0dEoK0NKCkaMUL7QsmV48UUAcDh6P8jw4UhORkUFjEa0tmLvXqnRiJx4iYl6qbgYGzbg+HEAmDMHaWlYsKD3o3VxialTdTVCQ9Hc7Pqj5Hcxffst2tuxdGnvRyBy4myUeuz2bSQkICICx4/jscdgteLcOakMddPEiW7dhO+mhQt/IkM5qaBeYIxSDzx8iLQ0BAbio4/g5YX4eFy7hoQE6PV91MAbb/Tg7v2estnw1FN4802pmwFoEDJo3QD1G/n52LABZWUAsGQJrFY8+aRig3/wgetqT0BAV28zGPDppzh9GgB8fBSr7vT55yguRnExDh3Czp2u2/6JusXZKHWvogLR0XjuOZSVYdo05OYiL0/JDAWwYgWammAyub5BpAszZsBkQlOT8jH329/iyy8RHOz69y5fjitXFC5BAxJjlLpy7x4SEzFzJr74Aj/7GVJSUFKC6GglS3R0YM8eBAdj0yYcPuzWIdnZ2LQJU6ciNRU2m5LNLF+O0lJkZsLfH//4B2OU3COIfordLrKyxJgxAhBeXsJoFP/5j/JVvv5ahIYKQABi6VJRXu7WUeXlYulS11GhoeLrr5Vv7OZN8eGHyg9LAxJjlH7CsWPHw8JszpxatEhcuKBKlb//3RWFQUEiJ6fHh+flPYrgJUtEWZkKLf6AzWY7fvy4ujWof+Kinn6kurp67dq1zz67YMSIdOfWdseOYfZsVWrFxCAyEhYLSksRE9Pjw5cswYULsFoxahTy8zFnDhIScP++Co0CAD7++OMFCxbExMRcv35drRrUT2md4+Qpmpqatm7d6uPjA8DPz+8vf0lra1O9qMOhwCANDSI+Xuj1AhCPPSasVtHRocCw/yMtLc3Pzw+Aj4/P1q1bm5qalK9B/RNjlITD4cjOzp40aRIAnU63atWqqqoqxaucOCG++ELxUR85f1786leuNf7cueLbb5UvUVtbazQadTodgPHjx3/33XfK16B+iDE62BUWFj79/UZM4eHhJ0+eVLxETY0wGoVOJ8aOFffvKz78j+TkiMmTXWEaHS1u3FC+RGFh4fz58+fPn+9QZC5N/R9jdPCqra01mUxeXl7OuVVmZqbdble2REuLsFiEr68AhK+vsFhEa6uyFX5Ca6tISRF+fgIQw4YJs1k0NipcwuFwNDQ0KDwo9VuM0cGovb3darWOGDECwNChQ+Pj4++rM0v89a8FIHQ68bvfiT5eAXdOgQHxi1+IrCxlzsN24dSpU219cDqZPA9jdNDJycl54oknnKv46Ojoa9euqVfr6FERFiY0vE3o7Fkxb55rjR8RIU6fVqtQfX39yJEjJ0yYkJWVxcX+YMMYHUTKy8uff/55Z4CGhIR89dVXfVBU80hxOERWlhg3zjUvNhrFv/+tfJWSkpJZ32+asmjRogsq3WpLHokxOijcvn07Pj5er9cDGD16tNVq7VD6nqD2drFzp7h3T9lRFdPcLCwW4e0tADF8uLBYhOLr746OjoyMDH9/fwB6vT49PV3hAuSpGKMDnM1my8zMdH62DQaDyWS6deuW4lXy8kRIiADExo2Kj62kq1fFqlWuNf7UqSI7W/kSd+/eNZvNvr6+paWlyo9OHokxOpDl5eXNmDHDudKMiopS44N98aJ47rlHj7fn5SleQXlHjoiZM109L14sSkqUL6HG7yryWIzRgamiomLVqlXOAA0KCspWY94lxL/+JYYMEYAYPVp89JF4+FCNIqp4+FBkZoqAAAEIg0GYTOLmTXUr1tTUXL58Wd0apBHG6EDT1NRksVi8vb2dz3RaLJYHDx6oV27NGmE0qp5BKrlzR8THC4NBAOLnPxdWq4q/CV566aUhQ4bEx8ffvXtXrRqkEcbowGG327OyssaOHQvAy8vLaDTW19erUeWHf9T8Qry80lIRFeVa48+YIfLzFX4GQQhht9s7n3QYM2bMnj17FH/SgTTEGB0gCgoKIiMjnav4yMjIgoICxUtUVVUZjcZt27YpPrInyMkRgYHOjQG3RkdHV1ZWKl7i/PnzC77/5r/IyEjFb5YgrTBG+73q6urO/TJUuv27sbHRbDY7TxQEBAS0tLQoO76HePBAvPdei79/AABvb2+z2dyo+GOkQuTk5EyZMmXz5s2Kj0xaYYz2Yy0tLRaLZdiwYQB8fX3NZrMau7d98sknjz/+uPNEwcsvv1xXV6d4CY9SV1dnMpmc99j6+/urcY9ta2trc3OzsmOShhij/diBAwecW9utXr26urpapSorV64EEBERcerUKZVKeKCioqJnnnnGuQAPCws7ceKE1h2R59IJIZTbA5r6lMPhMJlMr7zySudOd2q4cePG6dOnV69e7TxvMKjk5uauX7++qqoKQHR09K5duyZPnqx1U+RxGKP0Iw6Hw3lBmZxaW1t37dr1zjvvNDc3+/r6rl+/fvv27c5t8Imc+IGhR3Jzc0NDQ6/wa4V/wHnS+dKlS0ajsa2tLTU1NSQkZN++fZx/UCfGKAFAcXHxwoULX3jhhStXrmRkZGjdjseZMGHCvn37zpw5ExkZWVNTs27dunnz5hUUFGjdF3kGjc/NktZu3br16quvdt4ZnpmZyfsZu9A3zzhQ/8Jzo4NdbW3t9OnTbTbb66+//vbbb48aNUrrjvqB5ubm999/PyUlpb293c/Pb+PGjVu2bHHeV0uDEGOU8Nlnn82ZM2f69OlaN9LPXL16ddu2bYcPHwYQFBT07rvvdm4HQ4MKY3TQ4bV4ZR05cmTDhg0XL14EEBUVZbVaOzcnpEGCH6dB5N69e4mJiStWrNC6kQElKiqquLjYuTf2kSNH5s6dGxsb29DQoHVf1Hc4Gx0U7Hb73r17d+zY0dDQoNfri4qKZs+erXVTA82dO3feeuut9PR0u90+evTopKSkuLg4g8GgdV+kPm2vcFEf+Oabb/hta32mvLx82bJlzv/t4ODgvvneQNIWF/UD39mzZ0tKSiZOnJiVlXX06FHOQ1Xl/MpV57dYX758efny5TExMdevX9e6L1IRF/UDX3t7++7du2NjY318fLTuZRCx2WwZGRlJSUmNjY1Dhw597bXXkpOTR44cqXVfpDzGKJGK6urqEhMTDxw4IIQYP358UVHRuHHjtG6KFPZfKtinoJpHzM8AAAChelRYdHJka2l0UEtMIHJka2l0IDIwMjAuMDkuMQAAeJx7v2/tPQYg4AFiRgYIYAFiZiBuYGQU0AAJMEryAknHkvxcEM5M9ivNVZSCqZZh0GaHKGPBpowdoYxNIAPEYmLCpo6NdHXcDIwCkpxAtlN+Xop/UUpqEYqrGJkkeNCleZCkmfFKizCIw8wCBwgQHLBnQAMP3ZbtZ2Bw2A9l28PYYgDCPzFDTFn2bQAAAIl6VFh0TU9MIHJka2l0IDIwMjAuMDkuMQAAeJzjUgCBIBfvzBIFODBy4eJSUDBRUDBWUDDAiiwtLRXCjAwMDLhA6g30gCwDEMtQzxTKgop5KOAyAhmhmYLB8iPaFF1DPSNLS5BeXQM9c1MUU5yJdws1TDEEBiSYBHGMwIFpAueYQDm+Cgqufi5cAOjWPIWzaRCzAAAAS3pUWHRTTUlMRVMgcmRraXQgMjAyMC4wOS4xAAB4nIv2iPXTsHLWtHJWqNEw0DHUM9WxNtAx0LHWNdQzsrQ0MNHRNdAzBwqicjVrAGvDDCHeo3jBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<rdkit.Chem.rdchem.Mol at 0x7f5d182ef280>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Chem.MolFromSmarts(Chem.MolToSmarts(Chem.MolFromSmiles('[H]n(c)c',sanitize=False)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T14:20:32.262521Z",
     "start_time": "2021-05-06T14:20:32.002225Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Frequency</th>\n",
       "      <th>Popularity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cnc</th>\n",
       "      <td>2820013</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cOC</th>\n",
       "      <td>2743793</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC(C)=O</th>\n",
       "      <td>1765485</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]OC</th>\n",
       "      <td>2176832</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cCl</th>\n",
       "      <td>1362946</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC</th>\n",
       "      <td>1693827</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CN(C)C</th>\n",
       "      <td>993693</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]Oc</th>\n",
       "      <td>756164</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c=O</th>\n",
       "      <td>915341</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cn(c)C</th>\n",
       "      <td>933230</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(C)=O</th>\n",
       "      <td>783163</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]n(c)c</th>\n",
       "      <td>789959</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CF</th>\n",
       "      <td>1034442</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cF</th>\n",
       "      <td>864157</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]O[H]</th>\n",
       "      <td>280318</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)C(C)=O</th>\n",
       "      <td>349740</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cBr</th>\n",
       "      <td>616336</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCl</th>\n",
       "      <td>336597</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N([H])C</th>\n",
       "      <td>444994</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]Cl</th>\n",
       "      <td>254627</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(C)=O</th>\n",
       "      <td>560338</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cSC</th>\n",
       "      <td>288614</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]OC(C)=O</th>\n",
       "      <td>663587</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(=O)N(C)C</th>\n",
       "      <td>252191</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)C</th>\n",
       "      <td>432051</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC#N</th>\n",
       "      <td>293726</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(c)=O</th>\n",
       "      <td>331213</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C([H])=C([H])C</th>\n",
       "      <td>458134</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(=O)OC</th>\n",
       "      <td>435087</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC(=O)N(C)C</th>\n",
       "      <td>281355</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C</th>\n",
       "      <td>488125</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>coc</th>\n",
       "      <td>598414</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c[N+](=O)[O-]</th>\n",
       "      <td>624271</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC(=O)OC</th>\n",
       "      <td>693328</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N([H])c</th>\n",
       "      <td>506835</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cN(C)C</th>\n",
       "      <td>559314</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>csc</th>\n",
       "      <td>729693</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC(C)=O</th>\n",
       "      <td>473975</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(=O)OC</th>\n",
       "      <td>463462</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC(=O)C(C)(S(C)=O)S(c)(=O)=O</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C([H])=C1CC(c)=NN1C(c)=O</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]S[Mo](=O)(=O)S[H]</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]/C(SC)=C(/C(=O)OC)N([H])[H]</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C(=O)/C([H])=C(\\[H])SC(c)=O</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C1=C(OS(c)(=O)=O)CN(C(=O)OC)C([H])=C1[H]</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C1(SC)C(=O)C(=O)N1c</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]/C(Cl)=C(/c)B(Cl)Cl</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC(=C(C)F)C(F)=C(F)[Zn]Br</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]/C(C)=C(C)/C(C(C)=O)=C(/[H])N([H])C</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]P1P([H])[Si](C)(C)O[Si]1(C)C</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Frequency  Popularity\n",
       "FragmentSmiles                                                    \n",
       "cnc                                            2820013         6.0\n",
       "cOC                                            2743793         5.0\n",
       "COC(C)=O                                       1765485         4.0\n",
       "[H]OC                                          2176832         4.0\n",
       "cCl                                            1362946         3.0\n",
       "COC                                            1693827         3.0\n",
       "CN(C)C                                          993693         2.0\n",
       "[H]Oc                                           756164         2.0\n",
       "c=O                                             915341         2.0\n",
       "cn(c)C                                          933230         2.0\n",
       "[H]N(C)C(C)=O                                   783163         2.0\n",
       "[H]n(c)c                                        789959         2.0\n",
       "CF                                             1034442         2.0\n",
       "cF                                              864157         2.0\n",
       "[H]O[H]                                         280318         1.0\n",
       "[H]N(c)C(C)=O                                   349740         1.0\n",
       "cBr                                             616336         1.0\n",
       "CCl                                             336597         1.0\n",
       "[H]N([H])C                                      444994         1.0\n",
       "[H]Cl                                           254627         1.0\n",
       "CC(C)=O                                         560338         1.0\n",
       "cSC                                             288614         1.0\n",
       "[H]OC(C)=O                                      663587         1.0\n",
       "CC(=O)N(C)C                                     252191         1.0\n",
       "[H]N(c)C                                        432051         1.0\n",
       "cC#N                                            293726         1.0\n",
       "[H]N(C)C(c)=O                                   331213         1.0\n",
       "[H]C([H])=C([H])C                               458134         1.0\n",
       "CC(=O)OC                                        435087         1.0\n",
       "COC(=O)N(C)C                                    281355         1.0\n",
       "[H]N(C)C                                        488125         1.0\n",
       "coc                                             598414         1.0\n",
       "c[N+](=O)[O-]                                   624271         1.0\n",
       "cC(=O)OC                                        693328         1.0\n",
       "[H]N([H])c                                      506835         1.0\n",
       "cN(C)C                                          559314         1.0\n",
       "csc                                             729693         1.0\n",
       "cC(C)=O                                         473975         1.0\n",
       "[H]N(C)C(=O)OC                                  463462         1.0\n",
       "cC(=O)C(C)(S(C)=O)S(c)(=O)=O                         1         0.0\n",
       "[H]C([H])=C1CC(c)=NN1C(c)=O                          1         0.0\n",
       "[H]S[Mo](=O)(=O)S[H]                                 1         0.0\n",
       "[H]/C(SC)=C(/C(=O)OC)N([H])[H]                       1         0.0\n",
       "[H]C(=O)/C([H])=C(\\[H])SC(c)=O                       1         0.0\n",
       "[H]C1=C(OS(c)(=O)=O)CN(C(=O)OC)C([H])=C1[H]          2         0.0\n",
       "[H]C1(SC)C(=O)C(=O)N1c                               1         0.0\n",
       "[H]/C(Cl)=C(/c)B(Cl)Cl                               1         0.0\n",
       "cC(=C(C)F)C(F)=C(F)[Zn]Br                            1         0.0\n",
       "[H]/C(C)=C(C)/C(C(C)=O)=C(/[H])N([H])C               2         0.0\n",
       "[H]P1P([H])[Si](C)(C)O[Si]1(C)C                      1         0.0"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freqtable.sort_values(by=['Popularity'],ascending=False)[:50]\n",
    "# freqtable.loc[freqtable.Frequency>=100].sort_values(by=['Popularity'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-11-16T11:14:41.579198Z",
     "start_time": "2021-11-16T11:14:41.098457Z"
    }
   },
   "outputs": [],
   "source": [
    "# freqtable.to_pickle('/home/aa2133/Impurity-Project/Reaxys_Data/fragfreq.pickle')\n",
    "freqtable=openpickle('/home/aa2133/Impurity-Project/Reaxys_Data/fragfreq.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-11-18T11:05:27.603179Z",
     "start_time": "2021-11-18T11:05:27.547680Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cn(c)C',\n",
       " 'C[Si](C)(C)OC',\n",
       " 'COP(=O)(OC)OC',\n",
       " 'CC#N',\n",
       " 'cn(c)C',\n",
       " 'c=O',\n",
       " '[H]n(c)c',\n",
       " 'c=O',\n",
       " 'COC',\n",
       " '[H]OC',\n",
       " 'COC',\n",
       " 'c=O',\n",
       " 'cnc',\n",
       " '[H]N([H])c']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['COC(=O)N(C)C', '[H]OC']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAczklEQVR4nO3deVhU9f4H8PfMsDOyqPwS19hkU7TU5BqFC0opKeSFzBJLfcD8meWtnFxZXILUp7RuBb8nE+2XV+qqyc/uNbd6uBZuuSCryOKCgLLEKsvM9/fHIVQEQpiZ75kzn9cfPjhn5swbeXj7Pef7nXNkjDEQQgjpKTnvAIQQYtioRgkhpFeoRgkhpFeoRgkhpFeoRg1bSUkJ7wiEGDsT3gFID/3++++7d+9+7733GGMuLi7e3t7Ozs5eXl7e3t5eXl6Wlpa8AxJiLGS04MlARUdHx8TEmJubNzY2tttkYmIiVKqHh4eHh4enp6eHh4eNjQ2XnIRIHtWoQaqqqnJycqqqqvr55599fHyuXr2akZGRmZmZn5+fkZGRk5OjVqvbvcTe3l4Yq7YNWp2cnGQyGZf8hEgJ1ahBWr169aZNmwIDA//9738/vLW5ufn69etCsQp/5uTk1NbWtnuaubm5cDagrV69vb0tLCz08h0QIh1Uo4anvLzcycmppqYmLS1t/Pjx3XxVcXFx23BV+CI/P7/dc0xMTIYOHXr/OVYfHx86G0BI16hGDc+KFSs2b94cFBSUkpLSm/1UVlbe36p0NoCQnqEaNTC3b992dnauq6s7derUuHHjtLvzhoaGnJyc7OzsrKys7Oxs4euHp7Ds7Ozc3d29vLzc3d39/f19fX21G4MQw0I1amD+9re/ffTRRyEhIfv27dPPO7Y7G3D58uX7F6uGhIS8++67EyZM0E8YQkSIatSQ3Lp1y9XVtaGh4fz586NGjeIV4/bt25mZmdnZ2fv37z98+HBYWNjevXt5hSGEO1p+b0g2bdpUX18fFhbGsUMBODg4+Pv7+/v7jxkz5vDhw1lZWRzDEMIdjUYNxvXr193c3Jqbmy9cuDBy5EjecQCgtrbWxsbGzMysrq5OoVDwjkMIH/SZeoOxcePGxsbGOXPmiKRDASiVysGDBzc2NhYWFvLOQgg3VKOGoaio6KuvvlIoFGvXruWd5QEeHh4A6LieGDOqUcMQGxvb1NT06quvCrUlHp6enqAaJcaNatQAXL16ddeuXQqFYtWqVbyztCfUenZ2Nu8ghHBDNWoAYmJiWlpaXn/99eHDh/PO0p4wGqUaJcaMZurF7sqVK15eXjKZLCcnx8nJiXec9kpKShwdHe3s7CorK3lnIYQPGo2KXVRUVEtLy6JFi0TYoQAGDBhgb29fVVVVWlrKOwshfFCNilpGRsbevXvNzc1XrlzJO0un6PQoMXJUo6IWHR2t0WgiIyOHDBnCO0unaM0TMXJUo+KVnp6+b98+CwuLFStW8M7SFZplIkaOalS81q5dq9FolixZMmjQIN5ZukKjUWLkaKZepH777bexY8daWVldvXr1scce4x2nK3l5eW5ubkOHDi0qKuKdhRAOaDQqUmvXrmWMLV26VOQdCsDJycnCwuL69esP3+6JEGNANSpGZ8+e/de//qVUKt955x3eWf6cQqFwdXVljOXk5PDOQggHdL1RMVq1ahVj7O2333ZwcOCdpVtGjx7b0mJWUNAwZgzvKIToHY1GRefkyZNHjhyxtbVdvnw57yzd5eT0VXb2uQsX/HgHIYQDqlHRES6Ft3z58r59+/LO0l2engBAS55ES6PRHDt2jHcKyaIaFZfU1NQTJ07Y2dm99dZbvLM8AuHqfbTkSbT27t0bEBAwZ84c3kGkiWpUXNasWQPgvffes7Oz453lEbi7Qy7HlStobuYdhTxErVbHxsYCmDZtGu8s0kTrRkXkxx9/DAwM7NevX0FBQZ8+fXjHeTROTigsRHY23N15RyEP2rVr1/z5811dXbOyskxMaFZZ+2g0KiJRUVEAVCqVwXUo/jiup9OjYqNWqzdu3Ahg3bp11KE6QjUqFocOHUpLS3NwcHjjjTd4Z+kJYZaJTo+Kzc6dO3Nzc93c3F5++WXeWSSLalQshLNXq1atUiqVvLP0BI1GRai5uVkYisbGxtJQVHeoRkXhwIEDp0+fdnR0jIyM5J2lh2jNkwh9+eWXBQUF3t7eYWFhvLNIGdUof4yxmJgYAKtXr7a0tOQdp4fa1jzRnKVINDU1xcfHA4iOjpbL6Tddh+gfl7/vvvvuwoULQ4YMWbRoEe8sPefggP79UV2NW7d4RyEAgISEhMLCwpEjR7744ou8s0gc1ShnGo1mw4YNAFavXm1ubs47Tq/QInzxuHv3rjAUXb9+PQ1FdY3+fTnbu3fvpUuXhg0b9vrrr/PO0lt0elQ8Pv/885s3b44ZM2bmzJm8s0gf1Shnu3btArBy5UozMzPeWXqLJutFoq6uThiKxsbGymQy3nGkj2qUs6CgIJlMtn///rt37/LO0lvGdVAv4p/X3//+99LS0rFjxz7//PO8sxgFqlHOpkyZMmDAgMOHD8+aNcvQm9QoVuD/+iuCg2FrC0tLmJhg1Chs2YLGRt6x7qmtrd26dSuATZs20VBUP6hGOfPw8Dh+/Lijo+OPP/5o6E06bBisrFBcjN9/5x1FR3buxDPPIC0Nb7yBxERs2QJ7e7z3HqZORX0973Cttm3bVlZW9vTTT0+dOpV3FqPBiAhkZWU5OjoCmDZtWkNDA+84PTd6NBsyhGVm8s6hC1lZzNyceXuzO3ceeDw6mgFs+XJOsR5QVVUlXKb2+PHjvLMYEapRsZBGkzY18U6gO//93wxgJ060f1ytZt7ezNqaVVdzSPUg4eo2kydP5h3EuNBBvVgY9NH9wYOQyeDoiLq6Bx5fvx4yGS5f5hRLu44dg7U1/P3bPy6XY8YM1NUhLY1HrHuqqqq2b9+OPy4VRvSGalREDLpJAZSUYM0a3iF0p6AAQ4eiw0mbYcNan8DVli1bKisrp02b9uyzz/JNYmyoRsWlXZM2NDTwTvQIRozA55/j9GneOXSBMTQ1wcam463CDVy5/rDKy8uFoahwfQaiT1SjonN/kwYHBxtQk77zDvr3x+LFUKt5R9E6mQw2Nigu7nhrRQUA2NvrM1E7H374YU1NzYwZM3x9fTnGME5Uo2Lk4eFx4sQJg2tSGxts3Ijz5/HJJ7yj6IKPD27dan/2V/Dbb61PYIzL7aju3Lnz2WefAVi3bp3+351QjYqUu7u7ITbpggUYNw7r1uHGDd5RtEhoxuBgtLQgKan91ooKfPcdXFzg44P9++HqisREtLToM+AHH3xQW1sbHBz81FNP6fN9SSveSwVIV7Kzs9tWQdXX1/OO06nvv2cA++c/GWPszBkml7PZsxljLDaWASw9nW+6XqirYyoVmzyZaTSsupo5OTGlkv3ww70nlJWxgAAGsH/8gzHGQkMZwADm6sp27WItLXrIWFxcbGVlJZPJzp8/r4e3Iw+jGhU7g2jS+2uUMfbGGwxghw8beI0eOsSGDGEAMzNj584xxlhGBnv8cQYwHx8WGsoCA5m1NTMxYfHxrS9Rq1lyMvPwaC1TJyeWkKDrMn3zzTcBhIaG6vRdSBeoRg2AaJtUo2H797Pz59vXaEUF+6//YiNHtn7AJz2dHT3awbp18aqsZBERrVU4ejT77bd7m+rrWUICmzOHTZzIZsxgK1eyrKz2LxfKdPjw1j14ebGkJKZW6yLpzZs3LS0t5XL5xYsXdbF/0h1Uo4ZBhE165AgbO5YB7Lnn2tcoY2znTgYwNzcGsIsXmacnA9jTTzMD+IziwYNs4EAGMEtLFhfX87GkUKaurq1lOmIES05mGo1Ws7LFixcDePnll7W7W/JIqEYNhnia9PBhNn58azkMGsQ+/bSDGtVo2MSJrc85d45t2MDs7Vv/GhjI0tL4pe/CrVts9uzWlM88w7KztbDPpiaWlMScnVt36+OjxTItKioyMzNTKBRZD4+IiR5RjRoS7k2amsomTWotBAcHFhfHhBQP1yhj7PJlZmp679xoTQ2Li2N2dq0vDwhgZ87o/zvoXHIy69ePAczWln38sZaPwRsbWUICGzSo9Zv39WUHD/Z+rwsXLgQQHh7e+12R3qAaNTC8mvTkSTZlSmsJ9OvHoqIeuBBHZiZTqTqYSkpKYioVKym590h1NYuLY7a298pUmLzhKT+fTZ3aGmj6dHbtmq7eSChTR8fW95owgR092uOdFRQUCEPRbK2MmkkvUI0aHj036a+/sqCg1l/8vn1ZVBSrqurtPu/cYVFRzMaGAUwmY0FBjM9aHbWaJSQwpZIBzN6eJSTo401ra1l8POvfnwHM2vq1mTP/85//9GA38+fPB7Bw4UKtBySPimrUIOmnSS9eZKGhTCZjAOvTh6lUrLJSm/u/fZupVMzSkgFMLmehodo5G9ldubnM37/1/4fQUFZWpsf3Zqy6mm3YkPrcc8Ly7cDAwLRHOWGcm5trYmJiamp69epV3WUk3UQ1aqh02qTp6fcKVKlkKhWrqNDuO9xTWspUKmZhca9Mc3N19V6CpqamG5s3M3NzBrDBg9n//Z9u369zNTU1cXFxdnZ2QpkGBAScPn26Oy+cO3cugMWLF+s6IekOqlEDposmzcho/utfWwvU2pqpVOz2ba3s+E9cu8YiI5mZGQOYqSlbtIgVFjbr4o3Onz//5JNPjnZ3bzIzY/Pm6fD/h26rrq6Oi4uztbVtK9NzXZ4wzsjIkMvlZmZmBQUF+spIukI1ati02KT5+fkRERHjxj0vfGwnIoIVF2srZncVFbGICGZiwhQKNnz4zHnz5mnxoLW+vv7dd99VKBQAXF1dC3t0RlJ37ty5ExUVZWNjA0AmkwUFBXX24c7Q0FAAS5cu1XNC0hmqUYPX+ybNy8sLDw8X+sXc3Hz16gL9F+j9rlxhK1Zcacvz5ptvFvc6UGpqqru7OwC5XB4REVFbW6uVqFp3+/ZtlUplaWkpRA0NDW03EZ+eni6Xyy0sLG7cuMErJGmHalQKetykRUVFERERJiYmAExNTefNm5efn6+7nI9EGB0LZWpmZhYREdGzMv3999+XLVsml8sBjBgx4tSpU1qPqnWlpaUqlcrCwqKtTHP/OGEcHBwMYLk47qBHBFSjEvGoTXr9+vVly5YJv6hCgebl5ekh56PKyMiYN2+eUILW1tbLli0ruX8Z6p85dOjQ0KFDhe9RpVI1NjbqLqrWPfwzOnDggEwms7a2fqR/BKJrVKPS0c0mLSsra3fYmKvrqfFeS09PDw0NlclkQpmqVKry8vKuX1JRURERESFM2vj6+l6+fFk/UbWuoKBgwYIFwhGD8N8JTdCLDdWopHTdpH963k3kLl261FamSqVSpVJVdrKQNTk52cHBAYClpWVcXFyLXq77qVMFBQXBwcFyuVyYo4+IiKBzo+JBNSo1HTZp92eBxS8tLS0oKEgYZvbp00elUlXd97Gq4uLiF198Udj67LPP5uTkcIyqXYGBgQC8vb2FMamlpeXy5ctLS0t55yJUo1J0f5OWlpa2W5N49uxZ3gG14Jdffmkr0379+kVFRVVVVSUlJfXt2xeAra1tQkKCRttXpePo5MmTAGxsbMrLy4XzxcLkm5WV1aOeLyZaRzUqTRkZGY899piwYEjomunTp58R1yWVtOD48eN+fn7CN6hUKoUvgoODe79ASmwmT54MICoqqu2Ry5cvP+r5YqIjVKOSlZGRYW9v379/f19f359++ol3HO1LSUnZsGHDxYsXU1NT/f39Q0JChg0blpSUxBibMGGCtbV1ZmYm74zakZqaCsDOzq7ioc9cdf98MdEdqlHJqqysFA761Lq5fQV3r732GoAdO3YIf62rq2tqahK+HjduHACDWCLaHf7+/gA2bNjQ2RO6Pl9MdI1usCxZWVlZADw8PIQZCenp06cPgJqaGuGvVlZWpqam92+qrq7mlU2Ljhw58vPPP/fr12/ZsmWdPWf8+PEpKSnC+eKampr4+HgXF5fo6Ghp/AuInzR/wQiA7OxsAJ6enryD6Iqw8KDDphA2tTWsQVu/fj2AFStWCP83dOEvf/lLSkpKamrq5MmTy8vLY2JiXFxc4uPjGxoa9JLUeFGNSpZQox4eHryD6Eq70ejDmyQwFvvhhx9SU1P79++/ZMmSbr7Ez8/v2LFjwvniO3fuvP/++48//nh8fPzdu3d1GtWYUY1KVttBPe8gutJFjUpmNBoTEwNg5cqVbesQusnPz++nn346cuTIuHHjysrK3n//fTc3t23btjU2NuomqVGjGpUsyR/UdzHk7KJhDcj3339/+vRpR0dH4S7KPSBcB/rIkSNjxoy5cePG22+/PXz48MTExJaWFu1GNXJUo9LU1NRUUFCgUChcXV15Z9GVLoacEqhRxlh0dDSAlStXWllZ9WZXAQEBZ86cOXjw4BNPPHHt2rXIyEg3NzcqUy2iGpWm3NzclpYWZ2fntuX30vOno1GDPje6b9++CxcuDBw4cNGiRb3fm0wme+GFF86ePZucnOzh4VFYWBgZGSmMTNVqde/3b+SoRqVJ8kf06HI0aujnRjUaTWxsLIC1a9cKl5LRCuF6NBkZGcnJycOHDy8oKIiMjPTx8dm1axeVaW9QjUqT5OeXIOmZ+uTk5EuXLg0dOnTBggVa37lQpllZWcnJya6urpmZmfPnzx81atS3337LGNP62xkDqlFpkvxqJ0h33aharRaGouvWrTMzM9PRuwhlmpmZmZSU5OzsnJGRERYWRmXaM1Sj0mQMB/VSHY1+8803WVlZLi4u4eHhun4vU1PT8PDwrKyshISEQYMGpaenh4WFCcv4df3WUkI1KkGMsdzcXADCTdykytraWqFQ1NfXP3xez3BHo2q1euPGjQDWrVvX9tlWXROuA52fn5+QkODo6Hjq1KmZM2dOmDCByrSbqEYl6Nq1a7W1tQMGDLC3t+edRYeEuxIxxh6uS8Nd8JSUlJSTk+Pm5jZ37lw9v7VQpnl5eR9//PGAAQN+/fXXmTNn+vn5HT9+XM9JDA7VqAQZwxG9oLNRZxenTcWsublZGIrGxMQIN1/SPysrq7feeuvKlSsbN27s27fvyZMnp0yZsmbNGi5hDAXVqAQZwzS9oLNzoEqlUiaT1dbWajQaHrl6aMeOHfn5+V5eXi+99BLfJEqlctWqVUVFRXFxcfb29m1X4SMdohqVIGOYphd0NhqVy+XC8X5dXR2PXD3R1NQUFxcHIDo6WiTXNhSuA339+nVfX1/eWUSNz4ED0SnjOagfN2TIoIoKk466Mm3ixD4VFWbV1fiz68uJRGJiYmFh4YgRI2bPns07ywOsra15RxA7qlEJMp6D+k80Gly5gqqqhzd5X7mCnBwYyCzT3bt34+PjAaxfv14kQ1HSffQDk5qKioqysjKlUjl48GDeWXRPGGl22JXCJgOZZfriiy9u3Ljx5JNPzpo1i3cW8sioRqWm7cSocJszieuiK21sgE4aVmQaGho2b94MIDY21ih+apJDNSo1xnNED3TZlYYzGv3000+Li4vHjh07ffp03llIT1CNSo3xTNMDXR7UG8hotK6ubsuWLQA2btxIQ1EDRTUqNcJo1Bim6YEuh5wGMhrdtm1bWVnZhAkTpk2bxjsL6SGqUakxrtFoF0NOQxiNVldXb926FcCGDRt4ZyE9RzUqKXfv3i0sLDQxMXFxceGdRS/+dDQq7hr96KOPKioq/Pz8Jk2axDsL6TmqUUnJzc1Vq9UuLi4SvnfIAwx5iqmqqmrbtm2goajhoxqVFOM6oodhL3jaunVrZWXltGnT/P39eWchvUI1KinGNb8EAx6NlpeXb9++HX/ciZ4YNKpRSaHR6D3iHo1u3ry5urp6+vTpdNUPCaAalRTjuShJK8Mcjd65c+ezzz4DEBUVxTsL0QKqUenQaDTGcO+QB1hbQ6FAfT0evj+wiEejH3zwQU1NzaxZs5566ineWYgWUI1KR1FRUX19/cCBA21tbXln0ReZDNbWYKyDuhTraLSkpOSLL76QyWTR0dG8sxDtoBqVDqM7ohd0NuoU62h006ZN9fX1s2fPHj16NO8sRDuoRqXDuC5K0qazUadSCZkMtbUQ2X1EJk+e7OPjQ2dFpYQu2ywdRjdNL+hs1CmXIzgYJiZoaoKFhf5zdSY4ODg4OJh3CqJNVKPSYaQH9V9/DbkcAwd2sGnfPr2nIcaIDuqlw0gP6l1d4eyMxkbExODJJ2FnBxsbjByJlStx+zbvcMQoyBhjvDMQLSgvL+/fv7+NjU1VVZXRXbby+nVMnoyrV/H883j6aSgUOH0aBw5gwAAcPQpjG54TvaODeoloG4oaXYcCeOUVFBRg/37cfyOjY8cwYwbCwnDhAhQKfuGI9NFBvUQY6RE9gF9+QWoqXnsN7W4GN2UK3nwTly8jJYVTMmIsqEYlwnhr9MgRAOhw7nvOnHtPIERn6KDe4OXm5n7zzTc7duxwcHAwxhq9ehUAhg7tYJNwVjQvT695iPGhGjVUNTU133777Y4dO06ePCk8sm3btpCQEL6pOGhoAIC+fTvYZGYGmQz19XpORIwN1ajhOXfuXGJi4p49e2pqagDY2NjMmjUrPDx8ypQpvKPxIHyK6dYtDB7cftPNm2AMxnOFAcIJ1ajBKCkp2bt375dffpmeni48MmbMmIiIiLlz5yqVSr7ZeBoxAgDy8zFuXPtNV67cewIhOkPrRsVOrVafOHEiMTHxwIEDzc3NABwdHcPDwxcuXOjm5sY7nQjk5MDTE88/j0OH2m8KD8fu3Th5EhMm8EhGjAXVqHhlZ2fv3Llz586dpaWlABQKxaRJkyIiIkJCQkxM6DDiPvPm4euvsWkTVCrI/1h88sUXWLIEzz2HH37gGo5IH9Wo+FRXl3//fdBnn6WlpQkPjBgxYsGCBa+++qqDgwPfaCJVU4OQEBw7Bjc3+PlBJsOZM0hPh68vUlLQvz/vfETiqEbF5Nw5JCbim29QX+87YEBWba0wdxQQEMA7mehpNPjuO+zfj7w8aDRwdsYLL+CVV+jzS0QPqEZF4OZNJCXhq69aVzjKZPD3z16yZFhQkKWlJe9whJA/QTXKj1qNEyeQmIj9+9HSAgADB2LePCxaBFdX3uEIId1FNdql7Gzs2YOQELS738O1a/jySwQG3psCzs7GwYPIy4NajWHDMH06xo7tdLdZWa3Dz7IyADAzw7RpCA9HSAho7ogQQ0O/tF3KykJsLJyc2tdoURFiY6FUYsIEaDR4911s2waFAp6ekMuxZw+iovDSS/jqK9x/VF5djQMHsHs3jh5tfcTTE/PnY8EC0NwRIQaLarTXoqLw0UeYMweffNI6KVxbi1Wr8MknkMmwZw/wx9zR//4v6uoAwNYWL72EefPg58czOSFEG6hGe6ekBB9+iDFj8PXX9yaFlUps346iIvzjH3jnHdy+jenTAUAux5QpeP11vPgiaO6IEKmgC+X1TkoKmpoQGdnBwpqlSwHg228xdSrGj8e6dcjLw9GjeOUV6lBCpIRGo91w7RrOn3/gEeHD2gAuXgQAd/cOXuXlBQCXLsHEBH8spCeESA/VaDdERaGzu4pXVQHAoEEdbHJ0hIkJKit1GIwQIgJUo92wZg1mzHjgkYsXsXgxgNYboFdXd/AqxqBWi+oO6YQQXaAa7QYXF/j6PvBIc3PrF8JF169dwxNPtH9Vfj4Yw7Bhus9HCOGJpph6Z+JEAPfWgd7v4EEAmDRJn3EIIfpHNdo7zzyD0aPxP/+Ds2cfeLygAHFxGDgQYWGckhFC9IQO6ntHJsPu3Zg4ERMnYulS+PtDocCZM/j4Y9TXY+9eWFnxjkgI0S2q0S7JZJDL710JuMPHR4zAqVNYuRLbtyM+HgBMTREQgA8+wKhR+g5MCNE7ujSJ9jQ14cYNtLRgyBBaYE+I8aAaJYSQXqEpJkII6RWqUUII6RWqUUII6RWqUUII6RWqUUII6ZX/BwO0BGH4jFeXAAABH3pUWHRyZGtpdFBLTCByZGtpdCAyMDIwLjA5LjEAAHice79v7T0GIOABYkYGCOCD4gZGNoYEkDgzG4MCkGZBcCE0B4MGkGZmYoPQLDA+O4SGqmNiwk4zM8L4HBDzGLmBTmBkAhrMwMjCwMKqwMqmwcTKrsDOwcDBycDJxcDFzcDNw8DDy8DNziDCyMrOwcnFLQ5zNQOfVXW+g1DGKnsQp/GFs0O8+9rdIPaJUAmHBYdW7wexp+/scSheqg9mr9x9xz7sgw5Y/aGH7+zOfhXYB2IrTeW2F7h9Eazm4s64/YW8XGA187N790eEMDqA2BvaVQ/whgmA2ZlSgQeadbPBajrMeA/4uwuCzbGzkDmw6sx5sDlPrpUecAzgOgBiiwEANPA+dzveJB8AAAEhelRYdE1PTCByZGtpdCAyMDIwLjA5LjEAAHicnZM9bgQhDIX7OQUXWMt/GFxnUkXZSClyh/S5v2IWFlEk0i4Ijd6Tx58eY+ZIbX2eb98/aS4+jyORtp3wz+3u6YsR8WjvCxQlb4pAhKQphKhiekn/IdY9KGhEIS6tN29SGKQiN0pkGamepyiIl9KzWDXaoxAU1tx7reS8UD4epyComfUspGR7WRCy5vFdTHwvSySoJTo6T3yd0fVxCgGh1j4tZKxbJ7ow5GraKZw378tFgK1w73WqZTcLo+U5o71bFxRV9DEjFt09UVXizqP+Rz096Wji27OZULIaXU2eJpRNE6rc6sPUWQnl03givJtQLewwlGgmCEUzQSjSpTLR7ym9Xs/jFxqKvlDPWo8EAAAAzHpUWHRTTUlMRVMgcmRraXQgMjAyMC4wOS4xAAB4nC2PS67DMAhFt9JhIlHE54Kxoo48f9lQF/9wVMTocD/2Wsc6e+91fO7zT1ePrvv1PZwHdJKyuzpdzpIqg97CImFVEnQZe4mB3q2CVtEF9jkytiwrVRspDwOKmowIGF3CyIzMrVJopG8WiNbtrPTZ4X2sIWLezvBp2Vl9VEEFGYvJGE2MoxLaxCJs+5wtu7JtUyt3eItMEvErrIGHATL9KTSHP86C7v8Yaz8+6Pz+A1vfOhQ13uoKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<rdkit.Chem.rdchem.Mol at 0x7f748d284b80>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Frequency     281355.0\n",
       "Popularity         1.0\n",
       "Name: COC(=O)N(C)C, dtype: float64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'C[CH2-].[Mg+2].[Br-]'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['C[Mg]Br']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'C[Mg]Br'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Frequency     1157.0\n",
       "Popularity       0.0\n",
       "Name: C[Mg]Br, dtype: float64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from FunctionsDB import getCarrierFrags0\n",
    "getCarrierFrags0('Cc1cn([C@H]2C[C@H](O[Si](C)(C)C(C)(C)C)[C@@H](COP(=O)(OCCC#N)O[C@H]3C[C@H](n4cc(C)c(=O)[nH]c4=O)O[C@@H]3CO)O2)c(=O)nc1N',resFormat='smiles')\n",
    "getCarrierFrags0('CC(C)(C)OC(=O)N1CCCC1CO',resFormat='smiles')\n",
    "Chem.MolFromSmiles('CC(C)(C)OC(=O)N1CCCC1CO')\n",
    "freqtable.loc['COC(=O)N(C)C']\n",
    "getCarrierFrags0('C[CH2-].[Mg+2].[Br-]',resFormat='smiles')\n",
    "getCarrierFrags0('C[Mg]Br',resFormat='smiles')\n",
    "Chem.MolToSmiles(Chem.MolFromSmiles('C[Mg]Br'))\n",
    "freqtable.loc['C[Mg]Br']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-11-16T11:22:31.879665Z",
     "start_time": "2021-11-16T11:22:31.685780Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Frequency</th>\n",
       "      <th>Popularity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cnc</th>\n",
       "      <td>2820013</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cOC</th>\n",
       "      <td>2743793</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]OC</th>\n",
       "      <td>2176832</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC(C)=O</th>\n",
       "      <td>1765485</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC</th>\n",
       "      <td>1693827</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cCl</th>\n",
       "      <td>1362946</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CF</th>\n",
       "      <td>1034442</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CN(C)C</th>\n",
       "      <td>993693</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cn(c)C</th>\n",
       "      <td>933230</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c=O</th>\n",
       "      <td>915341</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cF</th>\n",
       "      <td>864157</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]n(c)c</th>\n",
       "      <td>789959</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(C)=O</th>\n",
       "      <td>783163</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]Oc</th>\n",
       "      <td>756164</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>csc</th>\n",
       "      <td>729693</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC(=O)OC</th>\n",
       "      <td>693328</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]OC(C)=O</th>\n",
       "      <td>663587</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c[N+](=O)[O-]</th>\n",
       "      <td>624271</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cBr</th>\n",
       "      <td>616336</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>coc</th>\n",
       "      <td>598414</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(C)=O</th>\n",
       "      <td>560338</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cN(C)C</th>\n",
       "      <td>559314</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N([H])c</th>\n",
       "      <td>506835</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C</th>\n",
       "      <td>488125</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC(C)=O</th>\n",
       "      <td>473975</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(=O)OC</th>\n",
       "      <td>463462</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C([H])=C([H])C</th>\n",
       "      <td>458134</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N([H])C</th>\n",
       "      <td>444994</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(=O)OC</th>\n",
       "      <td>435087</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)C</th>\n",
       "      <td>432051</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)C(C)=O</th>\n",
       "      <td>349740</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCl</th>\n",
       "      <td>336597</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(c)=O</th>\n",
       "      <td>331213</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC#N</th>\n",
       "      <td>293726</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cSC</th>\n",
       "      <td>288614</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC(=O)N(C)C</th>\n",
       "      <td>281355</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]O[H]</th>\n",
       "      <td>280318</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]Cl</th>\n",
       "      <td>254627</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(=O)N(C)C</th>\n",
       "      <td>252191</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC#N</th>\n",
       "      <td>249321</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C(C)=C(C)C</th>\n",
       "      <td>246559</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]OC(c)=O</th>\n",
       "      <td>239983</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C(C)=C([H])C</th>\n",
       "      <td>230197</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)c</th>\n",
       "      <td>222550</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn(c)C</th>\n",
       "      <td>217091</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c-n(c)c</th>\n",
       "      <td>214195</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CBr</th>\n",
       "      <td>207543</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cOc</th>\n",
       "      <td>205332</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)C(c)=O</th>\n",
       "      <td>202745</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C[Si](C)(C)OC</th>\n",
       "      <td>201115</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Frequency  Popularity\n",
       "FragmentSmiles                          \n",
       "cnc                  2820013         6.0\n",
       "cOC                  2743793         5.0\n",
       "[H]OC                2176832         4.0\n",
       "COC(C)=O             1765485         4.0\n",
       "COC                  1693827         3.0\n",
       "cCl                  1362946         3.0\n",
       "CF                   1034442         2.0\n",
       "CN(C)C                993693         2.0\n",
       "cn(c)C                933230         2.0\n",
       "c=O                   915341         2.0\n",
       "cF                    864157         2.0\n",
       "[H]n(c)c              789959         2.0\n",
       "[H]N(C)C(C)=O         783163         2.0\n",
       "[H]Oc                 756164         2.0\n",
       "csc                   729693         1.0\n",
       "cC(=O)OC              693328         1.0\n",
       "[H]OC(C)=O            663587         1.0\n",
       "c[N+](=O)[O-]         624271         1.0\n",
       "cBr                   616336         1.0\n",
       "coc                   598414         1.0\n",
       "CC(C)=O               560338         1.0\n",
       "cN(C)C                559314         1.0\n",
       "[H]N([H])c            506835         1.0\n",
       "[H]N(C)C              488125         1.0\n",
       "cC(C)=O               473975         1.0\n",
       "[H]N(C)C(=O)OC        463462         1.0\n",
       "[H]C([H])=C([H])C     458134         1.0\n",
       "[H]N([H])C            444994         1.0\n",
       "CC(=O)OC              435087         1.0\n",
       "[H]N(c)C              432051         1.0\n",
       "[H]N(c)C(C)=O         349740         1.0\n",
       "CCl                   336597         1.0\n",
       "[H]N(C)C(c)=O         331213         1.0\n",
       "cC#N                  293726         1.0\n",
       "cSC                   288614         1.0\n",
       "COC(=O)N(C)C          281355         1.0\n",
       "[H]O[H]               280318         1.0\n",
       "[H]Cl                 254627         1.0\n",
       "CC(=O)N(C)C           252191         1.0\n",
       "CC#N                  249321         0.0\n",
       "[H]C(C)=C(C)C         246559         0.0\n",
       "[H]OC(c)=O            239983         0.0\n",
       "[H]C(C)=C([H])C       230197         0.0\n",
       "[H]N(c)c              222550         0.0\n",
       "cnn(c)C               217091         0.0\n",
       "c-n(c)c               214195         0.0\n",
       "CBr                   207543         0.0\n",
       "cOc                   205332         0.0\n",
       "[H]N(c)C(c)=O         202745         0.0\n",
       "C[Si](C)(C)OC         201115         0.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Frequency</th>\n",
       "      <th>Popularity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cnc</th>\n",
       "      <td>2820013</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cOC</th>\n",
       "      <td>2743793</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]OC</th>\n",
       "      <td>2176832</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC(C)=O</th>\n",
       "      <td>1765485</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC</th>\n",
       "      <td>1693827</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cCl</th>\n",
       "      <td>1362946</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CF</th>\n",
       "      <td>1034442</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CN(C)C</th>\n",
       "      <td>993693</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cn(c)C</th>\n",
       "      <td>933230</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c=O</th>\n",
       "      <td>915341</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cF</th>\n",
       "      <td>864157</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]n(c)c</th>\n",
       "      <td>789959</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(C)=O</th>\n",
       "      <td>783163</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]Oc</th>\n",
       "      <td>756164</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>csc</th>\n",
       "      <td>729693</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC(=O)OC</th>\n",
       "      <td>693328</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]OC(C)=O</th>\n",
       "      <td>663587</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c[N+](=O)[O-]</th>\n",
       "      <td>624271</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cBr</th>\n",
       "      <td>616336</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>coc</th>\n",
       "      <td>598414</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(C)=O</th>\n",
       "      <td>560338</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cN(C)C</th>\n",
       "      <td>559314</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N([H])c</th>\n",
       "      <td>506835</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C</th>\n",
       "      <td>488125</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC(C)=O</th>\n",
       "      <td>473975</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(=O)OC</th>\n",
       "      <td>463462</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C([H])=C([H])C</th>\n",
       "      <td>458134</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N([H])C</th>\n",
       "      <td>444994</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(=O)OC</th>\n",
       "      <td>435087</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)C</th>\n",
       "      <td>432051</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)C(C)=O</th>\n",
       "      <td>349740</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCl</th>\n",
       "      <td>336597</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(C)C(c)=O</th>\n",
       "      <td>331213</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cC#N</th>\n",
       "      <td>293726</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cSC</th>\n",
       "      <td>288614</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COC(=O)N(C)C</th>\n",
       "      <td>281355</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]O[H]</th>\n",
       "      <td>280318</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]Cl</th>\n",
       "      <td>254627</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC(=O)N(C)C</th>\n",
       "      <td>252191</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CC#N</th>\n",
       "      <td>249321</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C(C)=C(C)C</th>\n",
       "      <td>246559</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]OC(c)=O</th>\n",
       "      <td>239983</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C(C)=C([H])C</th>\n",
       "      <td>230197</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)c</th>\n",
       "      <td>222550</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn(c)C</th>\n",
       "      <td>217091</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c-n(c)c</th>\n",
       "      <td>214195</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CBr</th>\n",
       "      <td>207543</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cOc</th>\n",
       "      <td>205332</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]N(c)C(c)=O</th>\n",
       "      <td>202745</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C[Si](C)(C)OC</th>\n",
       "      <td>201115</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Frequency  Popularity\n",
       "FragmentSmiles                          \n",
       "cnc                  2820013         6.0\n",
       "cOC                  2743793         5.0\n",
       "[H]OC                2176832         4.0\n",
       "COC(C)=O             1765485         4.0\n",
       "COC                  1693827         3.0\n",
       "cCl                  1362946         3.0\n",
       "CF                   1034442         2.0\n",
       "CN(C)C                993693         2.0\n",
       "cn(c)C                933230         2.0\n",
       "c=O                   915341         2.0\n",
       "cF                    864157         2.0\n",
       "[H]n(c)c              789959         2.0\n",
       "[H]N(C)C(C)=O         783163         2.0\n",
       "[H]Oc                 756164         2.0\n",
       "csc                   729693         1.0\n",
       "cC(=O)OC              693328         1.0\n",
       "[H]OC(C)=O            663587         1.0\n",
       "c[N+](=O)[O-]         624271         1.0\n",
       "cBr                   616336         1.0\n",
       "coc                   598414         1.0\n",
       "CC(C)=O               560338         1.0\n",
       "cN(C)C                559314         1.0\n",
       "[H]N([H])c            506835         1.0\n",
       "[H]N(C)C              488125         1.0\n",
       "cC(C)=O               473975         1.0\n",
       "[H]N(C)C(=O)OC        463462         1.0\n",
       "[H]C([H])=C([H])C     458134         1.0\n",
       "[H]N([H])C            444994         1.0\n",
       "CC(=O)OC              435087         1.0\n",
       "[H]N(c)C              432051         1.0\n",
       "[H]N(c)C(C)=O         349740         1.0\n",
       "CCl                   336597         1.0\n",
       "[H]N(C)C(c)=O         331213         1.0\n",
       "cC#N                  293726         1.0\n",
       "cSC                   288614         1.0\n",
       "COC(=O)N(C)C          281355         1.0\n",
       "[H]O[H]               280318         1.0\n",
       "[H]Cl                 254627         1.0\n",
       "CC(=O)N(C)C           252191         1.0\n",
       "CC#N                  249321         0.0\n",
       "[H]C(C)=C(C)C         246559         0.0\n",
       "[H]OC(c)=O            239983         0.0\n",
       "[H]C(C)=C([H])C       230197         0.0\n",
       "[H]N(c)c              222550         0.0\n",
       "cnn(c)C               217091         0.0\n",
       "c-n(c)c               214195         0.0\n",
       "CBr                   207543         0.0\n",
       "cOc                   205332         0.0\n",
       "[H]N(c)C(c)=O         202745         0.0\n",
       "C[Si](C)(C)OC         201115         0.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freqtable.sort_values(by=['Frequency'],ascending=False)[:50]\n",
    "freqtable.loc[freqtable.Frequency>1000].sort_values(by=['Frequency'],ascending=False)[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:24:52.212474Z",
     "start_time": "2021-12-03T02:24:47.339330Z"
    }
   },
   "outputs": [],
   "source": [
    "temp.reset_index(inplace=True)\n",
    "temp.drop(columns=['index'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:25:03.898170Z",
     "start_time": "2021-12-03T02:25:03.519931Z"
    }
   },
   "outputs": [],
   "source": [
    "temp.set_index('SubstanceID',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:27:55.895143Z",
     "start_time": "2021-12-03T02:27:36.387964Z"
    }
   },
   "outputs": [],
   "source": [
    "temp['Smiles']=substancedb.loc[substancedb.index.isin(temp.index)]['Smiles']\n",
    "temp['>1 Compound']=substancedb.loc[substancedb.index.isin(temp.index)]['>1 Compound']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:27:55.911866Z",
     "start_time": "2021-12-03T02:27:55.899863Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FragmentSmiles    object\n",
       "count              int64\n",
       "Smiles            object\n",
       ">1 Compound       object\n",
       "dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:27:56.364521Z",
     "start_time": "2021-12-03T02:27:55.914656Z"
    }
   },
   "outputs": [],
   "source": [
    "temp.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:28:18.893208Z",
     "start_time": "2021-12-03T02:27:58.831527Z"
    }
   },
   "outputs": [],
   "source": [
    "temp = temp.convert_dtypes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:28:32.691874Z",
     "start_time": "2021-12-03T02:28:18.896434Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index                    128\n",
       "SubstanceID        451630710\n",
       "FragmentSmiles    3308625317\n",
       "count              451630710\n",
       "Smiles            5967059055\n",
       ">1 Compound        100362380\n",
       "dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:29:35.752736Z",
     "start_time": "2021-12-03T02:29:01.123248Z"
    }
   },
   "outputs": [],
   "source": [
    "temp.set_index(['FragmentSmiles','SubstanceID'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:29:40.744063Z",
     "start_time": "2021-12-03T02:29:40.703249Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>Smiles</th>\n",
       "      <th>&gt;1 Compound</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FragmentSmiles</th>\n",
       "      <th>SubstanceID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CF</th>\n",
       "      <th>6890270</th>\n",
       "      <td>1</td>\n",
       "      <td>CC(C)(C)[C@@H]1CC[C@@](C)(F)[C@H](Br)C1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CBr</th>\n",
       "      <th>6890270</th>\n",
       "      <td>1</td>\n",
       "      <td>CC(C)(C)[C@@H]1CC[C@@](C)(F)[C@H](Br)C1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnc</th>\n",
       "      <th>27615522</th>\n",
       "      <td>1</td>\n",
       "      <td>Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cF</th>\n",
       "      <th>27615522</th>\n",
       "      <td>2</td>\n",
       "      <td>Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[H]C1([H])OccO1</th>\n",
       "      <th>27615522</th>\n",
       "      <td>1</td>\n",
       "      <td>Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cn(c)C(=O)[O-]</th>\n",
       "      <th>22727365</th>\n",
       "      <td>1</td>\n",
       "      <td>CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[Li+]</th>\n",
       "      <th>22727365</th>\n",
       "      <td>1</td>\n",
       "      <td>CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cOC</th>\n",
       "      <th>11404182</th>\n",
       "      <td>2</td>\n",
       "      <td>COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCl</th>\n",
       "      <th>11404182</th>\n",
       "      <td>1</td>\n",
       "      <td>COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CF</th>\n",
       "      <th>11404182</th>\n",
       "      <td>3</td>\n",
       "      <td>COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50181190 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             count  \\\n",
       "FragmentSmiles  SubstanceID          \n",
       "CF              6890270          1   \n",
       "CBr             6890270          1   \n",
       "cnc             27615522         1   \n",
       "cF              27615522         2   \n",
       "[H]C1([H])OccO1 27615522         1   \n",
       "...                            ...   \n",
       "cn(c)C(=O)[O-]  22727365         1   \n",
       "[Li+]           22727365         1   \n",
       "cOC             11404182         2   \n",
       "CCl             11404182         1   \n",
       "CF              11404182         3   \n",
       "\n",
       "                                                                        Smiles  \\\n",
       "FragmentSmiles  SubstanceID                                                      \n",
       "CF              6890270                CC(C)(C)[C@@H]1CC[C@@](C)(F)[C@H](Br)C1   \n",
       "CBr             6890270                CC(C)(C)[C@@H]1CC[C@@](C)(F)[C@H](Br)C1   \n",
       "cnc             27615522           Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F   \n",
       "cF              27615522           Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F   \n",
       "[H]C1([H])OccO1 27615522           Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F   \n",
       "...                                                                        ...   \n",
       "cn(c)C(=O)[O-]  22727365     CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O...   \n",
       "[Li+]           22727365     CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O...   \n",
       "cOC             11404182                   COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1   \n",
       "CCl             11404182                   COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1   \n",
       "CF              11404182                   COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1   \n",
       "\n",
       "                             >1 Compound  \n",
       "FragmentSmiles  SubstanceID               \n",
       "CF              6890270            False  \n",
       "CBr             6890270            False  \n",
       "cnc             27615522           False  \n",
       "cF              27615522           False  \n",
       "[H]C1([H])OccO1 27615522           False  \n",
       "...                                  ...  \n",
       "cn(c)C(=O)[O-]  22727365            True  \n",
       "[Li+]           22727365            True  \n",
       "cOC             11404182           False  \n",
       "CCl             11404182           False  \n",
       "CF              11404182           False  \n",
       "\n",
       "[50181190 rows x 3 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-03T02:33:31.870800Z",
     "start_time": "2021-12-03T02:33:07.073530Z"
    }
   },
   "outputs": [],
   "source": [
    "temp.to_pickle('/home/aa2133/Impurity-Project/Reaxys_Data/fragdb.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T08:02:53.668703Z",
     "start_time": "2021-05-07T07:54:04.775604Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UserWarning: The spaces in these column names will not be changed. In pandas versions < 0.14, spaces were converted to underscores.\n"
     ]
    }
   ],
   "source": [
    "temp.to_sql(\"FragmentDB1\",db,if_exists=\"replace\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T08:04:31.911230Z",
     "start_time": "2021-05-07T08:02:53.671653Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sqlite3.Cursor at 0x7f031ef79c70>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<sqlite3.Cursor at 0x7f031ef79c70>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cursor.execute('''CREATE INDEX Idx6 ON FragmentDB1(FragmentSmiles);''')\n",
    "cursor.execute('''CREATE INDEX Idx7 ON FragmentDB1(SubstanceID);''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T08:15:24.017465Z",
     "start_time": "2021-05-07T08:15:13.794286Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SubstanceID</th>\n",
       "      <th>Smiles</th>\n",
       "      <th>&gt;1 Compound</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>27615522</td>\n",
       "      <td>Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9019791</td>\n",
       "      <td>CCCCOc1ccc(-c2nc(-c3ccc(Cl)cc3)[nH]c2-c2ccncc2...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11185773</td>\n",
       "      <td>Cc1ccccc1CCCCCCC(=O)c1ncc(-c2ccccn2)o1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6156831</td>\n",
       "      <td>CCOC(=O)c1c(-c2cccc([N+](=O)[O-])c2)cc(-c2cccc...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11095222</td>\n",
       "      <td>CC(C)[Si](Cc1cc(-c2ccccc2)nc2ccc(Cl)cc12)(C(C)...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2820008</th>\n",
       "      <td>15730941</td>\n",
       "      <td>CCOP(=O)(COCCCONc1nc(NC=O)nc(Cl)c1NC=O)OCC</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2820009</th>\n",
       "      <td>18091249</td>\n",
       "      <td>Cc1ccncc1.O=C([O-])C(Cl)(Cl)Cl.[Mn+2]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2820010</th>\n",
       "      <td>24073116</td>\n",
       "      <td>COc1cc(C)nc(N=CN(C)C)n1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2820011</th>\n",
       "      <td>30477344</td>\n",
       "      <td>Cn1ccc2ccc(B(O)O)nc21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2820012</th>\n",
       "      <td>19632868</td>\n",
       "      <td>COc1cc(F)c(-c2c(Cl)nc(C)n2-c2ccc(C)cc2)c(F)c1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2820013 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         SubstanceID                                             Smiles  \\\n",
       "0           27615522        Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F   \n",
       "1            9019791  CCCCOc1ccc(-c2nc(-c3ccc(Cl)cc3)[nH]c2-c2ccncc2...   \n",
       "2           11185773             Cc1ccccc1CCCCCCC(=O)c1ncc(-c2ccccn2)o1   \n",
       "3            6156831  CCOC(=O)c1c(-c2cccc([N+](=O)[O-])c2)cc(-c2cccc...   \n",
       "4           11095222  CC(C)[Si](Cc1cc(-c2ccccc2)nc2ccc(Cl)cc12)(C(C)...   \n",
       "...              ...                                                ...   \n",
       "2820008     15730941         CCOP(=O)(COCCCONc1nc(NC=O)nc(Cl)c1NC=O)OCC   \n",
       "2820009     18091249              Cc1ccncc1.O=C([O-])C(Cl)(Cl)Cl.[Mn+2]   \n",
       "2820010     24073116                            COc1cc(C)nc(N=CN(C)C)n1   \n",
       "2820011     30477344                              Cn1ccc2ccc(B(O)O)nc21   \n",
       "2820012     19632868      COc1cc(F)c(-c2c(Cl)nc(C)n2-c2ccc(C)cc2)c(F)c1   \n",
       "\n",
       "         >1 Compound  \n",
       "0                  0  \n",
       "1                  0  \n",
       "2                  0  \n",
       "3                  0  \n",
       "4                  0  \n",
       "...              ...  \n",
       "2820008            0  \n",
       "2820009            1  \n",
       "2820010            0  \n",
       "2820011            0  \n",
       "2820012            0  \n",
       "\n",
       "[2820013 rows x 3 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_sql_query('''SELECT SubstanceID,Smiles,\">1 Compound\" from FragmentDB1 Where FragmentSmiles=\"cnc\";''',db)\n",
    "# pd.read_sql_query('''SELECT * from FragmentDB1 Where FragmentSmiles=\"[H]N(C)C(=O)OC\";''',db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T08:14:53.105515Z",
     "start_time": "2021-05-07T08:14:53.096450Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sqlite3.Cursor at 0x7f0449854e30>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cursor.execute('''EXPLAIN QUERY PLAN SELECT FragmentDB1.SubstanceID,FragmentDB1.FragmentSmiles,FragmentDB1.count,SubstanceDB.Smiles,SubstanceDB.\">1 Compound\" from FragmentDB1\n",
    "#                     INNER JOIN SubstanceDB ON FragmentDB1.SubstanceID=SubstanceDB.SubstanceID WHERE FragmentDB1.FragmentSmiles=\"CF\";''')\n",
    "\n",
    "cursor.execute('''EXPLAIN QUERY PLAN SELECT SubstanceID,Smiles,\">1 Compound\" from FragmentDB1 Where FragmentSmiles=\"CF\";''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-07T08:14:55.151955Z",
     "start_time": "2021-05-07T08:14:55.145495Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(3, 0, 0, 'SEARCH TABLE FragmentDB1 USING INDEX Idx6 (FragmentSmiles=?)')]\n"
     ]
    }
   ],
   "source": [
    "print(cursor.fetchall())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reaction database modification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-16T13:07:22.086149Z",
     "start_time": "2021-06-16T13:07:21.843381Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rxndb=pd.read_sql_query('''SELECT ReactionID,NumRefs,ReactantID,ProductID,NumSteps,ReagentID,Temperature,Pressure,ResidenceTime,SolventID,CatalystID,ReactionType from ReactionDB''',db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rxndb['NumRefs']=rxndb['NumRefs'].apply(lambda x:int(x))\n",
    "rxndb['ReactantID']=rxndb['ReactantID'].apply(lambda x:list(map(int,x.split(','))) if x else [])\n",
    "rxndb['ProductID']=rxndb['ProductID'].apply(lambda x:list(map(int,x.split(','))) if x else [])\n",
    "rxndb['NumSteps']=rxndb['NumSteps'].apply(lambda x:list(map(int,x.split(','))) if x else [])\n",
    "rxndb['ReagentID']=rxndb['ReagentID'].apply(lambda x:list(map(int,x.split(','))) if x else [])\n",
    "rxndb['SolventID']=rxndb['SolventID'].apply(lambda x:list(map(int,x.split(','))) if x else [])\n",
    "rxndb['CatalystID']=rxndb['CatalystID'].apply(lambda x:list(map(int,x.split(','))) if x else [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parsetemp(row):\n",
    "    temp=row['Temperature']\n",
    "    if temp:\n",
    "        tempsplit=temp.strip().split(',')\n",
    "        tempfinal=[]\n",
    "        for elem in tempsplit:\n",
    "            elem=elem.strip()\n",
    "            if elem[0]=='-':\n",
    "                templocal=elem[1:].split('-',1)\n",
    "                templocal[0]='-'+templocal[0]\n",
    "                tempfinal+=templocal\n",
    "            else:\n",
    "                tempfinal+=elem.split('-',1)\n",
    "    else:\n",
    "        tempfinal=[]\n",
    "    return [float(elem) for elem in tempfinal]\n",
    "def parsepressure(row):\n",
    "    pressure=row['Pressure']\n",
    "    errors=[]\n",
    "    if pressure:\n",
    "        try:\n",
    "            pp=list(map(float,re.split(',|-',pressure)))\n",
    "            return True\n",
    "        except Exception:\n",
    "            return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rxndbdis=mpd.DataFrame(rxndb)\n",
    "tempcol=rxndbdis.apply(parsetemp,axis=1,result_type='reduce')\n",
    "errors=rxndbdis.apply(parsepressure,axis=1,result_type='reduce')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tempcol=pd.DataFrame(data=tempcol.values,index=tempcol.index)\n",
    "rxndb['Temperature']=tempcol\n",
    "rxndb['Pressure']=rxndb['Pressure'].apply(lambda x:re.split(',|-',x) if x else [])\n",
    "rxndb['ResidenceTime']=rxndb['ResidenceTime'].apply(lambda x:re.split(',|-',x) if x else [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors[errors.values==False].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rxndb.set_index(['ReactionID'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-16T13:15:44.909327Z",
     "start_time": "2021-06-16T13:14:41.956986Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.to_pickle(rxndb,'/home/aa2133/Impurity-Project/Reaxys_Data/ReactionDB.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-16T13:09:07.191323Z",
     "start_time": "2021-06-16T13:07:25.768884Z"
    }
   },
   "outputs": [],
   "source": [
    "rxndb=pd.read_pickle('/home/aa2133/Impurity-Project/Reaxys_Data/ReactionDB.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-16T13:14:12.527151Z",
     "start_time": "2021-06-16T13:14:12.523564Z"
    }
   },
   "outputs": [],
   "source": [
    "rxndb\n",
    "rxndb.rename(columns={'ResidenceTime': 'ReactionTime'}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Substance database modification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "substancedb=pd.read_sql_query('''SELECT SubstanceID,Smiles,\">1 Compound\" from SubstanceDB''',db)\n",
    "substancedb.dtypes\n",
    "substancedb['>1 Compound']=substancedb['>1 Compound'].apply(lambda x:True if x=='1' else False)\n",
    "substancedb.set_index(['SubstanceID'],inplace=True)\n",
    "pd.to_pickle(substancedb,'/home/aa2133/Impurity-Project/Reaxys_Data/SubstanceSmiles.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-09T16:58:52.138540Z",
     "start_time": "2021-06-09T16:58:49.618320Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>MolFileAddress</th>\n",
       "      <th>Error</th>\n",
       "      <th>&gt;1 Compound</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SubstanceID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17039903</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/17039903</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27714712</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/27714712</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17632674</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/17632674</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16957119</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/16957119</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17509438</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_6500000/17509438</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17016447</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/17016447</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6619603</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/6619603</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14242245</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/14242245</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17499025</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/17499025</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21920694</th>\n",
       "      <td>Error</td>\n",
       "      <td>/home/projects/graph/data/batch_12600000/21920694</td>\n",
       "      <td>Valence error</td>\n",
       "      <td>Error</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1281117 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Smiles                                     MolFileAddress  \\\n",
       "SubstanceID                                                             \n",
       "17039903     Error   /home/projects/graph/data/batch_6500000/17039903   \n",
       "27714712     Error   /home/projects/graph/data/batch_6500000/27714712   \n",
       "17632674     Error   /home/projects/graph/data/batch_6500000/17632674   \n",
       "16957119     Error   /home/projects/graph/data/batch_6500000/16957119   \n",
       "17509438     Error   /home/projects/graph/data/batch_6500000/17509438   \n",
       "...            ...                                                ...   \n",
       "17016447     Error  /home/projects/graph/data/batch_12600000/17016447   \n",
       "6619603      Error   /home/projects/graph/data/batch_12600000/6619603   \n",
       "14242245     Error  /home/projects/graph/data/batch_12600000/14242245   \n",
       "17499025     Error  /home/projects/graph/data/batch_12600000/17499025   \n",
       "21920694     Error  /home/projects/graph/data/batch_12600000/21920694   \n",
       "\n",
       "                     Error >1 Compound  \n",
       "SubstanceID                             \n",
       "17039903     Valence error       Error  \n",
       "27714712     Valence error       Error  \n",
       "17632674     Valence error       Error  \n",
       "16957119     Valence error       Error  \n",
       "17509438     Valence error       Error  \n",
       "...                    ...         ...  \n",
       "17016447     Valence error       Error  \n",
       "6619603      Valence error       Error  \n",
       "14242245     Valence error       Error  \n",
       "17499025     Valence error       Error  \n",
       "21920694     Valence error       Error  \n",
       "\n",
       "[1281117 rows x 4 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "substancedb.loc[substancedb.Smiles=='Error']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conds=pd.read_sql_query('''SELECT ReactionID,CatalystID from ReactionDB''',db)\n",
    "rxnids=pd.read_sql_query('''SELECT ReactionID,ReactantID from ReactionDB''',db)\n",
    "rgtids=pd.read_sql_query('''SELECT ReactionID,ReagentID from ReactionDB''',db)\n",
    "substanceids=pd.read_sql_query('''SELECT SubstanceID from SubstanceDB''',db)\n",
    "unresolvedids=pd.read_sql_query('''SELECT SubstanceID from SubstanceDB Where Error!=\"No\"''',db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freqtable=copy.deepcopy(substanceids)\n",
    "rxnlist=','.join(rxnids.loc[rxnids.ReactantID.astype(bool)].ReactantID)\n",
    "rgtlist=','.join(rgtids.loc[rgtids.ReagentID.astype(bool)].ReagentID)\n",
    "catlist=','.join(conds.loc[conds.CatalystID.astype(bool)].CatalystID)\n",
    "freqtable=freqtable.merge(pd.DataFrame(data=Counter([int(ID) for ID in rxnlist.split(',')]).most_common(),columns=['SubstanceID','RctFreq']),on=['SubstanceID'],how='outer')\n",
    "freqtable=freqtable.merge(pd.DataFrame(data=Counter([int(ID) for ID in rgtlist.split(',')]).most_common(),columns=['SubstanceID','RgtFreq']),on=['SubstanceID'],how='outer')\n",
    "freqtable=freqtable.merge(pd.DataFrame(data=Counter([int(ID) for ID in catlist.split(',')]).most_common(),columns=['SubstanceID','CatFreq']),on=['SubstanceID'],how='outer')\n",
    "\n",
    "freqtable['RctFreq']=freqtable['RctFreq'].fillna(0)\n",
    "freqtable['RgtFreq']=freqtable['RgtFreq'].fillna(0)\n",
    "freqtable['CatFreq']=freqtable['CatFreq'].fillna(0)\n",
    "# bb=pd.DataFrame([('a',2),('b',6)],columns=['lol','lol2'])\n",
    "# bb\n",
    "# cc=pd.DataFrame([('a',6),('c',7)],columns=['lol','lol3'])\n",
    "# bb=bb.merge(cc,on=['lol'],how='left')\n",
    "# bb['lol3']=bb['lol3'].fillna(0)\n",
    "# bb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freqtable.loc[freqtable.SubstanceID==1722526]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freqtable['PCat']=freqtable['CatFreq'].div(freqtable[['RgtFreq','CatFreq']].sum(axis=1),axis=0).round(2)\n",
    "freqtable['PCat']=freqtable['PCat'].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.to_pickle(freqtable,'/home/aa2133/Impurity-Project/Reaxys_Data/SubstanceFreq.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PotCatList=list(freqtable.loc[freqtable.PCat>0.05].SubstanceID)\n",
    "writepickle(PotCatList,'/home/aa2133/Impurity-Project/Reaxys_Data/','PotCatList')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unresolvedids=set(unresolvedids.SubstanceID).union(set(freqtable.SubstanceID)-set(substanceids.SubstanceID)) #Includes substance IDs that appear in reactions but are not in our substance database\n",
    "writepickle(unresolvedids,'/home/aa2133/Impurity-Project/Reaxys_Data/','UnresolvedIDs')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sandbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-06T05:36:06.562445Z",
     "start_time": "2021-05-06T05:36:05.244647Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags0('COc1ccc2cccc(CC#N)c2c1',resFormat='smiles')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T17:15:07.594022Z",
     "start_time": "2021-02-10T17:15:07.583883Z"
    }
   },
   "outputs": [],
   "source": [
    "dd.utils.is_dataframe_like(pd.DataFrame({'FragmentSmiles':[['asdasds']],'FragmentSmarts':[['asdasd']]}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T04:41:23.027566Z",
     "start_time": "2021-02-08T04:41:16.075319Z"
    }
   },
   "outputs": [],
   "source": [
    "if CarrierFragmentDB.index.name or CarrierFragmentDB.index.names:\n",
    "    CarrierFragmentDB.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T17:11:27.437387Z",
     "start_time": "2021-02-08T17:11:27.394080Z"
    }
   },
   "outputs": [],
   "source": [
    "gk=pd.DataFrame(['BOO',None,'Do'])\n",
    "gk\n",
    "gk.dropna(inplace=True)\n",
    "gk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T15:55:36.036109Z",
     "start_time": "2021-02-08T15:55:32.291406Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.loc[CarrierFragmentDB['SubstanceID']==4502]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T17:38:17.586738Z",
     "start_time": "2021-02-08T17:38:17.580687Z"
    }
   },
   "outputs": [],
   "source": [
    "'>1 Compound' not in CarrierFragmentDB.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T05:59:08.218389Z",
     "start_time": "2021-02-09T05:57:35.697311Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask=dd.from_pandas(CarrierFragmentDB,npartitions=181)\n",
    "CarrierFragmentDBDask=client.persist(CarrierFragmentDBDask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T04:49:57.178407Z",
     "start_time": "2021-02-08T04:49:57.163796Z"
    }
   },
   "outputs": [],
   "source": [
    "type(CarrierFragmentDBDask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-05T14:23:29.968612Z",
     "start_time": "2021-02-05T14:23:29.962199Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.columns[CarrierFragmentDB.columns.str.contains('ragment')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T04:55:37.850311Z",
     "start_time": "2021-02-08T04:53:06.050119Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturecol=buildfragdb(fdb=CarrierFragmentDB,dfdb=CarrierFragmentDBDask,mixtures=True).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T04:55:53.168024Z",
     "start_time": "2021-02-08T04:55:53.153872Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturecol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T04:59:01.789909Z",
     "start_time": "2021-02-08T04:58:58.264089Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturedf=pd.DataFrame(mixturecol)\n",
    "CarrierFragmentDB=CarrierFragmentDB.join(mixturedf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T04:05:09.903185Z",
     "start_time": "2021-02-09T04:05:04.067199Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.loc[CarrierFragmentDB['>1 Compound']==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T06:38:51.700735Z",
     "start_time": "2021-02-09T06:38:51.668788Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB2=CarrierFragmentDB[:100].groupby(['SubstanceID','Smiles']).agg({'FragmentSmiles': lambda x: x.tolist(),'FragmentSmarts': lambda x: x.tolist(),'>1 Compound': lambda x: x.tolist()}) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T06:44:57.582122Z",
     "start_time": "2021-02-09T06:44:57.570567Z"
    }
   },
   "outputs": [],
   "source": [
    "type(CarrierFragmentDB2['FragmentSmiles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T06:02:59.807626Z",
     "start_time": "2021-02-09T06:02:59.617984Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T05:07:21.738792Z",
     "start_time": "2021-02-09T05:07:21.730284Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3.loc[925]['Smiles']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T05:03:25.528737Z",
     "start_time": "2021-02-09T05:03:24.368966Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.loc[:1000000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T04:28:55.666172Z",
     "start_time": "2021-02-09T04:28:55.651708Z"
    }
   },
   "outputs": [],
   "source": [
    "len(CarrierFragmentDB[:100].SubstanceID.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T07:08:17.946347Z",
     "start_time": "2021-02-09T07:08:16.605390Z"
    }
   },
   "outputs": [],
   "source": [
    "fdb=CarrierFragmentDB.loc[CarrierFragmentDB['>1 Compound']==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T07:08:26.068405Z",
     "start_time": "2021-02-09T07:08:20.977091Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbdask=dd.from_pandas(fdb,npartitions=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T07:59:29.880327Z",
     "start_time": "2021-02-09T07:59:29.843332Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbdask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T07:08:31.541851Z",
     "start_time": "2021-02-09T07:08:31.526088Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbdask=client.persist(fdbdask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T08:07:10.072829Z",
     "start_time": "2021-02-09T08:00:08.566167Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3=fdbdask.groupby(['Smiles']).agg([list]).compute() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CarrierFragmentDB3=CarrierFragmentDB3.drop('list',axis=1,level=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T08:31:30.734308Z",
     "start_time": "2021-02-10T08:31:22.435802Z"
    }
   },
   "outputs": [],
   "source": [
    "writepickle(CarrierFragmentDB3,'/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDBmixtures_reduced(n=6)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T11:59:49.348399Z",
     "start_time": "2021-02-10T11:59:37.065889Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3=openpickle('/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDBmixtures_reduced(n=6).pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T11:59:53.836572Z",
     "start_time": "2021-02-10T11:59:53.754098Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T09:39:12.603607Z",
     "start_time": "2021-02-09T09:39:12.574011Z"
    }
   },
   "outputs": [],
   "source": [
    "ty=CarrierFragmentDB3[:2].apply(getMixturefrags,natoms=6,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T10:22:36.637769Z",
     "start_time": "2021-02-09T10:22:36.617002Z"
    }
   },
   "outputs": [],
   "source": [
    "ty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T10:54:20.434613Z",
     "start_time": "2021-02-09T10:54:15.913978Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3Dask=dd.from_pandas(CarrierFragmentDB3,npartitions=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T10:54:23.677344Z",
     "start_time": "2021-02-09T10:54:22.623090Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3Dask=client.persist(CarrierFragmentDB3Dask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T10:56:50.957308Z",
     "start_time": "2021-02-09T10:56:50.813618Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3Dask.loc[CarrierFragmentDB3Dask['Smiles']=='B#N.N.[LiH].[LiH].[LiH]'].compute()['Smiles']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T08:59:43.837249Z",
     "start_time": "2021-02-09T08:59:43.062744Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.loc[CarrierFragmentDBDask['Smiles']=='B#N.N.[LiH].[LiH].[LiH]'].compute()['Smiles']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting mixture smarts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T10:57:06.257875Z",
     "start_time": "2021-02-09T10:57:06.188433Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturesmarts2=CarrierFragmentDB3Dask.loc[:2].map_partitions(getMixturefragspartition,natoms=6,meta=('FragmentSmarts','O')).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T11:09:20.600372Z",
     "start_time": "2021-02-09T10:57:27.942105Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturesmarts=CarrierFragmentDB3Dask.map_partitions(getMixturefragspartition,natoms=6,meta=('FragmentSmarts','O')).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T12:05:50.916311Z",
     "start_time": "2021-02-10T12:05:46.376491Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturesmarts=openpickle('/home/aa2133/Impurity-Project/Reaxys_Data/mixturesmarts.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T09:29:42.400884Z",
     "start_time": "2021-02-10T09:29:36.945219Z"
    }
   },
   "outputs": [],
   "source": [
    "writepickle(mixturesmarts,'/home/aa2133/Impurity-Project/Reaxys_Data/mixturesmarts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T12:06:00.639165Z",
     "start_time": "2021-02-10T12:06:00.631962Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturesmarts[1249046]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T12:13:52.413282Z",
     "start_time": "2021-02-09T12:13:52.194490Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturesmartsdf=pd.DataFrame(mixturesmarts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T12:16:58.571026Z",
     "start_time": "2021-02-09T12:16:58.333911Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3['FragmentSmarts']=mixturesmartsdf['FragmentSmarts']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T08:58:25.475175Z",
     "start_time": "2021-02-10T08:58:17.512132Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4=CarrierFragmentDB3.explode('FragmentSmarts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T08:59:39.130213Z",
     "start_time": "2021-02-10T08:59:38.424424Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4[['Smiles','FragmentSmarts']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T16:37:37.926172Z",
     "start_time": "2021-02-09T16:37:23.913739Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4Dask=dd.from_pandas(CarrierFragmentDB4,npartitions=64)\n",
    "CarrierFragmentDB4Dask=client.persist(CarrierFragmentDB4Dask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T16:39:16.934125Z",
     "start_time": "2021-02-09T16:38:11.207454Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturefragsmiles=CarrierFragmentDB4Dask.map_partitions(getsmilespartition,meta=('FragmentSmiles','O')).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T16:39:27.417317Z",
     "start_time": "2021-02-09T16:39:27.400226Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturefragsmiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T09:00:00.385331Z",
     "start_time": "2021-02-10T08:59:57.756102Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4['FragmentSmiles']=pd.DataFrame(mixturefragsmiles)['FragmentSmiles']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T09:00:04.908248Z",
     "start_time": "2021-02-10T09:00:03.908511Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4.loc[CarrierFragmentDB4['Smiles']=='B#N.N.[LiH].[LiH].[LiH]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T12:12:10.795127Z",
     "start_time": "2021-02-09T12:12:10.730131Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3Dask.loc[1249046].compute()['FragmentSmarts']['list'].values[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The final dataframe does have errors. Whilst errors converting molfile to smiles had been indentified and removed earlier, there are further errors as the conversion of molfile to smiles, whilst successful, may not be correct. These are all the entries where this is the case. Furthermore we can see where Reaxys has stored substances with the same smiles string but different substance IDs (The complexity of the substance cannot be translated to a valid smiles string. There are valence errors)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T07:44:34.677905Z",
     "start_time": "2021-02-10T07:44:34.317821Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4=explode(CarrierFragmentDB3[CarrierFragmentDB3.FragmentSmarts=='Error'],lst_cols=['SubstanceID','FragmentSmiles','>1 Compound'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T08:08:10.133759Z",
     "start_time": "2021-02-10T08:08:10.097570Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T07:56:55.987576Z",
     "start_time": "2021-02-10T07:56:55.755304Z"
    }
   },
   "outputs": [],
   "source": [
    "templist=CarrierFragmentDB3[CarrierFragmentDB3.FragmentSmarts=='Error'].SubstanceID.str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T07:57:04.672779Z",
     "start_time": "2021-02-10T07:57:04.648848Z"
    }
   },
   "outputs": [],
   "source": [
    "templist[templist.values>1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T07:58:56.774307Z",
     "start_time": "2021-02-10T07:58:56.540466Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3[CarrierFragmentDB3.FragmentSmarts=='Error'].loc[51293].Smiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T07:59:48.509706Z",
     "start_time": "2021-02-10T07:59:30.298483Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.loc[CarrierFragmentDB['Smiles']=='C1CCOC1.C[Si](C)(C)[GeH2-]([Si](C)(C)C)[Si](C)(C)C.[Li+]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T08:06:18.632589Z",
     "start_time": "2021-02-10T08:06:18.622781Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('C[Si](C)(C)[GeH2-]([Si](C)(C)C)[Si](C)(C)C')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T08:47:44.195966Z",
     "start_time": "2021-02-10T08:47:43.560055Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB5=CarrierFragmentDB3[~CarrierFragmentDB3.Smiles.isin(CarrierFragmentDB4.Smiles)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T08:52:17.023023Z",
     "start_time": "2021-02-10T08:51:26.228617Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB6=explode(CarrierFragmentDB5,lst_cols=['SubstanceID','FragmentSmiles'])#'FragmentSmarts','>1 Compound'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T07:27:29.011308Z",
     "start_time": "2021-02-10T07:27:28.976610Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3.loc[26211]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T07:30:02.733413Z",
     "start_time": "2021-02-10T07:29:59.942673Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.loc[CarrierFragmentDB['SubstanceID']==16800164]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T07:29:11.752341Z",
     "start_time": "2021-02-10T07:29:11.523453Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('Br[GeH2-](Br)Br.[Br-].[H][N+]([H])([H])[H]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:01:29.315662Z",
     "start_time": "2021-02-09T17:01:23.133353Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.loc[CarrierFragmentDB['Smiles']=='B#N.N.[LiH].[LiH].[LiH]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:14:51.793520Z",
     "start_time": "2021-02-10T04:14:50.971577Z"
    }
   },
   "outputs": [],
   "source": [
    "testdb2=CarrierFragmentDB4.loc[CarrierFragmentDB4['Smiles']=='c1ccncc1.c1nc[nH]n1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:37:06.530040Z",
     "start_time": "2021-02-09T17:37:00.332169Z"
    }
   },
   "outputs": [],
   "source": [
    "testdb=CarrierFragmentDB.loc[CarrierFragmentDB['Smiles']=='c1ccncc1.c1nc[nH]n1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:13:25.023373Z",
     "start_time": "2021-02-10T04:13:25.006231Z"
    }
   },
   "outputs": [],
   "source": [
    "testdb2=testdb2.explode('SubstanceID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:14:16.590645Z",
     "start_time": "2021-02-10T04:14:16.560132Z"
    }
   },
   "outputs": [],
   "source": [
    "def explode(df, lst_cols, fill_value=''):\n",
    "    # make sure `lst_cols` is a list\n",
    "    if lst_cols and not isinstance(lst_cols, list):\n",
    "        lst_cols = [lst_cols]\n",
    "    # all columns except `lst_cols`\n",
    "    idx_cols = df.columns.difference(lst_cols)\n",
    "\n",
    "    # calculate lengths of lists\n",
    "    lens = df[lst_cols[0]].str.len()\n",
    "\n",
    "    if (lens > 0).all():\n",
    "        # ALL lists in cells aren't empty\n",
    "        return pd.DataFrame({\n",
    "            col:np.repeat(df[col].values, df[lst_cols[0]].str.len())\n",
    "            for col in idx_cols\n",
    "        }).assign(**{col:np.concatenate(df[col].values) for col in lst_cols}) \\\n",
    "          .loc[:, df.columns]\n",
    "    else:\n",
    "        # at least one list in cells is empty\n",
    "        return pd.DataFrame({\n",
    "            col:np.repeat(df[col].values, df[lst_cols[0]].str.len())\n",
    "            for col in idx_cols\n",
    "        }).assign(**{col:np.concatenate(df[col].values) for col in lst_cols}) \\\n",
    "          .append(df.loc[lens==0, idx_cols]).fillna(fill_value) \\\n",
    "          .loc[:, df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:15:07.700501Z",
     "start_time": "2021-02-10T04:15:07.666253Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "testdb2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T06:46:59.366875Z",
     "start_time": "2021-02-10T06:46:59.163407Z"
    }
   },
   "outputs": [],
   "source": [
    "testdb3=CarrierFragmentDB3.loc[CarrierFragmentDB3['Smiles']=='c1ccncc1.c1nc[nH]n1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T06:47:43.666505Z",
     "start_time": "2021-02-10T06:47:43.626763Z"
    }
   },
   "outputs": [],
   "source": [
    "explode(testdb3,lst_cols=['SubstanceID','FragmentSmiles','FragmentSmarts','>1 Compound'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The below exposes some errors in the carrier frags function. The way RDKit treats smiles and smarts is not consistent at all, and fragments are even worse."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T05:51:27.645779Z",
     "start_time": "2021-02-10T05:51:27.608026Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('c1ccncc1.c1nc[nH]n1')\n",
    "Chem.MolFromSmarts('[#6]:[#6](:[#7]:[#6](:[#6])-[H])-[H]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T05:51:32.298460Z",
     "start_time": "2021-02-10T05:51:32.257624Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7]:[#7]:1)-[H])-[H]')\n",
    "Chem.MolFromSmiles('[H]C1:N:N:C([H]):N:1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T16:54:33.332144Z",
     "start_time": "2021-02-09T16:54:33.131834Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags('c1nc[nH]n1',6)\n",
    "Chem.MolToSmiles(Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7]:[#7]:1)-[H])-[H]'))\n",
    "moltest=Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7]:[#7]:1)-[H])-[H]')\n",
    "Chem.SanitizeMol(moltest)\n",
    "moltest.UpdatePropertyCache(strict=False)\n",
    "Chem.MolToSmiles(moltest,allHsExplicit=False)\n",
    "Chem.MolToSmiles(Chem.MolFromSmiles('[H]C1:N:N:C([H]):N:1'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T06:11:15.040514Z",
     "start_time": "2021-02-10T06:11:15.017135Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7]:[#7]:1)-[H])-[H]')\n",
    "Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7](:[#7]:1)-[H])-[H])-[H]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T06:12:08.297652Z",
     "start_time": "2021-02-10T06:12:08.286238Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolToSmiles(Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7](:[#7]:1)-[H])-[H])-[H]'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T06:12:23.208368Z",
     "start_time": "2021-02-10T06:12:23.200466Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('[H]C1:N:C([H]):N([H]):N:1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:37:46.076872Z",
     "start_time": "2021-02-10T04:37:45.920868Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('c1ccncc1.c1nc[nH]n1')\n",
    "Chem.MolFromSmiles('[H]C(:C):N:C([H]):C')\n",
    "Chem.MolFromSmiles('[H]C1:N:N:C([H]):N:1')\n",
    "Chem.MolFromSmiles('[H]C1:N:C([H]):N([H]):N:1')\n",
    "Chem.MolFromSmiles(Chem.MolToSmiles(Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7](:[#7]:1)-[H])-[H])-[H]'),allHsExplicit=True))\n",
    "Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7](:[#7]:1)-[H])-[H])-[H]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:42:20.157560Z",
     "start_time": "2021-02-10T04:42:20.145262Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolToSmarts(Chem.MolFromSmarts('[#6]1(:[#7]:[#6](:[#7]:[#7]:1)-[H])-[H]'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:41:11.451179Z",
     "start_time": "2021-02-10T04:41:11.426524Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolToSmarts(Chem.MolFromSmiles('c1nc[nH]n1'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:43:20.031375Z",
     "start_time": "2021-02-10T04:43:20.022403Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags('c1nc[nH]n1',0,resFormat='smarts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T04:44:55.014550Z",
     "start_time": "2021-02-10T04:44:54.993334Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmarts('[#7]:[#7]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T05:49:43.855776Z",
     "start_time": "2021-02-10T05:49:43.673653Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('[H]c1nc([H])n([H])n1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:08:20.524626Z",
     "start_time": "2021-02-10T18:08:20.518645Z"
    }
   },
   "outputs": [],
   "source": [
    "def mol_with_atom_index(mol):\n",
    "    for atom in mol.GetAtoms():\n",
    "        atom.SetAtomMapNum(atom.GetIdx())\n",
    "    return mol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T12:14:50.895480Z",
     "start_time": "2021-02-10T12:14:50.760827Z"
    }
   },
   "outputs": [],
   "source": [
    "testmol=Chem.MolFromSmiles('c1nc[nH]n1')\n",
    "Chem.SanitizeMol(testmol)\n",
    "testmol.UpdatePropertyCache(strict=False)\n",
    "Chem.AddHs(testmol)\n",
    "mol_with_atom_index(testmol)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T06:06:31.472507Z",
     "start_time": "2021-02-10T06:06:31.453508Z"
    }
   },
   "outputs": [],
   "source": [
    "IFG(testmol)\n",
    "testmol.GetAtomWithIdx(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:49:24.513849Z",
     "start_time": "2021-02-09T17:49:24.502452Z"
    }
   },
   "outputs": [],
   "source": [
    "testdb2.set_index(['SubstanceID'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:49:56.077383Z",
     "start_time": "2021-02-09T17:49:56.070323Z"
    }
   },
   "outputs": [],
   "source": [
    "testdb.set_index('SubstanceID',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:49:03.809060Z",
     "start_time": "2021-02-09T17:49:03.624635Z"
    }
   },
   "outputs": [],
   "source": [
    "testdb2.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:50:09.857787Z",
     "start_time": "2021-02-09T17:50:09.830016Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.concat([testdb,testdb2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:46:31.725197Z",
     "start_time": "2021-02-09T17:46:31.677415Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.concat([testdb,testdb2]).drop([16832508],level=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:47:50.235105Z",
     "start_time": "2021-02-09T17:47:50.094479Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4.index.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T17:51:28.358987Z",
     "start_time": "2021-02-09T17:51:28.269467Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T18:10:43.720336Z",
     "start_time": "2021-02-09T18:10:43.677649Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T09:05:34.760448Z",
     "start_time": "2021-02-10T09:05:34.702189Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T10:46:54.294738Z",
     "start_time": "2021-02-09T10:46:53.734641Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.loc[CarrierFragmentDBDask['SubstanceID']==31713109].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T10:47:10.060294Z",
     "start_time": "2021-02-09T10:47:10.042068Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('*(c1ccccc1)c1ccccc1.N.N.[1*].[1*]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T12:17:37.834934Z",
     "start_time": "2021-02-09T12:17:37.820433Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB3.loc[1239046]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T17:43:38.391753Z",
     "start_time": "2021-02-08T17:43:38.194235Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.rename(columns={'CarrierFragmentSmiles':'FragmentSmiles','CarrierFragmentSmarts':'FragmentSmarts'},inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T17:45:23.134380Z",
     "start_time": "2021-02-08T17:45:22.955153Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask=CarrierFragmentDBDask.rename(columns={'CarrierFragmentSmiles':'FragmentSmiles','CarrierFragmentSmarts':'FragmentSmarts'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T17:45:45.069267Z",
     "start_time": "2021-02-08T17:45:44.759440Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask=client.persist(CarrierFragmentDBDask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T17:45:25.804112Z",
     "start_time": "2021-02-08T17:45:25.604769Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T15:47:28.440531Z",
     "start_time": "2021-02-08T15:47:28.432632Z"
    }
   },
   "outputs": [],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T14:51:09.995385Z",
     "start_time": "2021-02-08T14:51:09.982523Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags('Cl',6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T14:37:27.802745Z",
     "start_time": "2021-02-08T14:37:27.790669Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags('Brc1ccc2ncccc2c1',6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T09:03:39.447057Z",
     "start_time": "2021-02-08T09:03:39.410451Z"
    }
   },
   "outputs": [],
   "source": [
    "mol2=Chem.MolFromSmarts('[#17]')\n",
    "Chem.SanitizeMol(mol2)\n",
    "mol2.UpdatePropertyCache(strict=False)\n",
    "Chem.MolToSmiles(mol2)\n",
    "mol2.GetAtoms()[0].GetTotalNumHs()\n",
    "Chem.MolToSmiles(Chem.MolFromSmiles(Chem.MolToSmiles(Chem.rdmolops.AddHs(mol2))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T09:05:22.319487Z",
     "start_time": "2021-02-08T09:05:22.305308Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolToSmiles(Chem.rdmolops.AddHs(Chem.MolFromSmiles('Brc1ccc2ncccc2c1')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T16:53:56.634549Z",
     "start_time": "2021-02-03T16:53:56.629102Z"
    }
   },
   "outputs": [],
   "source": [
    "# -- directories for Reaxys data\n",
    "# allrxtidsdir='/home/zg268/py2projects/reaction_1/AllRxtIDs.dat'\n",
    "# This file contains reactant IDs in Reaxys\n",
    "allrxtidsdir = '/home/aa2133/Impurity-Project/Reaxys_Data/AllRxtIDs.dat'\n",
    "# rxnSource='/home/projects/graph/11step_network_774355/zg_good_data_noDupRXID2.dat'\n",
    "# This file contains the reaction network\n",
    "rxnSource = '/home/aa2133/Impurity-Project/Reaxys_Data/zg_good_data_noDupRXID2.dat'\n",
    "\n",
    "# This is a directory containing all mol files\n",
    "molfilesource = '/home/projects/graph/data/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T03:35:05.231134Z",
     "start_time": "2021-02-11T03:35:04.683697Z"
    }
   },
   "outputs": [],
   "source": [
    "from dask.distributed import Client, LocalCluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T03:35:09.825687Z",
     "start_time": "2021-02-11T03:35:07.570589Z"
    }
   },
   "outputs": [],
   "source": [
    "cluster=LocalCluster(n_workers=16,threads_per_worker=2,scheduler_port=42117) # dashboard_address=':8786\n",
    "client = Client(cluster) #'127.0.0.1:8786' is default scheduler cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T04:16:47.818261Z",
     "start_time": "2021-02-08T04:16:47.138940Z"
    }
   },
   "outputs": [],
   "source": [
    "client = Client('tcp://127.0.0.1:42117') #Use when scheduler already initiated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T03:35:11.399863Z",
     "start_time": "2021-02-11T03:35:11.378385Z"
    }
   },
   "outputs": [],
   "source": [
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T05:01:24.231269Z",
     "start_time": "2021-02-09T05:01:15.713752Z"
    }
   },
   "outputs": [],
   "source": [
    "client.restart()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T08:44:38.948024Z",
     "start_time": "2021-02-10T08:44:36.310785Z"
    }
   },
   "outputs": [],
   "source": [
    "cluster.close()\n",
    "client.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below, the existing substance database is loaded as a pandas dataframe (pickle file) and processed(imputing missing values, defining column data types). Note that this takes up several gigabytes of space in memory but is nearly 10 times faster at retrieving indexed data (however, when doing intensive write/modifying operations on the entire database or non-indexed data slowdows will be expected due to inability to parallelize). For optimizing pandas, check https://engineering.upside.com/a-beginners-guide-to-optimizing-pandas-code-for-speed-c09ef2c6a4d6. Otherwise use pandas documentation: https://pandas.pydata.org/pandas-docs/stable/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-18T13:10:46.935608Z",
     "start_time": "2021-02-18T13:09:55.320325Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb = openpickle(\"/home/aa2133/Impurity-Project/Reaxys_Data/SubstanceDB.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T16:54:50.522612Z",
     "start_time": "2021-02-03T16:54:50.494031Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "substancedb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The error column has NaN values as there is no error in processing the molfile. Therefore, imputation is needed (filling in missing values) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T11:26:30.518072Z",
     "start_time": "2021-01-26T11:26:27.121702Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb['Error'].fillna('No', inplace=True)\n",
    "substancedb['Smiles'].fillna('Error', inplace=True)\n",
    "substancedb.Error.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some pandas functions are depicted below to test speed of retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T11:43:13.528976Z",
     "start_time": "2021-01-26T11:43:12.220814Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb.loc[6138726]\n",
    "\n",
    "substancedb.loc[substancedb['Smiles'] == 'COc1ccc(CCl)cc1OCc1ccc(C(F)(F)F)cc1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T11:59:39.398502Z",
     "start_time": "2021-01-26T11:59:28.280164Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb = substancedb.astype(\n",
    "    {'MolFileAddress': 'str', 'Smiles': 'str', 'Error': 'str'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T12:07:54.037294Z",
     "start_time": "2021-01-26T12:07:54.027301Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T12:08:21.989746Z",
     "start_time": "2021-01-26T12:08:19.446702Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb.MolFileAddress.isnull().values.any()\n",
    "substancedb.Smiles.isnull().values.any()\n",
    "substancedb.Error.isnull().values.any()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The below indicates all entries where molfiles are unable to be processed. Around 1,281,117 compounds are not processable by RDKit, of which 1,281,041 is due to valence error and 76 are due to kekulization errors (related to lack of specification of hydrogens on rings). Most of these are massive compounds containing in excess of 40 carbons and 80 hydrogens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T12:42:06.512786Z",
     "start_time": "2021-01-26T12:42:03.596509Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb[substancedb['Error'] != 'No'].count()\n",
    "substancedb[substancedb['Error'] == 'Valence error'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T04:34:22.629435Z",
     "start_time": "2021-02-01T04:34:19.405627Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb[(substancedb['Error'] != 'Valence error')\n",
    "            & (substancedb['Error'] != 'No')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:05:09.692916Z",
     "start_time": "2021-02-10T14:05:09.669727Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, the goal is to identify mixtures or reaxys compounds that contain multiple substances (eg. salts). This is useful in the future as retrieving carrier fragments will entail splitting of the smiles string."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:12:59.594662Z",
     "start_time": "2021-02-10T14:12:44.430813Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask = dd.from_pandas(substancedb, npartitions=16)\n",
    "substancedbdask = client.persist(substancedbdask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:25:13.926330Z",
     "start_time": "2021-02-10T14:24:44.964393Z"
    }
   },
   "outputs": [],
   "source": [
    "mixturecol=substancedbdask.map_partitions(findMixturespartition,meta=('>1 Compound','boolean')).compute()\n",
    "substancedb=joindf(pd.DataFrame(mixturecol),substancedb)\n",
    "substancedb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "substancedb.reset_index(inplace=True)\n",
    "substancedb.set_index(['Smiles','SubstanceID'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:28:13.152905Z",
     "start_time": "2021-02-10T14:27:41.787612Z"
    }
   },
   "outputs": [],
   "source": [
    "writepickle(substancedb, '/home/aa2133/Impurity-Project/Reaxys_Data/SubstanceDB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running fragment detection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building dask dataframe from cleaned substance database"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cells below build a dask dataframe with 16 partitions for faster parallel computation. It does not take up much memory in the server as a result, but is slower at reading/fetching indexed data. The final file is written to parquet (This has been deprecated as it takes too much memory. It is more advisable to generate a dask dataframe from pandas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:30:39.632811Z",
     "start_time": "2021-02-10T14:29:50.866250Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask = dd.from_pandas(substancedb, npartitions=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:30:58.718561Z",
     "start_time": "2021-02-10T14:30:53.896977Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask = client.persist(substancedbdask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T12:55:46.070957Z",
     "start_time": "2021-01-26T12:55:30.955957Z"
    }
   },
   "outputs": [],
   "source": [
    "# substancedbdask.to_parquet(\n",
    "#     \"/home/aa2133/Impurity-Project/Reaxys_Data/SubstanceDBDask\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading dataframe and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T07:24:49.851318Z",
     "start_time": "2021-01-28T07:24:49.790751Z"
    }
   },
   "outputs": [],
   "source": [
    "# substancedbdask = dd.read_parquet(\n",
    "#     \"/home/aa2133/Impurity-Project/Reaxys_Data/SubstanceDBDask\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:51:52.148744Z",
     "start_time": "2021-02-10T14:51:52.136009Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:52:29.404329Z",
     "start_time": "2021-02-10T14:52:29.328573Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask.get_partition(2)\n",
    "substancedbdask.memory_usage_per_partition().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:52:44.699430Z",
     "start_time": "2021-02-10T14:52:44.405683Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "substancedbdask.loc[2632386].compute()\n",
    "\n",
    "substancedbdask.loc[substancedbdask['Smiles']\n",
    "                    == 'Nc1cc(F)c(N2CC(F)C2)c(F)c1'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T14:52:55.724168Z",
     "start_time": "2021-02-10T14:52:55.691222Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask.loc[2632386].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T16:28:28.963251Z",
     "start_time": "2021-01-26T16:28:28.940694Z"
    }
   },
   "outputs": [],
   "source": [
    "g = substancedbdask.loc[2632386, 'Smiles'].compute()[2632386]\n",
    "g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T16:29:07.932660Z",
     "start_time": "2021-01-26T16:29:07.925224Z"
    }
   },
   "outputs": [],
   "source": [
    "f = substancedb.loc[2632386, 'Smiles']\n",
    "f"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, the question remains whether it is best to build another column row by row or build a dataframe from an entire column. https://towardsdatascience.com/apply-function-to-pandas-dataframe-rows-76df74165ee4 has more details. Building an entire column, and appending this to the dataframe is better and has been implemented."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating dask dataframe to house fragment data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a new dataframe to house information on fragments. This is a subset of the substance database containing only substance IDs, Smiles and mixture identification where molfiles CAN be processed. The index is reset (as indexing is the same across all partitions ie. 1 to 1000000 for partition 1, 1 to 1000000 for partition2 and so on), and a dask dataframe (substancedbdask2) is generated. Totally 14.8 million compounds out of the original 16 million compounds can be processed this way."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T03:38:13.233023Z",
     "start_time": "2021-02-11T03:38:00.676287Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbm=substancedb.reset_index()[['SubstanceID', 'Smiles','>1 Compound']]\n",
    "fdbm=fdbm[fdbm.Smiles!='Error']\n",
    "substancedbdask2=dd.from_pandas(fdbm,npartitions=16)\n",
    "substancedbdask2=client.persist(substancedbdask2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T16:14:06.657494Z",
     "start_time": "2021-02-10T16:14:05.812979Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask2['SubstanceID'].count().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple solution (Pandas)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The bottom section served to highlight how inefficient it is using pandas (single core)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T08:12:17.159361Z",
     "start_time": "2021-01-27T08:12:16.842279Z"
    }
   },
   "outputs": [],
   "source": [
    "testset = substancedbdask2.head(100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T08:07:14.231110Z",
     "start_time": "2021-01-27T08:07:14.211702Z"
    }
   },
   "outputs": [],
   "source": [
    "testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T08:06:18.285876Z",
     "start_time": "2021-01-27T08:06:15.841129Z"
    }
   },
   "outputs": [],
   "source": [
    "applycol=testset.apply(getfrags,natoms=0,axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "variables": {
     "2.17*(16000000/100)/3600": "96.44444444444444"
    }
   },
   "source": [
    "2.17 seconds per every 100 rows translates to {{2.17*(16000000/100)/3600}} hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T08:05:47.184176Z",
     "start_time": "2021-01-27T08:05:47.172136Z"
    }
   },
   "outputs": [],
   "source": [
    "applycol"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple solution (parallel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This employs dask. However, it is slower for smaller datasets. The get carrier frags function is applied to each partition in the modified substance database using all 16 cores for carrier fragments (n=6). Subsequently, fragment smiles are calculated by processing the fragment smarts. The reason for this is that there are many smarts representations of a fragment structure but only one canonical smiles, so filtering the database by canonical smiles can capture more analogue compounds. This workflow is first tested on a test set of 100 compounds to ensure everything work smoothly and then tested on the entire dataset. The results are stored in pickle files on the server"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T17:33:49.442052Z",
     "start_time": "2021-02-10T17:33:49.391989Z"
    }
   },
   "outputs": [],
   "source": [
    "dasktestset=dd.from_pandas(substancedbdask2.loc[:100].compute(),npartitions=16)\n",
    "dasktestset=client.persist(dasktestset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T17:49:24.161360Z",
     "start_time": "2021-02-10T17:49:24.078027Z"
    }
   },
   "outputs": [],
   "source": [
    "dasktestset.loc[dasktestset['>1 Compound']==True].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:21:09.757940Z",
     "start_time": "2021-02-10T18:21:09.593903Z"
    }
   },
   "outputs": [],
   "source": [
    "applycolp2=dasktestset.map_partitions(getfragpartition,natoms=6,meta=('FragmentSmarts','O')).compute()\n",
    "dasktestset=client.persist(dd.from_pandas(joindf(pd.DataFrame(applycolp2),substancedbdask2.loc[:100].compute()),npartitions=16)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:21:12.160040Z",
     "start_time": "2021-02-10T18:21:12.062617Z"
    }
   },
   "outputs": [],
   "source": [
    "applycolp3=dasktestset.map_partitions(getsmilespartition,meta=('FragmentSmiles','O')).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:21:14.162722Z",
     "start_time": "2021-02-10T18:21:14.120710Z"
    }
   },
   "outputs": [],
   "source": [
    "finaldb=joindf(pd.DataFrame(applycolp3),joindf(pd.DataFrame(applycolp2),substancedbdask2.loc[:100].compute()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:39:43.323246Z",
     "start_time": "2021-02-10T18:39:43.287405Z"
    }
   },
   "outputs": [],
   "source": [
    "finaldb2=joindf(pd.DataFrame(applycolp2),substancedbdask2.loc[:100].compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:40:25.649142Z",
     "start_time": "2021-02-10T18:40:25.382392Z"
    }
   },
   "outputs": [],
   "source": [
    "finaldb2=finaldb2.explode('FragmentSmarts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:44:38.480020Z",
     "start_time": "2021-02-10T18:44:38.465433Z"
    }
   },
   "outputs": [],
   "source": [
    "finaldb2.loc[finaldb2['FragmentSmarts']=='Error']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:41:50.675666Z",
     "start_time": "2021-02-10T18:41:50.646564Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(applycolp3).explode('FragmentSmiles')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:21:16.234203Z",
     "start_time": "2021-02-10T18:21:16.223144Z"
    }
   },
   "outputs": [],
   "source": [
    "finaldb.loc[1].FragmentSmarts\n",
    "finaldb.loc[1].FragmentSmiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:02:43.638010Z",
     "start_time": "2021-02-10T18:02:43.625314Z"
    }
   },
   "outputs": [],
   "source": [
    "finaldb.loc[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Something to take note is that the fragment smarts should be used in substructure searches instead of fragment smiles, as it is often invalid/cannot generate a valid molecule due to valence errors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:21:23.588577Z",
     "start_time": "2021-02-10T18:21:23.571432Z"
    }
   },
   "outputs": [],
   "source": [
    "mol_with_atom_index(Chem.MolFromSmarts('[#6]-[#6](:[#7]:[#6](:[#6]):[#6]):[#6]'))\n",
    "Chem.MolFromSmiles('C:C(C):N:C(:C):C')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:34:14.089014Z",
     "start_time": "2021-02-10T18:34:14.080931Z"
    }
   },
   "outputs": [],
   "source": [
    "testmol=Chem.MolFromSmiles('Cc1cc(-c2nc3cc(F)ccc3c3cc4c(cc23)OCO4)ccc1F')\n",
    "testmol=Chem.AddHs(testmol)\n",
    "patt=Chem.MolFromSmarts('[#6]-[#6](:[#7]:[#6](:[#6]):[#6]):[#6]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:34:15.908787Z",
     "start_time": "2021-02-10T18:34:15.900841Z"
    }
   },
   "outputs": [],
   "source": [
    "testmol.GetSubstructMatches(patt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T18:34:18.117038Z",
     "start_time": "2021-02-10T18:34:18.077156Z"
    }
   },
   "outputs": [],
   "source": [
    "testmol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T17:21:15.933388Z",
     "start_time": "2021-02-10T17:21:15.906051Z"
    }
   },
   "outputs": [],
   "source": [
    "dasktestset.loc[0].compute()['Smiles'].values[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Real dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2021-02-10T18:55:45.277Z"
    }
   },
   "outputs": [],
   "source": [
    "carrierfragsmarts=substancedbdask2.map_partitions(getfragpartition,natoms=6,meta=('FragmentSmarts','O')).compute()\n",
    "carrierfragsmartsdf=pd.DataFrame(carrierfragsmarts)\n",
    "writepickle(carrierfragsmartsdf,\"/home/aa2133/Impurity-Project/Reaxys_Data/fragsmarts(n=6)\")\n",
    "\n",
    "substancedbdask2=client.persist(dd.from_pandas(joindf(carrierfragsmartsdf,fdbm),npartitions=16))\n",
    "carrierfragsmiles=substancedbdask2.map_partitions(getsmilespartition,meta=('FragmentSmiles','O')).compute()\n",
    "carrierfragsmilesdf=pd.DataFrame(carrierfragsmiles)\n",
    "writepickle(carrierfragsmilesdf,\"/home/aa2133/Impurity-Project/Reaxys_Data/fragsmiles(n=6)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T03:41:07.212034Z",
     "start_time": "2021-02-11T03:39:37.346991Z"
    }
   },
   "outputs": [],
   "source": [
    "carrierfragsmartsdf=openpickle(\"/home/aa2133/Impurity-Project/Reaxys_Data/fragsmarts(n=6).pickle\")\n",
    "carrierfragsmilesdf=openpickle(\"/home/aa2133/Impurity-Project/Reaxys_Data/fragsmiles(n=6).pickle\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now have columns for fragment smarts and fragment smiles of each compound in the database, taking roughly 1.5 hours using 16 cores on the server. This is almost 100 times faster than just using a single core, which would take almost 3-4 days."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T03:41:11.297422Z",
     "start_time": "2021-02-11T03:41:10.962134Z"
    }
   },
   "outputs": [],
   "source": [
    "carrierfragsmartsdf\n",
    "carrierfragsmilesdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T02:47:19.530585Z",
     "start_time": "2021-02-11T02:47:17.369656Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask2.memory_usage(deep=True).compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Restarting workers after computation will free up memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T02:58:21.836690Z",
     "start_time": "2021-02-11T02:58:12.670683Z"
    }
   },
   "outputs": [],
   "source": [
    "client.restart()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assembling final dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:23:20.690943Z",
     "start_time": "2021-02-11T04:23:20.673431Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have all the columns we intend to add, we can merge the fragment smarts and fragment smiles columns (exploded so each column is a new fragment), and merge the combined dataframe with the master database (fdbm). The index is set to a multiindex, containing fragment smiles and substance ID.  Keep in mind that if generating a dask dataframe from this, the index needs to be reset as dask is not compatible with multiindex. The final file is writted to disk. A test is done on 20 rows and then replicated for the entire dataset. It is hard to reshuffle data in the final database due to its size (68 million rows) or implode it back to a list form. There may be a better way to handle such large datasets (SQL for instance, but on trying this, the disk space is massive due to metadata >18 gb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:30:39.376803Z",
     "start_time": "2021-02-11T04:30:39.370283Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbmlite=fdbm.loc[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:30:44.713147Z",
     "start_time": "2021-02-11T04:30:44.695040Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbmlite\n",
    "cfsmartslite=carrierfragsmartsdf.loc[:20]\n",
    "cfsmileslite=carrierfragsmilesdf.loc[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:46:57.864580Z",
     "start_time": "2021-02-11T04:46:57.808926Z"
    }
   },
   "outputs": [],
   "source": [
    "cfsmarts=cfsmartslite.explode('FragmentSmarts')\n",
    "cfsmarts['FragmentSmiles']=cfsmileslite.explode('FragmentSmiles')\n",
    "joindf(cfsmarts,fdbmlite)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:43:38.262644Z",
     "start_time": "2021-02-11T04:42:22.311816Z"
    }
   },
   "outputs": [],
   "source": [
    "carrierfragsmartsdf=carrierfragsmartsdf.explode('FragmentSmarts')\n",
    "carrierfragsmartsdf['FragmentSmiles']=carrierfragsmilesdf.explode('FragmentSmiles')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:43:41.683606Z",
     "start_time": "2021-02-11T04:43:41.663124Z"
    }
   },
   "outputs": [],
   "source": [
    "carrierfragsmartsdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:49:52.189506Z",
     "start_time": "2021-02-11T04:49:48.198118Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbm.drop('FragmentSmarts',axis=1,inplace=True) #Option (to clean dataframe if excess columns are already there)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:50:49.583228Z",
     "start_time": "2021-02-11T04:50:33.504912Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbm=joindf(carrierfragsmartsdf,fdbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:50:59.745618Z",
     "start_time": "2021-02-11T04:50:59.712474Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fdbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:53:04.287595Z",
     "start_time": "2021-02-11T04:53:04.279284Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbm.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T04:56:51.447772Z",
     "start_time": "2021-02-11T04:56:13.223819Z"
    }
   },
   "outputs": [],
   "source": [
    "fdbm.set_index(['FragmentSmiles','SubstanceID'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T05:00:18.606765Z",
     "start_time": "2021-02-11T04:58:55.735684Z"
    }
   },
   "outputs": [],
   "source": [
    "writepickle(fdbm,'/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDB(n=6)')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entire above workflow (Section 4) has been condensed into a function, `buildfragmentdb`in Section 1. Applying this function to generate the active fragment (n = 0) database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T10:29:46.578120Z",
     "start_time": "2021-01-29T10:26:19.265532Z"
    }
   },
   "outputs": [],
   "source": [
    "buildfragdb(fdbm=testset,fragseries=openpickle(\"/home/aa2133/Impurity-Project/Reaxys_Data/cf_res.pickle\"),pfdb=True,writepfdb=\"/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragment(n=6)DB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T16:39:31.379292Z",
     "start_time": "2021-01-29T16:37:10.877381Z"
    }
   },
   "outputs": [],
   "source": [
    "buildfragdb(fdb=ActiveFragmentDB,dfdb=True,writedfdb='/home/aa2133/Impurity-Project/Reaxys_Data/ActiveFragmentDBDask')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T16:43:06.269664Z",
     "start_time": "2021-01-29T16:40:24.573916Z"
    }
   },
   "outputs": [],
   "source": [
    "buildfragdb(fdb=CarrierFragmentDB,dfdb=True,writedfdb='/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDBDask(n=6)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-30T12:46:27.828322Z",
     "start_time": "2021-01-30T12:45:20.646873Z"
    }
   },
   "outputs": [],
   "source": [
    "buildfragdb(fdb=openpickle(\"/home/aa2133/Impurity-Project/Reaxys_Data/ActiveFragmentDB.pickle\"),pfdb=True,writepfdb=\"/home/aa2133/Impurity-Project/Reaxys_Data/ActiveFragmentDB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T18:17:42.476032Z",
     "start_time": "2021-02-03T18:17:09.523273Z"
    }
   },
   "outputs": [],
   "source": [
    "buildfragdb(fdb=CarrierFragmentDB,writepfdb=\"/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDB(n=6)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing and cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T17:03:15.086262Z",
     "start_time": "2021-01-29T17:03:15.079407Z"
    }
   },
   "source": [
    "The main takeaways here are that the dask dataframe is faster on multiple tries after persisting than pandas version due to more cores able to search. Also, it is more amenable for intensive parallel computation that requires processing of each row. However, it is far more memory intensive, takes up more space (by a factor of 8 times in some cases) on disk and does not support multi-indexing. \n",
    "\n",
    "As the dask database is likely to be grouped by fragment it makes the most sense to apply it as an index (keeping in mind that creating a dask dataframe needs to be done by resetting the index or setting substance id as an index). *This is difficult, as the kernel keeps dying for some reason. For now, substance ID is kept as the index.*\n",
    "\n",
    "For the pandas dataframe, multindexing by fragment and substance ID is most effective both in reducing disk space of the database, and speed of access. If the database can be loaded to memory, pandas is preferable (Takes roughly 1.5 minutes to load)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Carrier Fragment Database (n=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dask dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T09:34:43.052337Z",
     "start_time": "2021-02-01T09:34:42.812985Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask=dd.read_parquet('/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDBDask(n=6)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T08:51:54.696127Z",
     "start_time": "2021-02-01T08:51:54.684189Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.set_index('SubstanceID',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T09:34:58.904814Z",
     "start_time": "2021-02-01T09:34:58.811806Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask=dd.from_pandas(CarrierFragmentDB,npartitions=181)\n",
    "\n",
    "# CarrierFragmentDBDask=CarrierFragmentDBDask.repartition(partition_size=100000000)\n",
    "\n",
    "CarrierFragmentDBDask=client.persist(CarrierFragmentDBDask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T09:09:33.093669Z",
     "start_time": "2021-02-01T09:09:31.830506Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.loc[CarrierFragmentDBDask['CarrierFragments']=='[#7](-[#6](:[#6]):[#6])(-[H])-[H]'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T09:13:02.335962Z",
     "start_time": "2021-02-01T09:11:54.243033Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.to_parquet('/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDBDask(n=6)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T09:32:04.294902Z",
     "start_time": "2021-02-01T09:32:04.176536Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB2=openpickle('/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDBDask(n=6).pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T09:15:32.351835Z",
     "start_time": "2021-02-01T09:15:32.339996Z"
    }
   },
   "outputs": [],
   "source": [
    "writepickle(CarrierFragmentDBDask,'/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDBDask(n=6)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T17:56:03.346652Z",
     "start_time": "2021-01-29T17:56:02.601437Z"
    }
   },
   "outputs": [],
   "source": [
    "demoset=CarrierFragmentDBDask.loc['[#6]-[#6](=[#8])-[#8]-[#6](-[#6])=[#8]'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T17:29:19.476263Z",
     "start_time": "2021-01-29T17:29:18.761047Z"
    }
   },
   "outputs": [],
   "source": [
    "demoset=CarrierFragmentDBDask.loc[CarrierFragmentDBDask['CarrierFragments']=='[#6]-[#6](=[#8])-[#8]-[#6](-[#6])=[#8]'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T18:11:55.677000Z",
     "start_time": "2021-01-29T18:11:55.502562Z"
    }
   },
   "outputs": [],
   "source": [
    "demoset['SubstanceID']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T17:34:09.068228Z",
     "start_time": "2021-01-29T17:34:09.045381Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles(demoset.loc[11305,'Smiles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-30T11:19:26.531638Z",
     "start_time": "2021-01-30T11:19:26.492470Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask2.loc['Error'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-30T11:19:23.475167Z",
     "start_time": "2021-01-30T11:19:22.689173Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.loc[CarrierFragmentDBDask['CarrierFragments']=='Error'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-30T11:12:32.388518Z",
     "start_time": "2021-01-30T11:12:32.130181Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.loc[71120:73000].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T17:13:27.787399Z",
     "start_time": "2021-01-29T17:13:27.780678Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.known_divisions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T17:01:49.474820Z",
     "start_time": "2021-02-03T17:01:38.939319Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB=openpickle(\"/home/aa2133/Impurity-Project/Reaxys_Data/CarrierFragmentDB(n=6).pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T17:02:12.768687Z",
     "start_time": "2021-02-03T17:02:09.225962Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T17:37:17.273774Z",
     "start_time": "2021-02-03T17:37:17.257928Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask=CarrierFragmentDBDask.rename(columns={'CarrierFragments':'CarrierFragmentSmarts'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T17:41:29.472002Z",
     "start_time": "2021-02-03T17:41:29.439029Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For information on meta types see https://www.geeksforgeeks.org/data-type-object-dtype-numpy-python/. For how to use it on dask, see make_meta under dask dataframe API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-09T06:15:02.630227Z",
     "start_time": "2021-02-09T06:15:02.581412Z"
    }
   },
   "outputs": [],
   "source": [
    "help(np.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T17:56:24.665809Z",
     "start_time": "2021-02-03T17:46:54.161836Z"
    }
   },
   "outputs": [],
   "source": [
    "smiles=CarrierFragmentDBDask.map_partitions(getsmilespartition,meta=('CarrierFragmentSmiles','O')).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T17:57:07.858406Z",
     "start_time": "2021-02-03T17:57:05.529841Z"
    }
   },
   "outputs": [],
   "source": [
    "smilesdf=pd.DataFrame(smiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T17:58:58.472761Z",
     "start_time": "2021-02-03T17:58:50.803409Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB=CarrierFragmentDB.join(smilesdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T18:00:34.010228Z",
     "start_time": "2021-02-03T18:00:33.999324Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.rename(columns={'CarrierFragments':'CarrierFragmentSmarts'},inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T18:04:40.967300Z",
     "start_time": "2021-02-03T18:04:07.765687Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.set_index(['CarrierFragmentSmiles','SubstanceID'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T18:04:45.235121Z",
     "start_time": "2021-02-03T18:04:45.213748Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T18:05:05.702865Z",
     "start_time": "2021-02-03T18:04:57.008815Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.loc['Error']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ".xs is used for multiindexed dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T18:05:23.615916Z",
     "start_time": "2021-02-03T18:05:23.343488Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.xs('Error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T08:45:14.175075Z",
     "start_time": "2021-02-01T08:45:13.980749Z"
    }
   },
   "outputs": [],
   "source": [
    "test1=CarrierFragmentDB.xs('[#7](-[#6](:[#6]):[#6])(-[H])-[H]').index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T08:45:16.975788Z",
     "start_time": "2021-02-01T08:45:16.117790Z"
    }
   },
   "outputs": [],
   "source": [
    "test2=CarrierFragmentDBDask.loc[CarrierFragmentDBDask['CarrierFragments']=='[#7](-[#6](:[#6]):[#6])(-[H])-[H]'].index.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T08:50:00.384976Z",
     "start_time": "2021-02-01T08:49:57.409689Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDBDask.index.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-05T14:45:10.525408Z",
     "start_time": "2021-02-05T14:45:10.367899Z"
    }
   },
   "outputs": [],
   "source": [
    "# CarrierFragmentDB.xs(294913,level=1)\n",
    "CarrierFragmentDB.xs(4502,level=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-05T14:45:46.559290Z",
     "start_time": "2021-02-05T14:45:46.546090Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('Nc1ccc2ncccc2c1N')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T18:07:53.633774Z",
     "start_time": "2021-02-03T18:07:53.104531Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.xs('[H]N([H])C(:C):C')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-03T18:09:00.276407Z",
     "start_time": "2021-02-03T18:08:54.647423Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.loc[CarrierFragmentDB['CarrierFragmentSmarts']=='[#7](-[#6](:[#6]):[#6])(-[H])-[H]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T04:29:06.777494Z",
     "start_time": "2021-02-01T04:29:05.363994Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmarts('[#7](-[#6](:[#6]):[#6])(-[H])-[H]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T15:23:36.999050Z",
     "start_time": "2021-02-01T15:23:36.835868Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.xs('[#6]-[#6](=[#8])-[#8]-[#6](-[#6])=[#8]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T04:31:18.808784Z",
     "start_time": "2021-02-01T04:31:18.338503Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.xs(11305,level=1,drop_level=False)\n",
    "CarrierFragmentDB.xs('[#6]-[#6](=[#8])-[#8]-[#6](-[#6])=[#8]',drop_level=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T16:59:26.771439Z",
     "start_time": "2021-01-29T16:59:26.390101Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('Nc1ccc2[nH]cnc2c1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SQL Database"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Very slow write speed (>15 mins for 65 million rows). Only benefit is compatibility outside python, and reading from disk due to memory constraints. It is interestingly quite fast (1 second), and at most half a minute. See https://pandas.pydata.org/pandas-docs/version/0.15.0/io.html#sql-queries. Obviously it cannot compare to in-memory pandas which when indexed properly takes less than a second. However, this could be useful in cases where memory is scarce or the database is far too large to load in memory. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T15:51:04.794304Z",
     "start_time": "2021-02-01T15:51:04.789770Z"
    }
   },
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T15:50:39.291174Z",
     "start_time": "2021-02-01T15:50:36.283705Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T16:24:52.938745Z",
     "start_time": "2021-02-01T16:24:52.933688Z"
    }
   },
   "outputs": [],
   "source": [
    "db2=sqlite3.connect(\"/home/aa2133/Impurity-Project/Reaxys_Data/SQL/Reaxys_Data.db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T16:07:58.031285Z",
     "start_time": "2021-02-01T16:07:56.527901Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.set_index('CarrierFragments',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T16:22:55.781769Z",
     "start_time": "2021-02-01T16:08:20.694193Z"
    }
   },
   "outputs": [],
   "source": [
    "CarrierFragmentDB.to_sql(\"CarrierFragment(n=6)\",db,if_exists=\"replace\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T05:33:22.946237Z",
     "start_time": "2021-02-02T05:30:09.992455Z"
    }
   },
   "outputs": [],
   "source": [
    "cursor = db2.cursor()\n",
    "df = pd.read_sql_query('''select * from [CarrierFragment(n=6)]''', db2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T05:34:38.814077Z",
     "start_time": "2021-02-02T05:34:38.658735Z"
    }
   },
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T16:31:37.728334Z",
     "start_time": "2021-02-01T16:31:37.719654Z"
    }
   },
   "outputs": [],
   "source": [
    "cursor.execute(\"SELECT name FROM sqlite_master WHERE type='table';\")\n",
    "tables = cursor.fetchall()\n",
    "for table_name in tables:\n",
    "    table_name = table_name[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T05:07:44.243532Z",
     "start_time": "2021-02-02T05:06:00.960603Z"
    }
   },
   "outputs": [],
   "source": [
    "cursor.execute('''CREATE INDEX CarrierFragments ON [CarrierFragment(n=6)] (CarrierFragments);''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T05:09:18.452064Z",
     "start_time": "2021-02-02T05:09:18.196011Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.read_sql_query('''SELECT * from [CarrierFragment(n=6)] Where CarrierFragments=\"[#6]-[#6](=[#8])-[#8]-[#6](-[#6])=[#8]\"''',db2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T05:10:31.913430Z",
     "start_time": "2021-02-02T05:09:32.049876Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.read_sql_query('''SELECT * from [CarrierFragment(n=6)] Where Smiles=\"CC(=O)Nc1ccc(O)cc1\"''',db2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-02T05:11:34.185243Z",
     "start_time": "2021-02-02T05:11:25.869377Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.read_sql_query('''SELECT * from [CarrierFragment(n=6)] Where Smiles=\"CC(=O)OC(=O)c1ccc(C(=O)OC(C)=O)o1\"''',db2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T16:56:44.546703Z",
     "start_time": "2021-02-01T16:56:44.337825Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolToSmiles(Chem.MolFromSmiles('CC(=O)NC1=CC=C(C=C1)O'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T17:09:46.892023Z",
     "start_time": "2021-02-01T17:09:46.873081Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmarts('[#6]-[#6](=[#8])-[#7](-[#6])-[H]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-01T17:10:44.556822Z",
     "start_time": "2021-02-01T17:10:44.358705Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('CC(=O)Nc1ccc(O)cc1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Active Fragment Database (n=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dask dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T17:39:53.801299Z",
     "start_time": "2021-01-29T17:39:53.448312Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDBDask=dd.read_parquet('/home/aa2133/Impurity-Project/Reaxys_Data/ActiveFragmentDBDask')\n",
    "ActiveFragmentDBDask=client.persist(ActiveFragmentDBDask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-29T17:45:51.323764Z",
     "start_time": "2021-01-29T17:45:48.812747Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDBDask.loc[ActiveFragmentDBDask['ActiveFragments']=='[#8]-[Si]'].compute()\n",
    "ActiveFragmentDBDask.loc[ActiveFragmentDBDask['ActiveFragments']=='[Si]-[#8]'].compute()\n",
    "ActiveFragmentDBDask.loc[ActiveFragmentDBDask['ActiveFragments']=='Error'].compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-30T12:47:46.136432Z",
     "start_time": "2021-01-30T12:47:36.658261Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDB=openpickle(\"/home/aa2133/Impurity-Project/Reaxys_Data/ActiveFragmentDB.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-30T12:48:27.693414Z",
     "start_time": "2021-01-30T12:48:27.614734Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-30T12:48:06.824390Z",
     "start_time": "2021-01-30T12:48:06.025191Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDB.loc['[#8]-[Si]']\n",
    "ActiveFragmentDB.loc['[Si]-[#8]']\n",
    "ActiveFragmentDB.loc['Error']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As shown above, it is possible to group compounds and their smiles based on active fragment. Some records encountered errors (238). This is because of incorrect RDKit translation of molfile to smiles strings. Whilst the issue is resolved by reading smarts instead of smiles, the carrierfrags function will need to be adapted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T05:26:07.086341Z",
     "start_time": "2021-01-28T05:26:07.066682Z"
    }
   },
   "outputs": [],
   "source": [
    "errormol=Chem.MolFromMolFile(substancedb.loc[71120,'MolFileAddress'])\n",
    "errormol\n",
    "Chem.MolFromSmiles(Chem.MolToSmiles(errormol))\n",
    "Chem.MolFromSmiles('C=Cc1cc(OC)c(OC)c2c1/c=c\\\\c1ccc(cc1)-o-c1c(c(C=C)cc(OC)c1OC)/c=c\\\\c1ccc(cc1)-o-2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T06:45:42.259791Z",
     "start_time": "2021-01-28T06:45:42.241915Z"
    }
   },
   "outputs": [],
   "source": [
    "deem=Chem.MolFromSmarts(Chem.MolToSmarts(errormol))\n",
    "Chem.SanitizeMol(deem)\n",
    "deem.UpdatePropertyCache(strict=False)\n",
    "deem=Chem.rdmolops.AddHs(deem)\n",
    "IFG(deem)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Two cells below check whether it is faster to load a mol from molfile, or generating it from smiles. The latter is slightly faster, but the former is useful in cases where the smiles is incorrect as is the case above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T06:39:51.538152Z",
     "start_time": "2021-01-28T06:39:51.530269Z"
    }
   },
   "outputs": [],
   "source": [
    "%%timeit -n 1 -r 1\n",
    "Chem.MolFromMolFile(substancedb.loc[47408,'MolFileAddress'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T06:39:53.531232Z",
     "start_time": "2021-01-28T06:39:53.525669Z"
    }
   },
   "outputs": [],
   "source": [
    "%%timeit -n 1 -r 1\n",
    "Chem.MolFromSmiles(substancedb.loc[47408,'Smiles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T06:21:27.644928Z",
     "start_time": "2021-01-28T06:21:27.640198Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolToSmarts(errormol)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T04:56:16.139575Z",
     "start_time": "2021-01-28T04:56:14.337898Z"
    }
   },
   "outputs": [],
   "source": [
    "len(ActiveFragmentDB.loc['[#8]-[Si]','Smiles'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T04:58:00.133655Z",
     "start_time": "2021-01-28T04:57:52.302519Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDB.index.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T18:37:21.694365Z",
     "start_time": "2021-01-27T18:37:21.510919Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDB.loc[ActiveFragmentDB['SubstanceID']==33195,'Smiles']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T04:56:24.722122Z",
     "start_time": "2021-01-28T04:56:24.703538Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('CO[C@@H]1O[C@@H]2[C@H](O[Si](C)(C)C)C(=O)O[C@@H]2[C@H]1O[Si](C)(C)C')\n",
    "getCarrierFrags('CO[C@@H]1O[C@@H]2[C@H](O[Si](C)(C)C)C(=O)O[C@@H]2[C@H]1O[Si](C)(C)C',0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T03:47:47.022436Z",
     "start_time": "2021-01-28T03:47:47.010076Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolFromSmiles('C[Si]1(C)CCCO1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T03:50:09.474012Z",
     "start_time": "2021-01-28T03:50:09.405672Z"
    }
   },
   "outputs": [],
   "source": [
    "Chem.MolToSmiles(Chem.MolFromSmarts('[Si]-[#8]'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T03:51:15.841959Z",
     "start_time": "2021-01-28T03:51:15.832865Z"
    }
   },
   "outputs": [],
   "source": [
    "IFG(Chem.MolFromSmiles('C[Si]1(C)CCCO1'))\n",
    "IFG(Chem.MolFromSmiles('CO[C@@H]1O[C@@H]2[C@H](O[Si](C)(C)C)C(=O)O[C@@H]2[C@H]1O[Si](C)(C)C'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T03:53:43.039256Z",
     "start_time": "2021-01-28T03:53:43.032263Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags('C[Si]1(C)CCCO1',0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T18:05:16.475253Z",
     "start_time": "2021-01-27T18:05:16.470811Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDB.loc[1000001,'ActiveFragments']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T18:06:55.730369Z",
     "start_time": "2021-01-27T18:06:55.722270Z"
    }
   },
   "outputs": [],
   "source": [
    "ActiveFragmentDB.loc[1000001,'Smiles']\n",
    "ActiveFragmentDB.loc[1000001,'SubstanceID']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T18:05:36.499522Z",
     "start_time": "2021-01-27T18:05:36.491435Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags(ActiveFragmentDB.loc[1000001,'Smiles'],0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T17:43:14.860072Z",
     "start_time": "2021-01-27T17:43:14.832573Z"
    }
   },
   "outputs": [],
   "source": [
    "getCarrierFrags(testset.loc[testset['SubstanceID']==28713377,'Smiles'][0],0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T17:50:11.881699Z",
     "start_time": "2021-01-27T17:50:11.868422Z"
    }
   },
   "outputs": [],
   "source": [
    "dfaf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T17:26:17.203656Z",
     "start_time": "2021-01-27T17:25:37.457536Z"
    }
   },
   "outputs": [],
   "source": [
    "dfafdask=dd.from_pandas(dfaf,npartitions=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T17:58:08.721682Z",
     "start_time": "2021-01-27T17:58:03.897254Z"
    }
   },
   "outputs": [],
   "source": [
    "dfafdask.loc[0,'ActiveFragments'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T06:54:24.312014Z",
     "start_time": "2021-01-27T06:54:24.279936Z"
    }
   },
   "outputs": [],
   "source": [
    "testset.loc[0,'Smiles']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T16:01:24.384809Z",
     "start_time": "2021-01-28T16:01:24.089287Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedbdask.loc[substancedbdask['Smiles']=='COC(OC)C(=O)COC(C)=O'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T16:01:49.385205Z",
     "start_time": "2021-01-28T16:01:47.629266Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb.loc[substancedb['Smiles']=='COC(OC)C(=O)COC(C)=O']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Further results have been discussed/included in Preidct_Impurities.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-15T06:15:52.134808Z",
     "start_time": "2021-02-15T06:15:51.952275Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%whos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-18T13:11:24.179454Z",
     "start_time": "2021-02-18T13:11:24.128813Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding an entry by Smiles and Reaxys ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-18T13:14:58.714458Z",
     "start_time": "2021-02-18T13:14:58.651789Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb.xs('CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O-].[Li+]')\n",
    "substancedb.xs('CC(C)(C)c1ccc2c(c1)c1cc(C(C)(C)C)ccc1n2C(=O)[O-].[Li+]').index[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding an entry by Reaxys ID and Smiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-18T13:15:26.470808Z",
     "start_time": "2021-02-18T13:15:26.404872Z"
    }
   },
   "outputs": [],
   "source": [
    "substancedb.xs(22727365,level=1)\n",
    "substancedb.xs(22727365,level=1).index[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:impurityproject] *",
   "language": "python",
   "name": "conda-env-impurityproject-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "330.641px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}