content/03_results/12_data_testing_bkp.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c6068afb",
   "metadata": {},
   "source": [
    "# Data Testing\n",
    "\n",
    "This notebook contains some testing of our data sources to identify potential data coding errors in the main and best practice datasets.\n",
    "\n",
    "We include the following groupings for testing\n",
    "\n",
    "* Smoke tests: These test that the data files exist and can be loaded for analysis. Nothing can run without passing these tests.\n",
    "* Testing data fields and subgroups: These test that the data is coded correctly and completely.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a034af2",
   "metadata": {},
   "source": [
    "## 1. Imports \n",
    "\n",
    "### 1.1. Standard Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "45a52a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# used in smoke tests\n",
    "import requests"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3b7d139-8137-42ef-9785-7f0c9102e9a6",
   "metadata": {},
   "source": [
    "### 1.2 Testing imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "72db49d2-fe46-41a0-9310-3d3ba09153c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ipytest\n",
    "import pytest\n",
    "ipytest.autoconfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1560e13-c8b7-4fa9-a1b7-c6614ad0001c",
   "metadata": {},
   "source": [
    "## 1.2 Imports from preprocessing module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b2c8c53b-036c-476d-8fa9-d7d4573b0e27",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for loading full dataset\n",
    "from preprocessing import load_clean_dataset, load_clean_bpa"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67a596c0",
   "metadata": {},
   "source": [
    "## 2. Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "195aa1fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE_NAME_MAIN = 'https://raw.githubusercontent.com/TomMonks/' \\\n",
    "    + 'des_sharing_lit_review/main/data/share_sim_data_extract.zip'\n",
    "\n",
    "FILE_NAME_BPA = 'https://raw.githubusercontent.com/TomMonks/' \\\n",
    "    + 'des_sharing_lit_review/main/data/bp_audit.zip'\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a177b96-65a5-447d-93a1-b16fe06464ce",
   "metadata": {},
   "source": [
    "## 3. Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6a3d5251-ec1f-48a8-bd95-dbd628d2f085",
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_covid_in_year(df, year):\n",
    "    '''\n",
    "    Return the integer number of studies with a Covid-19 DES model\n",
    "    in a specified publication year.\n",
    "\n",
    "    Params:\n",
    "    -----\n",
    "    df: pd.DataFrame\n",
    "        Subgroup dataset\n",
    "        \n",
    "    year: int\n",
    "        Year of publication\n",
    "        \n",
    "    Returns:\n",
    "    -------\n",
    "    int\n",
    "    '''\n",
    "    return len(df[(df['pub_yr']==year) & (df['covid'] == 1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "794198f2-0f2a-4ea2-874e-a8204b2b7c02",
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_included_in_year(df, year):\n",
    "    '''\n",
    "    Return the integer number of studies included\n",
    "    in a specified publication year.\n",
    "\n",
    "    Params:\n",
    "    -----\n",
    "    df: pd.DataFrame\n",
    "        Subgroup dataset\n",
    "        \n",
    "    year: int\n",
    "        Year of publication\n",
    "        \n",
    "    Returns:\n",
    "    -------\n",
    "    int\n",
    "    '''\n",
    "    return len(df[(df['pub_yr']==year) & (df['study_included'] == 1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7aeb4180-8144-48d7-b780-7444b33081b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nan_fields(df, field):\n",
    "    '''\n",
    "    Return the dataframe containing all rows\n",
    "    where study_included is NaN (empty)\n",
    "    \n",
    "    Params:\n",
    "    -----\n",
    "    df: pd.DataFrame\n",
    "        Subgroup dataset\n",
    "        \n",
    "    field: str\n",
    "        The name of the field to check for NaNs\n",
    "        \n",
    "        \n",
    "    Returns:\n",
    "    -------\n",
    "    int\n",
    "    '''\n",
    "    return len(df[(df[field].isnull()) & (df['study_included'] == 1)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "727c8ba3-160b-49d1-ac11-ccf3248ca3d6",
   "metadata": {},
   "source": [
    "## 4. Smoke Testing\n",
    "\n",
    "### 4.1. Data Files Exist\n",
    "\n",
    "* First check if the data files exist at the specified URLs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f3bad34a-06d7-43dc-9f6a-7be801b02bb5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m                                                                                           [100%]\u001b[0m\n",
      "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 0.55s\u001b[0m\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%ipytest\n",
    "@pytest.mark.parametrize('url', [\n",
    "    (FILE_NAME_MAIN),\n",
    "    (FILE_NAME_BPA),\n",
    "]) \n",
    "def test_file_exists_at_url(url):\n",
    "    '''\n",
    "    Request data file from URL. Test is status code = 200 (exists)\n",
    "    \n",
    "    Params:\n",
    "    ------\n",
    "    url: str\n",
    "        URL to test\n",
    "    '''\n",
    "    r = requests.head(url, allow_redirects=True)\n",
    "    # if code = 200 then resource exists.\n",
    "    assert r.status_code == 200"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b6af259-ce1f-4863-a3f6-4d2b0f5e0420",
   "metadata": {},
   "source": [
    "### 4.2 Preprocessing logic returns DataFrame containing data\n",
    "* Read in check for any data and that it loads into a `DataFrame`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "23453b39-88f8-4229-b301-a353b0cc2e42",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m                                                                                           [100%]\u001b[0m\n",
      "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 0.61s\u001b[0m\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%ipytest\n",
    "\n",
    "@pytest.mark.parametrize('url, func', [\n",
    "    (FILE_NAME_MAIN, load_clean_dataset),\n",
    "    (FILE_NAME_BPA, load_clean_bpa),\n",
    "]) \n",
    "def test_load_dataframe(url, func):\n",
    "    '''\n",
    "    Test that the preprocessing function can return a populated dataframe\n",
    "    for the specified URL.\n",
    "    \n",
    "    tested by checking df has at least 1 row.\n",
    "    \n",
    "    Params:\n",
    "    ------\n",
    "    url: str\n",
    "        URL for file\n",
    "        \n",
    "    func: object\n",
    "        Python function containing preprocessing logic. Assume that it returns\n",
    "        a dataframe.\n",
    "    '''\n",
    "    df = func(url)\n",
    "    assert len(df) > 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37aad232-3a42-45fe-b818-7fd866e1ca9b",
   "metadata": {},
   "source": [
    "## 5. Read in data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2b2ff536-d209-4803-81d1-6068b1df3def",
   "metadata": {},
   "outputs": [],
   "source": [
    "clean = load_clean_dataset(FILE_NAME_MAIN)\n",
    "clean_bpa = load_clean_bpa(FILE_NAME_BPA)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "475c28ba-73fc-483b-92bf-98426025238a",
   "metadata": {},
   "source": [
    "## 6. Testing data fields and subgroups"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e61ff3c-f61f-46d9-bbcc-3d18a0d4d947",
   "metadata": {},
   "source": [
    "### 6.1 Check coding for Covid-19 models\n",
    "\n",
    "* There should be no Covid-19 DES models pre-2020."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8dc18509-b668-4686-b8d5-6443141215e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m.\u001b[0m\u001b[32m                                                                                            [100%]\u001b[0m\n",
      "\u001b[32m\u001b[32m\u001b[1m1 passed\u001b[0m\u001b[32m in 0.01s\u001b[0m\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%ipytest\n",
    "\n",
    "@pytest.mark.parametrize('df, year, expected', [\n",
    "    (clean, 2019, 0),\n",
    "]) \n",
    "def test_num_covid_in_year(df, year, expected):\n",
    "    assert num_covid_in_year(df, year) == expected\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e7c5812-9852-4cea-af98-f18e14a8e5c4",
   "metadata": {},
   "source": [
    "## 6.2 Studies included\n",
    "\n",
    "* Test each year has at least 1 study included.\n",
    "* Test that there are no studies where the study included field is set to null."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f30d8889-f26c-4ccd-934e-91874796abce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m                                                                                        [100%]\u001b[0m\n",
      "\u001b[32m\u001b[32m\u001b[1m5 passed\u001b[0m\u001b[32m in 0.01s\u001b[0m\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%ipytest\n",
    "\n",
    "@pytest.mark.parametrize('df, year', [\n",
    "    (clean, 2019),\n",
    "    (clean, 2020),\n",
    "    (clean, 2021),\n",
    "    (clean, 2022),\n",
    "]) \n",
    "def test_studies_included_in_year(df, year):\n",
    "    '''\n",
    "    Test the number of studies in a particular year\n",
    "    of a subgroup is at least 1.\n",
    "    '''\n",
    "    assert num_included_in_year(df, year) >= 1\n",
    "    \n",
    "\n",
    "    \n",
    "def test_all_studies_considered():\n",
    "    '''\n",
    "    Test that all studies have been coded as 0 or 1\n",
    "    '''\n",
    "    assert len(clean[clean['study_included'].isnull()]) == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08bbbce2-0bbd-4db6-9fed-43e1e785a8be",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 6.3. NaN fields in mandatory data extraction fields\n",
    "\n",
    "* Test that mandatory fields contain no NaN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0895cf22-b3f2-4914-a3a4-c6315975be21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m                                                                                        [100%]\u001b[0m\n",
      "\u001b[32m\u001b[32m\u001b[1m5 passed\u001b[0m\u001b[32m in 0.01s\u001b[0m\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%ipytest\n",
    "\n",
    "@pytest.mark.parametrize('df, field', [\n",
    "    (clean, 'study_included'),\n",
    "    (clean, 'covid'),\n",
    "    (clean, 'foss_sim'),\n",
    "    (clean, 'reporting_guidelines_mention'),\n",
    "    (clean, 'sim_software'),\n",
    "]) \n",
    "def test_no_nan_fields_included_studies(df, field):\n",
    "    '''\n",
    "    Test that a selected field contains no NaN data.\n",
    "    '''\n",
    "    assert nan_fields(df, field) == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b25e5f50-1332-4766-8c5c-d06a9221da9e",
   "metadata": {},
   "source": [
    "### 6.4 Binary fields only contain 0 or 1\n",
    "\n",
    "* covid and mode_code_available fields are mandatory 0 or 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "565bccbc-d8bc-471e-bbb0-9d270dbc5178",
   "metadata": {},
   "outputs": [],
   "source": [
    "def non_zero_one_coding(df, field):\n",
    "    return len(df[(df[field] != 1) & (df[field] != 0) & \n",
    "                  (df['study_included'] == 1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1710cfcf-b6ca-4db3-921b-b95d637f9701",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m                                                                                           [100%]\u001b[0m\n",
      "\u001b[32m\u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 0.01s\u001b[0m\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%ipytest\n",
    "\n",
    "@pytest.mark.parametrize('df, field', [\n",
    "    (clean, 'covid'),\n",
    "    (clean, 'model_code_available'),\n",
    "]) \n",
    "def test_zero_one_coding(df, field):\n",
    "    assert non_zero_one_coding(df, field) == 0"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}