Updated country inclusion notebook

notebooks/user/rragonnet/project_specific/School_Closure/country_inclusion.ipynb

@@ -95,17 +95,6 @@
     "print(len(pop_iso3s))"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "_list = pop_iso3s\n",
-    "print(\"GBR\" in _list)\n",
-    "print(\"USA\" in _list)"
-   ]
-  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -281,7 +270,7 @@
     "# Eligibility filters\n",
     "unity_filter = sero_data[\"is_unity_aligned\"] == \"Unity-Aligned\"\n",
     "size_filter = sero_data[\"denominator_value\"] >= 599\n",
-    "max_date_filter = sero_data.end_date <= datetime(2021, 5, 1)\n",
+    "# max_date_filter = sero_data.end_date <= datetime(2021, 5, 1)\n",
     "min_date_filter = sero_data.start_date >= datetime(2020, 5, 1)\n",
     "\n",
     "\n",
@@ -319,7 +308,45 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "filtered_sero_data_national = sero_data[unity_filter & size_filter & min_date_filter & max_date_filter & national_filter & subgroup_filter & redflag_filter] "
+    "filtered_sero_data_national = sero_data[unity_filter & size_filter & min_date_filter & national_filter & subgroup_filter & redflag_filter] "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Remove estimates where vaccine coverage is too high at the time\n",
+    "max_vacc_seroprev_ratio = .10\n",
+    "\n",
+    "from datetime import datetime\n",
+    "\n",
+    "vacc_data = input_db.query(\n",
+    "    table_name=\"owid\",\n",
+    "    columns=[\"date\", \"people_fully_vaccinated_per_hundred\", \"iso_code\"],\n",
+    ")\n",
+    "vacc_data.dropna(inplace=True)\n",
+    "vacc_data[\"date\"] = pd.to_datetime(owid_data[\"date\"])\n",
+    "\n",
+    "filtered_sero_data_national[\"low_vaccination\"] = [None] * len(filtered_sero_data_national)\n",
+    "\n",
+    "vcc = []\n",
+    "for idx in filtered_sero_data_national.index:\n",
+    "    seroprev = filtered_sero_data_national.loc[idx, 'serum_pos_prevalence']\n",
+    "    \n",
+    "    start_date = datetime.fromisoformat(filtered_sero_data_national.loc[idx, 'sampling_start_date'])\n",
+    "    end_date = datetime.fromisoformat(filtered_sero_data_national.loc[idx, 'sampling_end_date'])\n",
+    "    midpoint = start_date + (end_date - start_date) / 2  \n",
+    "\n",
+    "    if len(vacc_data[(vacc_data[\"iso_code\"] == iso3) & (vacc_data[\"date\"] >= midpoint)]) == 0:\n",
+    "        filtered_sero_data_national.loc[idx, \"low_vaccination\"] = False\n",
+    "    else:\n",
+    "        vacc_coverage = vacc_data[(vacc_data[\"iso_code\"] == iso3) & (vacc_data[\"date\"] >= midpoint)].iloc[0][\"people_fully_vaccinated_per_hundred\"] / 100.\n",
+    "\n",
+    "        filtered_sero_data_national.loc[idx, \"low_vaccination\"] = vacc_coverage < max_vacc_seroprev_ratio * seroprev\n",
+    "\n",
+    "filtered_sero_data_national = filtered_sero_data_national[filtered_sero_data_national[\"low_vaccination\"] == True]"
    ]
   },
   {
@@ -380,6 +407,9 @@
     "    )        \n",
     "    country_data = country_data[country_data['overall_risk_of_bias'] == max(country_data['overall_risk_of_bias'])]\n",
     "\n",
+    "    # we prioritise the latest estimates\n",
+    "    country_data = country_data[country_data['sampling_start_date'] == max(country_data['sampling_start_date'])]\n",
+    "\n",
     "    # we prioritise the largest sample size\n",
     "    country_data = country_data[country_data['denominator_value'] == max(country_data['denominator_value'])]\n",
     "    \n",
@@ -558,6 +588,13 @@
     "    yaml.dump(included_dict, outfile, default_flow_style=False)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,