Consider only the survey of interest

Now that PP processes test surveys besides the imaging survey,
the notebook needs to filter more carefully based on the survey.

Also, file counting of all tasks is moved to the V&V notebook.

prompt-processing/groups.ipynb

@@ -537,44 +537,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def count_files(dataset_type = \"isr_log\"):\n",
+    "def count_files(dataset_type = \"apdb_marker\", extra=\"\"):\n",
     "    df = get_df_file(\n",
     "        df_expId,\n",
     "        butler,\n",
     "        dataset_type,\n",
     "        collections=collections,\n",
+    "        where=extra,\n",
     "    )\n",
     "    count = len(df[df[\"file\"].notnull()].index)\n",
     "    logger.info(f\"{count} {dataset_type} were stored\")\n",
     "    return df.reset_index().set_index([\"groupId\", \"detector\"])"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "014c2f3c-1441-4567-aadf-bf14da1369ff",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# TODO: this will be moved to the V&V notebook, with up-to-date tasks\n",
-    "tasks = [\"isr\", \"calibrateImage\", \"retrieveTemplate\", \"subtractImages\",\n",
-    "         \"detectAndMeasure\", \"diffimTaskCore\", \"filterDiaSrcCat\",\n",
-    "         \"transformDiaSrcCat\", \"diaPipe\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c8fbfa89-b312-4005-86d2-d425e8f793e1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "file_counts = dict()\n",
-    "if level < 20:\n",
-    "    for dataset_type in [_+\"_log\" for _ in tasks]:\n",
-    "        file_counts[dataset_type] = count_files(dataset_type)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -584,7 +559,8 @@
    "source": [
     "df_output = count_files(\"apdb_marker\")\n",
     "df_output = count_files(\"initial_pvi\")\n",
-    "df_output = count_files(\"isr_log\")\n",
+    "df_output = count_files(\"isr_log\",\n",
+    "                        extra=f\"exposure.science_program IN ('{survey}') and instrument='{instrument}' and day_obs={dayobs}\")\n",
     "count = len(df_output[df_output[\"file\"].notnull()].index)"
    ]
   },
@@ -698,7 +674,7 @@
    "outputs": [],
    "source": [
     "def examine_log(key):\n",
-    "    df2 = get_df_from_loki(date, phrases[key])\n",
+    "    df2 = get_df_from_loki(date, phrases[key], extra=f\" |~ \\\"{instrument}\\\" |~ \\\"{survey}\\\" \")\n",
     "    ids = df2[df2[\"ts\"].notnull()][[\"group\", \"detector\"]]\n",
     "    if len(ids):\n",
     "        logger.debug(f\"{len(ids)}   with error {phrases[key]}: {ids} \")\n",