diff --git a/prompt-processing/groups.ipynb b/prompt-processing/groups.ipynb index 520eecc..287c7a0 100644 --- a/prompt-processing/groups.ipynb +++ b/prompt-processing/groups.ipynb @@ -537,44 +537,19 @@ "metadata": {}, "outputs": [], "source": [ - "def count_files(dataset_type = \"isr_log\"):\n", + "def count_files(dataset_type = \"apdb_marker\", extra=\"\"):\n", " df = get_df_file(\n", " df_expId,\n", " butler,\n", " dataset_type,\n", " collections=collections,\n", + " where=extra,\n", " )\n", " count = len(df[df[\"file\"].notnull()].index)\n", " logger.info(f\"{count} {dataset_type} were stored\")\n", " return df.reset_index().set_index([\"groupId\", \"detector\"])" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "014c2f3c-1441-4567-aadf-bf14da1369ff", - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: this will be moved to the V&V notebook, with up-to-date tasks\n", - "tasks = [\"isr\", \"calibrateImage\", \"retrieveTemplate\", \"subtractImages\",\n", - " \"detectAndMeasure\", \"diffimTaskCore\", \"filterDiaSrcCat\",\n", - " \"transformDiaSrcCat\", \"diaPipe\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c8fbfa89-b312-4005-86d2-d425e8f793e1", - "metadata": {}, - "outputs": [], - "source": [ - "file_counts = dict()\n", - "if level < 20:\n", - " for dataset_type in [_+\"_log\" for _ in tasks]:\n", - " file_counts[dataset_type] = count_files(dataset_type)" - ] - }, { "cell_type": "code", "execution_count": null, @@ -584,7 +559,8 @@ "source": [ "df_output = count_files(\"apdb_marker\")\n", "df_output = count_files(\"initial_pvi\")\n", - "df_output = count_files(\"isr_log\")\n", + "df_output = count_files(\"isr_log\",\n", + " extra=f\"exposure.science_program IN ('{survey}') and instrument='{instrument}' and day_obs={dayobs}\")\n", "count = len(df_output[df_output[\"file\"].notnull()].index)" ] }, @@ -698,7 +674,7 @@ "outputs": [], "source": [ "def examine_log(key):\n", - " df2 = get_df_from_loki(date, phrases[key])\n", + " df2 = get_df_from_loki(date, phrases[key], extra=f\" |~ \\\"{instrument}\\\" |~ \\\"{survey}\\\" \")\n", " ids = df2[df2[\"ts\"].notnull()][[\"group\", \"detector\"]]\n", " if len(ids):\n", " logger.debug(f\"{len(ids)} with error {phrases[key]}: {ids} \")\n",