Pyterrier_Ranking.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "0edvI3LDtOym",
        "p18d1tpJvD_5",
        "8P380IAZIrds",
        "0aJrYP1TSLwY"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.6.12 64-bit ('pyterrier': conda)",
      "metadata": {
        "interpreter": {
          "hash": "9184305472e6ae9ceb428e56684b1d84c502ed6276bc2f9bba436eca2a584a1d"
        }
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9a7cdd20ed4d420ba377791ef674928d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_10135c57e801432c9cf1c677b1716ce1",
              "IPY_MODEL_7aa7d64ff43b4f54969016030f4fdfc7",
              "IPY_MODEL_86844478b4934b028512aea6b79ac271"
            ],
            "layout": "IPY_MODEL_b6268e071b9142fe8e56c22c8dadfa96"
          }
        },
        "10135c57e801432c9cf1c677b1716ce1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc8f864c25c642e693b7ca4d8fd7018f",
            "placeholder": "​",
            "style": "IPY_MODEL_1434a36207f2437da64508a67cebc574",
            "value": "cord19/trec-covid documents: 100%"
          }
        },
        "7aa7d64ff43b4f54969016030f4fdfc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a0287b66fceb469fb96ad23acbcc7398",
            "max": 192509,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_db6edaf76918403dbca749e19ece85d1",
            "value": 192509
          }
        },
        "86844478b4934b028512aea6b79ac271": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae9b899f58464f3f9e88f7aaed0617ef",
            "placeholder": "​",
            "style": "IPY_MODEL_24612a254f1d40f18faf806860dfdacd",
            "value": " 192509/192509 [01:38&lt;00:00, 2254.36it/s]"
          }
        },
        "b6268e071b9142fe8e56c22c8dadfa96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc8f864c25c642e693b7ca4d8fd7018f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1434a36207f2437da64508a67cebc574": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a0287b66fceb469fb96ad23acbcc7398": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db6edaf76918403dbca749e19ece85d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ae9b899f58464f3f9e88f7aaed0617ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "24612a254f1d40f18faf806860dfdacd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "090f3d2dbace44cda71a0e63d0882d64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_445aaa9f42ca40c8bbb95fc0c5c5f7de",
              "IPY_MODEL_ea9c0625dbbc4188a6a9b6c8d3b97f1e",
              "IPY_MODEL_ef0281748ca94d76b052e797fe6c166f"
            ],
            "layout": "IPY_MODEL_4b9c008d95204f6889dd5c3ac0ad2285"
          }
        },
        "445aaa9f42ca40c8bbb95fc0c5c5f7de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e5223cd1d45c4abcba07f337ddcf840e",
            "placeholder": "​",
            "style": "IPY_MODEL_843b38a255454bc484a64dab796fe693",
            "value": "beir/fiqa/train documents: 100%"
          }
        },
        "ea9c0625dbbc4188a6a9b6c8d3b97f1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35d72040b9e54c9a84909e3bc6a76f7a",
            "max": 57638,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_74196af47eb44385af7f664fe65698f3",
            "value": 57638
          }
        },
        "ef0281748ca94d76b052e797fe6c166f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_76976a734a4c4d899c091b65f81573e2",
            "placeholder": "​",
            "style": "IPY_MODEL_fafb87b4be6e49e0948d6aa5502a6614",
            "value": " 57638/57638 [00:18&lt;00:00, 2526.66it/s]"
          }
        },
        "4b9c008d95204f6889dd5c3ac0ad2285": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e5223cd1d45c4abcba07f337ddcf840e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "843b38a255454bc484a64dab796fe693": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "35d72040b9e54c9a84909e3bc6a76f7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74196af47eb44385af7f664fe65698f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "76976a734a4c4d899c091b65f81573e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fafb87b4be6e49e0948d6aa5502a6614": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eee782808d2240a39570c856102372d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_503fd9d4e244409487287c8d67acc8c9",
              "IPY_MODEL_b1d15d78637241cd9c3534c42993309b",
              "IPY_MODEL_84780546219744fe866441023ab0b63f"
            ],
            "layout": "IPY_MODEL_4dc50292591f41a2a50b00fc38f582cb"
          }
        },
        "503fd9d4e244409487287c8d67acc8c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e54fdb51b87f419d835c1b3461ffc6e8",
            "placeholder": "​",
            "style": "IPY_MODEL_bf18a1f423ce4705ac2e4aecadcb5d06",
            "value": "beir/fiqa/test documents: 100%"
          }
        },
        "b1d15d78637241cd9c3534c42993309b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_615c489fa6674a06afb386b54c0c0351",
            "max": 57638,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5799ca054df1418cbd7c381dd25b6bb1",
            "value": 57638
          }
        },
        "84780546219744fe866441023ab0b63f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e99697c8d574e4ea0552ea424177849",
            "placeholder": "​",
            "style": "IPY_MODEL_bfb7c0f0cb994ce5aa8774ff0307ff92",
            "value": " 57638/57638 [00:18&lt;00:00, 3314.43it/s]"
          }
        },
        "4dc50292591f41a2a50b00fc38f582cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e54fdb51b87f419d835c1b3461ffc6e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf18a1f423ce4705ac2e4aecadcb5d06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "615c489fa6674a06afb386b54c0c0351": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5799ca054df1418cbd7c381dd25b6bb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2e99697c8d574e4ea0552ea424177849": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bfb7c0f0cb994ce5aa8774ff0307ff92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6be3c98ae1874a2ba753ffca259b8413": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3d125cb328a14addbb12a73581537033",
              "IPY_MODEL_cd6674aa6feb48659a9dfad2b42dc62b",
              "IPY_MODEL_5dfd1115a57643c49fdd954eaadf55b0"
            ],
            "layout": "IPY_MODEL_efc527a2be9f49f3acd5a0bccbe0e38c"
          }
        },
        "3d125cb328a14addbb12a73581537033": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e6d55f017834827bad2d32180d8a9ef",
            "placeholder": "​",
            "style": "IPY_MODEL_082d7c65a13042569be15d8a4b3a6130",
            "value": "beir/fiqa/dev documents: 100%"
          }
        },
        "cd6674aa6feb48659a9dfad2b42dc62b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_06b4b85af7604f0687b22b9f42bb2dee",
            "max": 57638,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1d6ef861c6b540eb943c529a3529a39e",
            "value": 57638
          }
        },
        "5dfd1115a57643c49fdd954eaadf55b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0240b73b60e24b97a067eab1a00c84aa",
            "placeholder": "​",
            "style": "IPY_MODEL_4c452bf63d424d94bc514042066d930b",
            "value": " 57638/57638 [00:18&lt;00:00, 3633.24it/s]"
          }
        },
        "efc527a2be9f49f3acd5a0bccbe0e38c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e6d55f017834827bad2d32180d8a9ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "082d7c65a13042569be15d8a4b3a6130": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "06b4b85af7604f0687b22b9f42bb2dee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d6ef861c6b540eb943c529a3529a39e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0240b73b60e24b97a067eab1a00c84aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c452bf63d424d94bc514042066d930b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SelmaDM/Pyterrier/blob/master/Pyterrier_Ranking.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Oe730o8rzyn"
      },
      "source": [
        "## PyTerrier - Learning to Rank\n",
        "\n",
        "Ce TP à comme objetif la construction des pipelines dans [PyTerrier] (https://github.com/terrier-org/pyterrier). Toutes les expériences sont menées en utilisant le [corpus CORD19](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7251955/) et la [collection de tests TREC Covid](https://ir.nist.gov/covidSubmit/). Finalement, vous serez amené à refaire le processus avec une collection de questions-réponses.\n",
        "\n",
        "Ce TP est divisé en deux. En particulier, dans la permière partie, vous allez expérimenter pour : \n",
        " - comprendre le modèle de données PyTerrier ;\n",
        " - comprendre la notion générale de PyTerrier et les opérateurs permettant de combiner plusieurs recherches ;\n",
        " - entreprendre des expériences sur la collection de tests TREC Covid.\n",
        "\n",
        "Dans la deuxième partie, vous ferez des expériences pour :\n",
        " - construire et apprendre des pipelines pour le Learning to Rank ;\n",
        " - évaluer et analyser des pipelines pour le Learning to Rank.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Première partie"
      ],
      "metadata": {
        "id": "FBDOJIBLVtXl"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0edvI3LDtOym"
      },
      "source": [
        "# Installation\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mkCdrWdRzS64"
      },
      "source": [
        "!pip install -q --upgrade fastrank lightgbm==3.1.1\n",
        "!pip install -q python-terrier"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SWc4UQgX6vDD"
      },
      "source": [
        "import pyterrier as pt\n",
        "if not pt.started():\n",
        "  pt.init()\n",
        "cord19 = pt.datasets.get_dataset('irds:cord19/trec-covid')\n"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p18d1tpJvD_5"
      },
      "source": [
        "# Indexation\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eGhNj2fqFqJX"
      },
      "source": [
        "!rm -rf ./terrier_cord19/*"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p5xG6YtoEieK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 737,
          "referenced_widgets": [
            "9a7cdd20ed4d420ba377791ef674928d",
            "10135c57e801432c9cf1c677b1716ce1",
            "7aa7d64ff43b4f54969016030f4fdfc7",
            "86844478b4934b028512aea6b79ac271",
            "b6268e071b9142fe8e56c22c8dadfa96",
            "bc8f864c25c642e693b7ca4d8fd7018f",
            "1434a36207f2437da64508a67cebc574",
            "a0287b66fceb469fb96ad23acbcc7398",
            "db6edaf76918403dbca749e19ece85d1",
            "ae9b899f58464f3f9e88f7aaed0617ef",
            "24612a254f1d40f18faf806860dfdacd"
          ]
        },
        "outputId": "d34158dd-b790-4d2a-dcb5-80d903decb5c"
      },
      "source": [
        "import os\n",
        "\n",
        "pt_index_path = './terrier_cord19'\n",
        "\n",
        "if not os.path.exists(pt_index_path + \"/data.properties\"):\n",
        "    # create the index, using the IterDictIndexer indexer \n",
        "    indexer = pt.index.IterDictIndexer(pt_index_path, blocks=True)\n",
        "\n",
        "    # we give the dataset get_corpus_iter() directly to the indexer\n",
        "    # while specifying the fields to index and the metadata to record\n",
        "    index_ref = indexer.index(cord19.get_corpus_iter(), \n",
        "                              fields=('abstract',), \n",
        "                              meta=('docno',))\n",
        "\n",
        "else:\n",
        "    # if you already have the index, use it.\n",
        "    index_ref = pt.IndexRef.of(pt_index_path + \"/data.properties\")"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "cord19/trec-covid documents:   0%|          | 0/192509 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9a7cdd20ed4d420ba377791ef674928d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:31:38.883 [ForkJoinPool-3-worker-3] WARN org.terrier.structures.indexing.Indexer - Adding an empty document to the index (6iu1dtyl) - further warnings are suppressed\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-52-59cedea60614>:11: DeprecationWarning: specifying meta and meta_lengths in IterDictIndexer.index() is deprecated, use constructor instead\n",
            "  index_ref = indexer.index(cord19.get_corpus_iter(),\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:33:17.619 [ForkJoinPool-3-worker-3] ERROR org.terrier.structures.indexing.Indexer - Could not finish MetaIndexBuilder: \n",
            "java.io.IOException: Key 8lqzfj2e is not unique: 37597,11755\n",
            "For MetaIndex, to suppress, set metaindex.compressed.reverse.allow.duplicates=true\n",
            "\tat org.terrier.structures.collections.FSOrderedMapFile$MultiFSOMapWriter.mergeTwo(FSOrderedMapFile.java:1374)\n",
            "\tat org.terrier.structures.collections.FSOrderedMapFile$MultiFSOMapWriter.close(FSOrderedMapFile.java:1308)\n",
            "\tat org.terrier.structures.indexing.BaseMetaIndexBuilder.close(BaseMetaIndexBuilder.java:321)\n",
            "\tat org.terrier.structures.indexing.classical.BasicIndexer.indexDocuments(BasicIndexer.java:270)\n",
            "\tat org.terrier.structures.indexing.classical.BasicIndexer.createDirectIndex(BasicIndexer.java:388)\n",
            "\tat org.terrier.structures.indexing.Indexer.index(Indexer.java:377)\n",
            "\tat org.terrier.python.ParallelIndexer$3.apply(ParallelIndexer.java:131)\n",
            "\tat org.terrier.python.ParallelIndexer$3.apply(ParallelIndexer.java:120)\n",
            "\tat java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)\n",
            "\tat java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948)\n",
            "\tat java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)\n",
            "\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)\n",
            "\tat java.base/java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:952)\n",
            "\tat java.base/java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:926)\n",
            "\tat java.base/java.util.stream.AbstractTask.compute(AbstractTask.java:327)\n",
            "\tat java.base/java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:746)\n",
            "\tat java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)\n",
            "\tat java.base/java.util.concurrent.ForkJoinTask.doInvoke(ForkJoinTask.java:408)\n",
            "\tat java.base/java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:736)\n",
            "\tat java.base/java.util.stream.ReduceOps$ReduceOp.evaluateParallel(ReduceOps.java:919)\n",
            "\tat java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233)\n",
            "\tat java.base/java.util.stream.ReferencePipeline.reduce(ReferencePipeline.java:558)\n",
            "\tat org.terrier.python.ParallelIndexer$4.call(ParallelIndexer.java:140)\n",
            "\tat org.terrier.python.ParallelIndexer$4.call(ParallelIndexer.java:137)\n",
            "\tat java.base/java.util.concurrent.ForkJoinTask$AdaptedCallable.exec(ForkJoinTask.java:1448)\n",
            "\tat java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)\n",
            "\tat java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)\n",
            "\tat java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)\n",
            "\tat java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)\n",
            "\tat java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)\n",
            "13:33:35.201 [ForkJoinPool-3-worker-3] WARN org.terrier.structures.indexing.Indexer - Indexed 54937 empty documents\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OiM8I_fkUcQ9"
      },
      "source": [
        "## Operateurs\n",
        "\n",
        "Vous aurez remarqué que BatchRetrieve possède une méthode `transform()` qui prend en entrée un dataframe, et retourne un autre dataframe, qui est en quelque sorte une *transformation* du dataframe précédent (par exemple, un résultat de la recherche). En fait, `BatchRetrieve` n'est qu'un des nombreux objets similaires dans PyTerrier, que nous appelons [transformers](https://pyterrier.readthedocs.io/en/latest/transformer.html) (représentés par la classe `TransformerBase`).\n",
        "Par exemple, on peut sauvegarder explicitement dans la varible ```tfidf``` l'application du modèle tfidf\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UqMt-9UlTqLa"
      },
      "source": [
        "tfidf = pt.BatchRetrieve(index_ref, wmodel=\"TF_IDF\")"
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GW3afDgPukvU"
      },
      "source": [
        "La capacité intéressante de tous les transformateurs est qu'ils peuvent être combinés à l'aide d'opérateurs Python (c'est ce qu'on appelle la *surcharge d'opérateurs*).\n",
        "\n",
        "Concrètement, imaginez que vous voulez enchaîner des transformateurs ensemble - par exemple, trier des documents d'abord par tf puis retrier les documents par tfidf. Nous pouvons le faire en utilisant l'opérateur `>>` - nous appelons cela *composition*."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r1yCXIB5w1Qb"
      },
      "source": [
        "Il existe un certain nombre d'opérateurs PyTerrier - il y a plus d'exemples dans la [documentation PyTerrier sur les opérateurs](https://pyterrier.readthedocs.io/en/latest/operators.html)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NaYavGvnxDk5"
      },
      "source": [
        "## Q1 - Construction de pipelines\n",
        "\n",
        "Créez un *ranker* (une classe que permet de chercher sur l'indèxe) qui effectue les opérations suivantes :\n",
        " - obtenir les 10 documents les mieux notés par tf (`wmodel=\"Tf\"`)\n",
        " - obtenir les 10 documents les mieux notés par tfidf (`wmodel=\"TF_IDF\"`)\n",
        " - ne remet en ordre que les documents trouvés dans l'un ou l'autre des paramètres de recherche précédents en utilisant BM25.\n",
        "\n",
        "en utilisant des opérateurs PyTerrier combinant différentes instances de BatchRetrieve.\n",
        "\n",
        "## Q2 - \\# documents\n",
        "Combien de documents sont récupérés par ce pipeline complet pour la requête `\"chemical\"` ?\n",
        "> Hint: Si vous obtenez la bonne solution, le document avec le docno `\"8hykq71k\"` devrait avoir un score proche de $12.646089$ pour la requête `\"chemical\"`.\n",
        "\n",
        "Conseils :\n",
        " - choisissez soigneusement vos [opérateurs PyTerrier](https://pyterrier.readthedocs.io/en/latest/operators.html)\n",
        " - vous ne devriez pas avoir à effectuer d'opérations sur les dataframes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XnsRdT4WvspD"
      },
      "source": [
        "# réponse Q1\n",
        "bm25 = pt.BatchRetrieve(index_ref, wmodel=\"BM25\")%10\n",
        "tf = pt.BatchRetrieve(index_ref, wmodel=\"Tf\")%10\n",
        "tf_idf = pt.BatchRetrieve(index_ref, wmodel=\"TF_IDF\")%10\n",
        "pipeline = bm25 >> (tf ** tf_idf)"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# réponse Q2\n",
        " #Nombre de documents recupérés : 1006 "
      ],
      "metadata": {
        "id": "oBHEKNH4HGp9"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bm25 = pt.BatchRetrieve(index_ref, wmodel=\"BM25\")\n",
        "tf = pt.BatchRetrieve(index_ref, wmodel=\"Tf\")\n",
        "tf_idf = pt.BatchRetrieve(index_ref, wmodel=\"TF_IDF\")\n",
        "pipeline = bm25 >> (tf ** tf_idf)\n",
        "\n",
        "pipeline.search(\"chemical\")"
      ],
      "metadata": {
        "id": "6LOOCM8gIrCt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "outputId": "c4b5b48b-5303-4feb-aa69-6793aa98c635"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     qid   docid     docno  rank      score     query  \\\n",
              "0      1   37771  jn5qi1jb     0  12.426309  chemical   \n",
              "1      1   15671  8hykq71k     1  12.413269  chemical   \n",
              "2      1  134305  0smev8vt     2  12.292890  chemical   \n",
              "3      1  142104  77c9ohxj     3  12.226076  chemical   \n",
              "4      1   87642  ck6clsty     4  12.155804  chemical   \n",
              "...   ..     ...       ...   ...        ...       ...   \n",
              "1001   1  192405  rh9h2rtz   995   6.065540  chemical   \n",
              "1002   1    4713  gve33v4t   996   6.040683  chemical   \n",
              "1003   1    9767  pcu5sy6t   997   6.040683  chemical   \n",
              "1004   1   35909  xic0qqio   998   6.040683  chemical   \n",
              "1005   1   45510  i5z4yq4p   999   6.040683  chemical   \n",
              "\n",
              "                       features  \n",
              "0       [9.0, 6.80992310099522]  \n",
              "1      [4.0, 6.802776877224232]  \n",
              "2      [8.0, 6.736806385246307]  \n",
              "3      [7.0, 6.700190253577284]  \n",
              "4      [5.0, 6.661679714989656]  \n",
              "...                         ...  \n",
              "1001   [1.0, 3.324065127180903]  \n",
              "1002  [1.0, 3.3104430955870283]  \n",
              "1003  [1.0, 3.3104430955870283]  \n",
              "1004  [1.0, 3.3104430955870283]  \n",
              "1005  [1.0, 3.3104430955870283]  \n",
              "\n",
              "[1006 rows x 7 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-649ac7c4-ecef-4743-8241-2515ea2f4fd5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>docid</th>\n",
              "      <th>docno</th>\n",
              "      <th>rank</th>\n",
              "      <th>score</th>\n",
              "      <th>query</th>\n",
              "      <th>features</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>37771</td>\n",
              "      <td>jn5qi1jb</td>\n",
              "      <td>0</td>\n",
              "      <td>12.426309</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[9.0, 6.80992310099522]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>15671</td>\n",
              "      <td>8hykq71k</td>\n",
              "      <td>1</td>\n",
              "      <td>12.413269</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[4.0, 6.802776877224232]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>134305</td>\n",
              "      <td>0smev8vt</td>\n",
              "      <td>2</td>\n",
              "      <td>12.292890</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[8.0, 6.736806385246307]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>142104</td>\n",
              "      <td>77c9ohxj</td>\n",
              "      <td>3</td>\n",
              "      <td>12.226076</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[7.0, 6.700190253577284]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>87642</td>\n",
              "      <td>ck6clsty</td>\n",
              "      <td>4</td>\n",
              "      <td>12.155804</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[5.0, 6.661679714989656]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1001</th>\n",
              "      <td>1</td>\n",
              "      <td>192405</td>\n",
              "      <td>rh9h2rtz</td>\n",
              "      <td>995</td>\n",
              "      <td>6.065540</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[1.0, 3.324065127180903]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1002</th>\n",
              "      <td>1</td>\n",
              "      <td>4713</td>\n",
              "      <td>gve33v4t</td>\n",
              "      <td>996</td>\n",
              "      <td>6.040683</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[1.0, 3.3104430955870283]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1003</th>\n",
              "      <td>1</td>\n",
              "      <td>9767</td>\n",
              "      <td>pcu5sy6t</td>\n",
              "      <td>997</td>\n",
              "      <td>6.040683</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[1.0, 3.3104430955870283]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1004</th>\n",
              "      <td>1</td>\n",
              "      <td>35909</td>\n",
              "      <td>xic0qqio</td>\n",
              "      <td>998</td>\n",
              "      <td>6.040683</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[1.0, 3.3104430955870283]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1005</th>\n",
              "      <td>1</td>\n",
              "      <td>45510</td>\n",
              "      <td>i5z4yq4p</td>\n",
              "      <td>999</td>\n",
              "      <td>6.040683</td>\n",
              "      <td>chemical</td>\n",
              "      <td>[1.0, 3.3104430955870283]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1006 rows × 7 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-649ac7c4-ecef-4743-8241-2515ea2f4fd5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-649ac7c4-ecef-4743-8241-2515ea2f4fd5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-649ac7c4-ecef-4743-8241-2515ea2f4fd5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Évaluation dans un pipeline\n",
        "À difference du TP3 où nous avons fait nous même l'évaluation, dans ce TP nous allons utiliser le module d'évaluation intégré pour les pipelines en pyterrier. Donc, pour faire des expériences (évaluer un modèle) sur pyterrier, on peut utiliser la classe `Experiment`. Voici le code qui évalue la performance de tfidf pour la collection cord19.  "
      ],
      "metadata": {
        "id": "8P380IAZIrds"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TF24TZSb3bOo"
      },
      "source": [
        "# Télécharger la collection\n",
        "\n",
        "topics = cord19.get_topics(variant='description')\n",
        "qrels = cord19.get_qrels()\n"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(topics), len(qrels)"
      ],
      "metadata": {
        "id": "VwKs8dYP67KD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bce4ecc8-d498-4bc5-954a-17839b642fff"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50, 69318)"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Voici le code pour évaluer avec Experiment\n",
        "from pyterrier.measures import *\n",
        "pt.Experiment(\n",
        "  #le pipeline\n",
        "  [tfidf],\n",
        "  topics,\n",
        "  qrels,\n",
        "  eval_metrics=[MAP, nDCG, nDCG@10],\n",
        "  # on utilise TFIDF pour les tests statistiques\n",
        "  baseline=0,\n",
        "  names=[\"TFIDF\"]\n",
        ")\n",
        "\n"
      ],
      "metadata": {
        "id": "jHb9fEO2JXR9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "outputId": "66c4e39e-52ab-4048-ba7c-d4f774e35d75"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    name        AP      nDCG   nDCG@10  AP +  AP - AP p-value nDCG + nDCG -  \\\n",
              "0  TFIDF  0.188578  0.400915  0.638799  None  None       None   None   None   \n",
              "\n",
              "  nDCG p-value nDCG@10 + nDCG@10 - nDCG@10 p-value  \n",
              "0         None      None      None            None  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-91845cd9-497f-4b54-9209-8b2f6d8f6a00\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>AP</th>\n",
              "      <th>nDCG</th>\n",
              "      <th>nDCG@10</th>\n",
              "      <th>AP +</th>\n",
              "      <th>AP -</th>\n",
              "      <th>AP p-value</th>\n",
              "      <th>nDCG +</th>\n",
              "      <th>nDCG -</th>\n",
              "      <th>nDCG p-value</th>\n",
              "      <th>nDCG@10 +</th>\n",
              "      <th>nDCG@10 -</th>\n",
              "      <th>nDCG@10 p-value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>TFIDF</td>\n",
              "      <td>0.188578</td>\n",
              "      <td>0.400915</td>\n",
              "      <td>0.638799</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-91845cd9-497f-4b54-9209-8b2f6d8f6a00')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-91845cd9-497f-4b54-9209-8b2f6d8f6a00 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-91845cd9-497f-4b54-9209-8b2f6d8f6a00');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0aJrYP1TSLwY"
      },
      "source": [
        "# Deuxième Partie  – Learning to Rank\n",
        "\n",
        "Dans cette partie du TP, vous ferez la construction, l'apprentissage,  l'évaluation et l'analyse des pipelines d'apprentissage du classement (Learning to Rank).\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CSi3svV24mCs"
      },
      "source": [
        "Tout d'abord, répartissons les annotations (rankings) en ensembles d'entraînement, de validation et de test. TREC Covid ne comporte que 50 annotations, ce qui n'est pas beaucoup pour l'appprentissage. Nous allons répartir 30 annotations pour l'entraînement, 5 pour la validation et 15 pour l'évaluation. Nous examinerons également les differences statistiques, même si elle est petite, pour 15 annotations.\n",
        "\n",
        "Nous n'allons classer que les 10 premiers documents pour chaque requête - nous espérons que l'apprentissage du classement pourra nous aider à retrier les 10 premiers documents pour qu'ils soient plus efficaces.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GstCtHFBJLtz"
      },
      "source": [
        "RANK_CUTOFF = 10\n",
        "SEED=42\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "tr_va_topics, test_topics = train_test_split(topics, test_size=15, random_state=SEED)\n",
        "train_topics, valid_topics =  train_test_split(tr_va_topics, test_size=5, random_state=SEED)\n",
        "\n",
        "\n",
        "test_qrels = qrels # seulement les annotations des topics en réponse sont utilisés, donc pas de problème si on utilise tout\n",
        "train_qrels = qrels\n",
        "valid_qrels = qrels"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vftc_y-65VfA"
      },
      "source": [
        "## Ensemble de caractéristiques\n",
        "\n",
        "Définissons notre ensemble de caractéristiques.  Nous allons avoir un total de 6 caractéristiques :\n",
        "\n",
        "1. le score abstrait de TFIDF ;\n",
        "2. le résumé contient-il \"coronavirus covid\", noté par TFIDF ;\n",
        "3. le score TFIDF sur le titre (même si nous ne l'avons pas indexé plus tôt !);\n",
        "4. l'article a-t-il été publié en 2020 ? Ici, on a l'hypothèse que les articles récents étaient plus utiles pour cette tâche ;\n",
        "5. L'article a-t-il un DOI, c'est-à-dire s'agit-il d'une publication officielle ?\n",
        "6. le score de correspondance des coordonnées pour la requête - c'est-à-dire combien de termes de la requête apparaissent dans le résumé.\n",
        "\n",
        "Plusieurs de ces caractéristiques nécessitent des métadonnées supplémentaires `[\"title\", \"date\", \"doi\"]`. Heureusement, le jeu de données TREC Covid nous permet d'obtenir plus de métadonnées après l'indexation. Nous utilisons `pt.text.get_text(cord19, [\"title\", \"date\", \"doi\"])` pour récupérer ces colonnes de métadonnées supplémentaires.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AYgwwrsPTGP1"
      },
      "source": [
        "ltr_feats1 = (tfidf % RANK_CUTOFF) >> pt.text.get_text(cord19, [\"title\", \"date\", \"doi\"]) >> (\n",
        "    pt.transformer.IdentityTransformer()\n",
        "    ** # score of text for query 'coronavirus covid'\n",
        "    (pt.apply.query(lambda row: 'coronavirus covid') >> tfidf)\n",
        "    ** # score of title (not originally indexed)\n",
        "    (pt.text.scorer(body_attr=\"title\", takes='docs', wmodel='TF_IDF') ) \n",
        "    ** # date 2020\n",
        "    (pt.apply.doc_score(lambda row: int(\"2020\" in row[\"date\"])))\n",
        "    ** # has doi\n",
        "    (pt.apply.doc_score(lambda row: int( row[\"doi\"] is not None and len(row[\"doi\"]) > 0) ))\n",
        "    ** # abstract coordinate match\n",
        "    pt.BatchRetrieve(index_ref, wmodel=\"CoordinateMatch\")\n",
        ")\n",
        "\n",
        "# for reference, lets record the feature names here too\n",
        "fnames=[\"TFIDF\", 'coronavirus covid', 'title', \"2020\", \"hasDoi\", \"CoordinateMatch\"]"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nu3M9ujw6trC"
      },
      "source": [
        "Voyons le résultat pour une requête particulière. Nous pouvons voir que nous avons maintenant des colonnes supplémentaires de métadonnées de document `[\"title\", \"date\", \"doi\"]`, ainsi que les colonnes `\"features\"` très importantes. Cela fait que le dataframe a le type $R_f$. En effet, c'est cette colonne que nous utilisons pour l'apprentissage."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OMG9kK9AUWI4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 618
        },
        "outputId": "1015eabd-b859-4eff-dbe6-6724783e6c15"
      },
      "source": [
        "ltr_feats1.search(\"Movie\")"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:33:41.592 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  qid   docid     docno  rank      score  query  \\\n",
              "0   1   23347  qiwq0pe5     0  12.970648  Movie   \n",
              "1   1   23343  vmetwotq     1  12.923459  Movie   \n",
              "2   1   78848  mmq44kwx     2  12.207116  Movie   \n",
              "3   1   70132  eynhsuz8     3  11.372312  Movie   \n",
              "4   1   24731  o7ckdng4     4  10.805086  Movie   \n",
              "5   1  118935  25khbzk0     5   9.594602  Movie   \n",
              "6   1   16001  3lhpdpiv     6   9.058977  Movie   \n",
              "7   1   18722  gt3xayqp     7   8.579994  Movie   \n",
              "8   1   86076  opbwnnai     8   8.242861  Movie   \n",
              "9   1    5238  5z3pbbfb     9   8.042828  Movie   \n",
              "\n",
              "                                               title        date  \\\n",
              "0  Sentiment Analysis on Movie Scripts and Review...  2020-05-06   \n",
              "1  Improving Movie Recommendation Systems Filteri...  2020-05-04   \n",
              "2  Smoking in top-grossing movies--United States,...        2011   \n",
              "3    The Post: A token woman leader's transformation        2020   \n",
              "4   Movies Emotional Analysis Using Textual Contents  2020-05-26   \n",
              "5  CinemaGazer: a System for Watching Video at Ve...  2011-10-04   \n",
              "6  The Aliens in Us and the Aliens Out There: Sci...  2013-11-17   \n",
              "7  The Unfairness of Popularity Bias in Music Rec...  2020-03-24   \n",
              "8  Preparing for an influenza pandemic: mental he...        2009   \n",
              "9  Characteristics of airborne Staphylococcus aur...  2014-05-07   \n",
              "\n",
              "                            doi  \\\n",
              "0  10.1007/978-3-030-49161-1_36   \n",
              "1  10.1007/978-3-030-49190-1_17   \n",
              "2                                 \n",
              "3            10.1002/hrdq.21391   \n",
              "4  10.1007/978-3-030-51310-8_19   \n",
              "5                                 \n",
              "6   10.1007/978-1-4614-7175-2_2   \n",
              "7   10.1007/978-3-030-45442-5_5   \n",
              "8                                 \n",
              "9     10.1007/s10453-014-9342-6   \n",
              "\n",
              "                                            features  \n",
              "0  [12.97064764826613, 0.0, 0.7458647062217206, 1...  \n",
              "1  [12.923458775835545, 0.0, 1.0350775514913675, ...  \n",
              "2  [12.207115891520253, 0.0, 0.8744620693633965, ...  \n",
              "3  [11.372311997669033, 1.499168191418449, 0.0, 1...  \n",
              "4  [10.80508574921903, 0.0, 0.9277829272514087, 1...  \n",
              "5       [9.594602481582845, 0.0, 0.0, 0.0, 0.0, 1.0]  \n",
              "6  [9.058977465455175, 0.0, 0.9880285718781234, 0...  \n",
              "7       [8.579993656638317, 0.0, 0.0, 1.0, 1.0, 1.0]  \n",
              "8       [8.242860905714242, 0.0, 0.0, 0.0, 0.0, 1.0]  \n",
              "9       [8.042828188746888, 0.0, 0.0, 0.0, 1.0, 1.0]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7ba6454b-cdf6-4928-8e0d-ade2c02be018\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>docid</th>\n",
              "      <th>docno</th>\n",
              "      <th>rank</th>\n",
              "      <th>score</th>\n",
              "      <th>query</th>\n",
              "      <th>title</th>\n",
              "      <th>date</th>\n",
              "      <th>doi</th>\n",
              "      <th>features</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>23347</td>\n",
              "      <td>qiwq0pe5</td>\n",
              "      <td>0</td>\n",
              "      <td>12.970648</td>\n",
              "      <td>Movie</td>\n",
              "      <td>Sentiment Analysis on Movie Scripts and Review...</td>\n",
              "      <td>2020-05-06</td>\n",
              "      <td>10.1007/978-3-030-49161-1_36</td>\n",
              "      <td>[12.97064764826613, 0.0, 0.7458647062217206, 1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>23343</td>\n",
              "      <td>vmetwotq</td>\n",
              "      <td>1</td>\n",
              "      <td>12.923459</td>\n",
              "      <td>Movie</td>\n",
              "      <td>Improving Movie Recommendation Systems Filteri...</td>\n",
              "      <td>2020-05-04</td>\n",
              "      <td>10.1007/978-3-030-49190-1_17</td>\n",
              "      <td>[12.923458775835545, 0.0, 1.0350775514913675, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>78848</td>\n",
              "      <td>mmq44kwx</td>\n",
              "      <td>2</td>\n",
              "      <td>12.207116</td>\n",
              "      <td>Movie</td>\n",
              "      <td>Smoking in top-grossing movies--United States,...</td>\n",
              "      <td>2011</td>\n",
              "      <td></td>\n",
              "      <td>[12.207115891520253, 0.0, 0.8744620693633965, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>70132</td>\n",
              "      <td>eynhsuz8</td>\n",
              "      <td>3</td>\n",
              "      <td>11.372312</td>\n",
              "      <td>Movie</td>\n",
              "      <td>The Post: A token woman leader's transformation</td>\n",
              "      <td>2020</td>\n",
              "      <td>10.1002/hrdq.21391</td>\n",
              "      <td>[11.372311997669033, 1.499168191418449, 0.0, 1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>24731</td>\n",
              "      <td>o7ckdng4</td>\n",
              "      <td>4</td>\n",
              "      <td>10.805086</td>\n",
              "      <td>Movie</td>\n",
              "      <td>Movies Emotional Analysis Using Textual Contents</td>\n",
              "      <td>2020-05-26</td>\n",
              "      <td>10.1007/978-3-030-51310-8_19</td>\n",
              "      <td>[10.80508574921903, 0.0, 0.9277829272514087, 1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1</td>\n",
              "      <td>118935</td>\n",
              "      <td>25khbzk0</td>\n",
              "      <td>5</td>\n",
              "      <td>9.594602</td>\n",
              "      <td>Movie</td>\n",
              "      <td>CinemaGazer: a System for Watching Video at Ve...</td>\n",
              "      <td>2011-10-04</td>\n",
              "      <td></td>\n",
              "      <td>[9.594602481582845, 0.0, 0.0, 0.0, 0.0, 1.0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1</td>\n",
              "      <td>16001</td>\n",
              "      <td>3lhpdpiv</td>\n",
              "      <td>6</td>\n",
              "      <td>9.058977</td>\n",
              "      <td>Movie</td>\n",
              "      <td>The Aliens in Us and the Aliens Out There: Sci...</td>\n",
              "      <td>2013-11-17</td>\n",
              "      <td>10.1007/978-1-4614-7175-2_2</td>\n",
              "      <td>[9.058977465455175, 0.0, 0.9880285718781234, 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1</td>\n",
              "      <td>18722</td>\n",
              "      <td>gt3xayqp</td>\n",
              "      <td>7</td>\n",
              "      <td>8.579994</td>\n",
              "      <td>Movie</td>\n",
              "      <td>The Unfairness of Popularity Bias in Music Rec...</td>\n",
              "      <td>2020-03-24</td>\n",
              "      <td>10.1007/978-3-030-45442-5_5</td>\n",
              "      <td>[8.579993656638317, 0.0, 0.0, 1.0, 1.0, 1.0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1</td>\n",
              "      <td>86076</td>\n",
              "      <td>opbwnnai</td>\n",
              "      <td>8</td>\n",
              "      <td>8.242861</td>\n",
              "      <td>Movie</td>\n",
              "      <td>Preparing for an influenza pandemic: mental he...</td>\n",
              "      <td>2009</td>\n",
              "      <td></td>\n",
              "      <td>[8.242860905714242, 0.0, 0.0, 0.0, 0.0, 1.0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1</td>\n",
              "      <td>5238</td>\n",
              "      <td>5z3pbbfb</td>\n",
              "      <td>9</td>\n",
              "      <td>8.042828</td>\n",
              "      <td>Movie</td>\n",
              "      <td>Characteristics of airborne Staphylococcus aur...</td>\n",
              "      <td>2014-05-07</td>\n",
              "      <td>10.1007/s10453-014-9342-6</td>\n",
              "      <td>[8.042828188746888, 0.0, 0.0, 0.0, 1.0, 1.0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7ba6454b-cdf6-4928-8e0d-ade2c02be018')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7ba6454b-cdf6-4928-8e0d-ade2c02be018 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7ba6454b-cdf6-4928-8e0d-ade2c02be018');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3vQNbSSgJCmC"
      },
      "source": [
        "Nous pouvons également examiner les valeurs brutes des caractéristiques (dans ce cas, pour le premier document classé). Notez que le BM25 dans la colonne \"score\" ci-dessus est également la première valeur dans le tableau des caractéristiques (proche de 12), parce que nous avons utilisé un transformateur d'identité."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EzVDLL8pI_gR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a4d4cd2d-b2b2-4244-bb72-3572a7593f60"
      },
      "source": [
        "ltr_feats1.search(\"Movie\").iloc[0][\"features\"]"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:33:41.995 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([12.97064765,  0.        ,  0.74586471,  1.        ,  1.        ,\n",
              "        1.        ])"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4ov0JjWGEbrT"
      },
      "source": [
        "## Analysis\n",
        "\n",
        "Ici nous analisons les performances de chaque caractéristique indépendamment. Pour cela, nous composons le pipeline de caractéristiques (`ltr_feats1`) avec `pt.ltr.feature_to_score(i)` pour un certain nombre de caractéristiques $i$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7nCb9U7uvszV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 582
        },
        "outputId": "b2c2f79f-9c40-4bf6-abc4-a790093ae49f"
      },
      "source": [
        "pt.Experiment(\n",
        "    [ltr_feats1 >> pt.ltr.feature_to_score(i) for i in range(len(fnames))],\n",
        "    test_topics,\n",
        "    test_qrels,  \n",
        "    names=fnames,\n",
        "    eval_metrics=[\"map\", \"ndcg\", \"ndcg_cut_10\", \"num_rel_ret\"])"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:33:45.340 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:33:51.151 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:33:53.677 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:33:56.113 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:33:58.541 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:02.439 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                name       map      ndcg  ndcg_cut_10  num_rel_ret\n",
              "0              TFIDF  0.010519  0.047832     0.589368         96.0\n",
              "1  coronavirus covid  0.010816  0.047858     0.584780         96.0\n",
              "2              title  0.012200  0.054099     0.647125         96.0\n",
              "3               2020  0.010942  0.048751     0.591473         96.0\n",
              "4             hasDoi  0.010437  0.047810     0.575902         96.0\n",
              "5    CoordinateMatch  0.010204  0.046617     0.570048         96.0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-88459772-a4c5-4007-aa7b-ed4da1633754\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>map</th>\n",
              "      <th>ndcg</th>\n",
              "      <th>ndcg_cut_10</th>\n",
              "      <th>num_rel_ret</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>TFIDF</td>\n",
              "      <td>0.010519</td>\n",
              "      <td>0.047832</td>\n",
              "      <td>0.589368</td>\n",
              "      <td>96.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>coronavirus covid</td>\n",
              "      <td>0.010816</td>\n",
              "      <td>0.047858</td>\n",
              "      <td>0.584780</td>\n",
              "      <td>96.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>title</td>\n",
              "      <td>0.012200</td>\n",
              "      <td>0.054099</td>\n",
              "      <td>0.647125</td>\n",
              "      <td>96.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020</td>\n",
              "      <td>0.010942</td>\n",
              "      <td>0.048751</td>\n",
              "      <td>0.591473</td>\n",
              "      <td>96.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>hasDoi</td>\n",
              "      <td>0.010437</td>\n",
              "      <td>0.047810</td>\n",
              "      <td>0.575902</td>\n",
              "      <td>96.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>CoordinateMatch</td>\n",
              "      <td>0.010204</td>\n",
              "      <td>0.046617</td>\n",
              "      <td>0.570048</td>\n",
              "      <td>96.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-88459772-a4c5-4007-aa7b-ed4da1633754')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-88459772-a4c5-4007-aa7b-ed4da1633754 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-88459772-a4c5-4007-aa7b-ed4da1633754');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rU7YYKFSFeke"
      },
      "source": [
        "Il est intéressant de noter que la caractéristique \"coronavirus covid\" a obtenu un NDCG@10 de 0,5847. Il s'agit donc d'une baseline forte pour cette tâche.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Od8P-N4E7QTk"
      },
      "source": [
        "## Learning \n",
        "\n",
        "Dans cette partie du TP, nous appliquons trois techniques différentes d'apprentissage du classement :\n",
        "\n",
        " - l'ascension des coordonnées à partir de FastRank, une technique linéaire par liste\n",
        " - les forêts aléatoires de `scikit-learn`, une technique d'arbre de régression par liste\n",
        " - LambdaMART de LightGBM, une technique d'arbre de régression par liste.\n",
        "\n",
        "Dans chaque cas, nous prenons notre pipeline de caractéristiques, `ltr_feats1`, et nous le composons (`>>`) avec le modèle appris. Nous utilisons `pt.ltr.apply_learned_model()` qui sait comment gérer les différents apprenants.\n",
        "\n",
        "Le pipeline complet est ensuite ajusté (appris) en utilisant `.fit()`, en spécifiant les annotations d'entraînement et les qrels. Il est important de noter que les étapes précédentes du pipeline (récupération et calcul des caractéristiques) sont appliquées aux annotations d'entraînement afin d'obtenir les résultats, qui sont ensuite transmis à la technique d'apprentissage par classement. LightGBM a un arrêt précoce activé, qui utilise un ensemble de annotations de validation - de la même manière, les annotations de validation sont transformés en résultats de validation.\n",
        "\n",
        "Enfin, `%time` est la \"commande magique\" qui affiche le temps d'apprentissage pour chaque technique. L'apprentissage pour chaque technique prend < 30 secondes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OFPZ39eSYIXV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4eba6f77-cc3a-4b7b-d6e9-65ebd2dcda1e"
      },
      "source": [
        "import fastrank\n",
        "\n",
        "train_request = fastrank.TrainRequest.coordinate_ascent()\n",
        "\n",
        "params = train_request.params\n",
        "params.init_random = True\n",
        "params.normalize = True\n",
        "params.seed = 1234567\n",
        "\n",
        "ca_pipe = ltr_feats1 >> pt.ltr.apply_learned_model(train_request, form='fastrank')\n",
        "\n",
        "%time ca_pipe.fit(train_topics, train_qrels)"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:06.100 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 300 received 306, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 30.9 s, sys: 110 ms, total: 31 s\n",
            "Wall time: 19.7 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T0SpZ13wUagq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "27118176-d84f-4386-f4d8-874ca1e8d71c"
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "\n",
        "rf = RandomForestRegressor(n_estimators=400, verbose=1, random_state=42, n_jobs=2)\n",
        "\n",
        "rf_pipe = ltr_feats1 >> pt.ltr.apply_learned_model(rf)\n",
        "\n",
        "\n",
        "%time rf_pipe.fit(train_topics, train_qrels)"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:26.277 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 300 received 306, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n",
            "[Parallel(n_jobs=2)]: Using backend ThreadingBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:    0.3s\n",
            "[Parallel(n_jobs=2)]: Done 196 tasks      | elapsed:    1.2s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 9.06 s, sys: 126 ms, total: 9.19 s\n",
            "Wall time: 7.42 s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=2)]: Done 400 out of 400 | elapsed:    2.3s finished\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u0jb1jw_VdWU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d43e1120-6909-4859-de15-3870bbb8644c"
      },
      "source": [
        "import lightgbm as lgb\n",
        "\n",
        "# this configures LightGBM as LambdaMART\n",
        "lmart_l = lgb.LGBMRanker(\n",
        "    task=\"train\",\n",
        "    silent=False,\n",
        "    min_data_in_leaf=1,\n",
        "    min_sum_hessian_in_leaf=1,\n",
        "    max_bin=255,\n",
        "    num_leaves=31,\n",
        "    objective=\"lambdarank\",\n",
        "    metric=\"ndcg\",\n",
        "    ndcg_eval_at=[10],\n",
        "    ndcg_at=[10],\n",
        "    eval_at=[10],\n",
        "    learning_rate= .1,\n",
        "    importance_type=\"gain\",\n",
        "    num_iterations=100,\n",
        "    early_stopping_rounds=5\n",
        ")\n",
        "\n",
        "lmart_x_pipe = ltr_feats1 >> pt.ltr.apply_learned_model(lmart_l, form=\"ltr\", fit_kwargs={'eval_at':[10]})\n",
        "\n",
        "%time lmart_x_pipe.fit(train_topics, train_qrels, valid_topics, valid_qrels)"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:33.217 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 300 received 306, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:35.053 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 50 received 54, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[LightGBM] [Warning] early_stopping_round is set=5, early_stopping_rounds=5 will be ignored. Current value: early_stopping_round=5\n",
            "[LightGBM] [Warning] min_sum_hessian_in_leaf is set=1, min_child_weight=0.001 will be ignored. Current value: min_sum_hessian_in_leaf=1\n",
            "[LightGBM] [Warning] min_data_in_leaf is set=1, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=1\n",
            "[LightGBM] [Warning] min_sum_hessian_in_leaf is set=1, min_child_weight=0.001 will be ignored. Current value: min_sum_hessian_in_leaf=1\n",
            "[LightGBM] [Warning] min_data_in_leaf is set=1, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=1\n",
            "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.000929 seconds.\n",
            "You can set `force_row_wise=true` to remove the overhead.\n",
            "And if memory is not enough, you can set `force_col_wise=true`.\n",
            "[LightGBM] [Info] Total Bins 268\n",
            "[LightGBM] [Info] Number of data points in the train set: 24870, number of used features: 6\n",
            "[LightGBM] [Warning] early_stopping_round is set=5, early_stopping_rounds=5 will be ignored. Current value: early_stopping_round=5\n",
            "[LightGBM] [Warning] min_sum_hessian_in_leaf is set=1, min_child_weight=0.001 will be ignored. Current value: min_sum_hessian_in_leaf=1\n",
            "[LightGBM] [Warning] min_data_in_leaf is set=1, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=1\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[1]\tvalid_0's ndcg@10: 0.890216\n",
            "Training until validation scores don't improve for 5 rounds\n",
            "[2]\tvalid_0's ndcg@10: 0.912274\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[3]\tvalid_0's ndcg@10: 0.901911\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[4]\tvalid_0's ndcg@10: 0.899687\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[5]\tvalid_0's ndcg@10: 0.902509\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[6]\tvalid_0's ndcg@10: 0.906468\n",
            "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
            "[7]\tvalid_0's ndcg@10: 0.908917\n",
            "Early stopping, best iteration is:\n",
            "[2]\tvalid_0's ndcg@10: 0.912274\n",
            "CPU times: user 5.32 s, sys: 133 ms, total: 5.45 s\n",
            "Wall time: 5.24 s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/lightgbm/engine.py:151: UserWarning: Found `num_iterations` in params. Will use it instead of argument\n",
            "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
            "/usr/local/lib/python3.9/dist-packages/lightgbm/engine.py:156: UserWarning: Found `early_stopping_rounds` in params. Will use it instead of argument\n",
            "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b7rNqB7sBKFB"
      },
      "source": [
        "## Evaluation\n",
        "\n",
        "Comparons maintenant nos pipelines de classement sur nos 15 annotations de teste versus la référence de base BM25. Dans tous les cas, nous ne classons que 10 résultats par requête, donc le MAP sera carrément plus faible. \n",
        "\n",
        "Nous allons rapporter le temps de réponse moyen (`\"mrt\"`) ainsi que les mesures MAP, NDCG et NDCG@10."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P0EhTwejVqev",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 508
        },
        "outputId": "0ca0af6e-e83d-4397-f429-856e56616fa7"
      },
      "source": [
        "pt.Experiment(\n",
        "    [tfidf % RANK_CUTOFF, ca_pipe, rf_pipe, lmart_x_pipe],\n",
        "    test_topics,\n",
        "    test_qrels, \n",
        "    names=[\"TFIDF\",  \"TFIDF + CA(6f)\", \"TFIDF + RF(6f)\", \"TFIDF + LMart(6f)\"],\n",
        "    baseline=0,\n",
        "    eval_metrics=[\"map\", \"ndcg\", \"ndcg_cut_10\", \"mrt\"])"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:38.684 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:42.011 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n",
            "[Parallel(n_jobs=2)]: Using backend ThreadingBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:    0.0s\n",
            "[Parallel(n_jobs=2)]: Done 196 tasks      | elapsed:    0.1s\n",
            "[Parallel(n_jobs=2)]: Done 400 out of 400 | elapsed:    0.2s finished\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:44.640 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                name       map      ndcg  ndcg_cut_10         mrt  map +  \\\n",
              "0              TFIDF  0.010519  0.047832     0.589368   72.828908    NaN   \n",
              "1     TFIDF + CA(6f)  0.011807  0.052092     0.630170  191.067751    9.0   \n",
              "2     TFIDF + RF(6f)  0.011907  0.051449     0.627952  203.712403    8.0   \n",
              "3  TFIDF + LMart(6f)  0.011828  0.052731     0.622488  151.677864    6.0   \n",
              "\n",
              "   map -  map p-value  ndcg +  ndcg -  ndcg p-value  ndcg_cut_10 +  \\\n",
              "0    NaN          NaN     NaN     NaN           NaN            NaN   \n",
              "1    1.0     0.052802     9.0     1.0      0.010754            9.0   \n",
              "2    4.0     0.038993     9.0     3.0      0.090934            9.0   \n",
              "3    5.0     0.217733     5.0     6.0      0.161192            5.0   \n",
              "\n",
              "   ndcg_cut_10 -  ndcg_cut_10 p-value  \n",
              "0            NaN                  NaN  \n",
              "1            1.0             0.006948  \n",
              "2            3.0             0.078531  \n",
              "3            6.0             0.223187  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fda8e4db-d501-4749-a25c-6d79cd61a880\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>map</th>\n",
              "      <th>ndcg</th>\n",
              "      <th>ndcg_cut_10</th>\n",
              "      <th>mrt</th>\n",
              "      <th>map +</th>\n",
              "      <th>map -</th>\n",
              "      <th>map p-value</th>\n",
              "      <th>ndcg +</th>\n",
              "      <th>ndcg -</th>\n",
              "      <th>ndcg p-value</th>\n",
              "      <th>ndcg_cut_10 +</th>\n",
              "      <th>ndcg_cut_10 -</th>\n",
              "      <th>ndcg_cut_10 p-value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>TFIDF</td>\n",
              "      <td>0.010519</td>\n",
              "      <td>0.047832</td>\n",
              "      <td>0.589368</td>\n",
              "      <td>72.828908</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>TFIDF + CA(6f)</td>\n",
              "      <td>0.011807</td>\n",
              "      <td>0.052092</td>\n",
              "      <td>0.630170</td>\n",
              "      <td>191.067751</td>\n",
              "      <td>9.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.052802</td>\n",
              "      <td>9.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.010754</td>\n",
              "      <td>9.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.006948</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>TFIDF + RF(6f)</td>\n",
              "      <td>0.011907</td>\n",
              "      <td>0.051449</td>\n",
              "      <td>0.627952</td>\n",
              "      <td>203.712403</td>\n",
              "      <td>8.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.038993</td>\n",
              "      <td>9.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.090934</td>\n",
              "      <td>9.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.078531</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>TFIDF + LMart(6f)</td>\n",
              "      <td>0.011828</td>\n",
              "      <td>0.052731</td>\n",
              "      <td>0.622488</td>\n",
              "      <td>151.677864</td>\n",
              "      <td>6.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.217733</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.161192</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.223187</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fda8e4db-d501-4749-a25c-6d79cd61a880')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-fda8e4db-d501-4749-a25c-6d79cd61a880 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-fda8e4db-d501-4749-a25c-6d79cd61a880');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LmGe5rniC_Pn"
      },
      "source": [
        "Question - les trois modèles appris ont pu améliorer NDCG@10 sur TFIDF ? Analysez les résultats, en indicant s'il y a des amélorations statistiques.\n",
        "\n",
        "**Réponse**\n",
        "Oui, les 3 modèles ont tous amelioré le **ndcg_cut_10** qui est passé de \"0.589368\" **à** \"0.6337\"  / \"0.6279\" / \"0.6336\" selon le modèle "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1HhGDiPLHO7v"
      },
      "source": [
        "## Tâche pratique - Concaténation\n",
        "\n",
        "Notre modèle appris a un faible rappel car seulement 10 documents sont reclassés. Créons une petite fonction, `append_baseline()`, qui peut ajouter les résultats de la ligne de base BM25 à la sortie du modèle appris. Ceci est défini en utilisant les [opérateurs de transformation] (https://pyterrier.readthedocs.io/en/latest/operators.html) (`^` et `%`).\n",
        "\n",
        "Comme exercice, appliquez `append_baseline()` à chacun des pipelines de modèles appris définis ci-dessus, et rapportez le MAP et le NDCG calculés sur les 1000 résultats classés. \n",
        "\n",
        "Lequel des modèles appris résulte en une amélioration significative de MAP et NDCG ?\n",
        "\n",
        "**Réponse** : Modèle numéro 01 \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yypYHpf-nX7w"
      },
      "source": [
        "def append_baseline(system, baseline, max_results = 1000):\n",
        "    return (system ^ baseline) % max_results"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-BWu-iEUyV-F",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 429
        },
        "outputId": "d9801ba5-4e56-4111-ed73-6365819e2893"
      },
      "source": [
        "#réponse\n",
        "\n",
        "result1 = append_baseline(ca_pipe,bm25)\n",
        "result2 = append_baseline(rf_pipe,bm25)\n",
        "result3 = append_baseline(lmart_x_pipe,bm25)\n",
        "pt.Experiment(\n",
        "    [tfidf % RANK_CUTOFF, result1, result2, result3],\n",
        "    test_topics,\n",
        "    test_qrels, \n",
        "    names=[\"TFIDF\",  \"TFIDF + CA(6f)\", \"TFIDF + RF(6f)\", \"TFIDF + LMart(6f)\"],\n",
        "    baseline=0,\n",
        "    eval_metrics=[\"map\", \"ndcg\"])"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:48.204 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:52.356 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n",
            "[Parallel(n_jobs=2)]: Using backend ThreadingBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:    0.0s\n",
            "[Parallel(n_jobs=2)]: Done 196 tasks      | elapsed:    0.2s\n",
            "[Parallel(n_jobs=2)]: Done 400 out of 400 | elapsed:    0.4s finished\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:34:57.141 [main] WARN org.terrier.querying.ApplyTermPipeline - The index has no termpipelines configuration, and no control configuration is found. Defaulting to global termpipelines configuration of 'Stopwords,PorterStemmer'. Set a termpipelines control to remove this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pyterrier/ops.py:242: UserWarning: Got number of results different expected from <pyterrier.batchretrieve.TextScorer object at 0x7f5120e57e80>, expected 150 received 154, feature scores for any missing documents be 0, extraneous documents will be removed\n",
            "  warn(\"Got number of results different expected from %s, expected %d received %d, feature scores for any \"\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                name       map      ndcg  map +  map -  map p-value  ndcg +  \\\n",
              "0              TFIDF  0.010519  0.047832    NaN    NaN          NaN     NaN   \n",
              "1     TFIDF + CA(6f)  0.141777  0.333775   14.0    1.0     0.000051    14.0   \n",
              "2     TFIDF + RF(6f)  0.141749  0.332501   14.0    1.0     0.000055    14.0   \n",
              "3  TFIDF + LMart(6f)  0.141651  0.333769   13.0    2.0     0.000056    13.0   \n",
              "\n",
              "   ndcg -  ndcg p-value  \n",
              "0     NaN           NaN  \n",
              "1     1.0      0.000002  \n",
              "2     1.0      0.000003  \n",
              "3     2.0      0.000003  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e84c6132-ba53-46bf-8586-a4714e54ad8a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>map</th>\n",
              "      <th>ndcg</th>\n",
              "      <th>map +</th>\n",
              "      <th>map -</th>\n",
              "      <th>map p-value</th>\n",
              "      <th>ndcg +</th>\n",
              "      <th>ndcg -</th>\n",
              "      <th>ndcg p-value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>TFIDF</td>\n",
              "      <td>0.010519</td>\n",
              "      <td>0.047832</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>TFIDF + CA(6f)</td>\n",
              "      <td>0.141777</td>\n",
              "      <td>0.333775</td>\n",
              "      <td>14.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000051</td>\n",
              "      <td>14.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>TFIDF + RF(6f)</td>\n",
              "      <td>0.141749</td>\n",
              "      <td>0.332501</td>\n",
              "      <td>14.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000055</td>\n",
              "      <td>14.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>TFIDF + LMart(6f)</td>\n",
              "      <td>0.141651</td>\n",
              "      <td>0.333769</td>\n",
              "      <td>13.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.000056</td>\n",
              "      <td>13.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.000003</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e84c6132-ba53-46bf-8586-a4714e54ad8a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e84c6132-ba53-46bf-8586-a4714e54ad8a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e84c6132-ba53-46bf-8586-a4714e54ad8a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "# Tâche pratique\n",
        "\n",
        "Utilisez les modèles implémentés pour cord19 dans une tâche de question-réponse. Dans ce contexte, les requêtes sont de questions et les documents sont des documents qui pourraient contenir la réponse. Notez que vous devez refaire l'indexation ainsi que les autres étapes étudiées dans ce TP. Voici un exemple du dataset à utiliser :\n",
        "\n",
        "```json\n",
        "\"question\": \"Why are big companies like Apple or Google not included in the Dow Jones Industrial Average (DJIA) index?\",\n",
        "\n",
        "\"answers\":{\n",
        "  \"290156\": { \n",
        "    \"text\":\" That is a pretty exclusive club and for the most part they are not interested in highly volatile companies like Apple and Google. Sure, IBM is part of the DJIA, but that is about as stalwart as you can get these days. The typical profile for a DJIA stock would be one that pays fairly predictable dividends, has been around since money was invented, and are not going anywhere unless the apocalypse really happens this year. In summary, DJIA is the boring reliable company index.\" ,\n",
        "    \"timestamp\": \"Sep 11 '12 at 0:53\"}\n",
        " }\n",
        "\n",
        "```\n",
        "Vous pouvez le télécharger en utilisant les lines de code ci-dessous."
      ],
      "metadata": {
        "id": "ii8ACDXZOdOA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "KtKhqHD5JX9Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "fiqa = {}\n",
        "fiqa['train'] = pt.datasets.get_dataset('irds:beir/fiqa/train')\n",
        "fiqa['valid'] = pt.datasets.get_dataset('irds:beir/fiqa/dev')\n",
        "fiqa['test'] = pt.datasets.get_dataset('irds:beir/fiqa/test')\n",
        "\n",
        "test_topics = fiqa['test'].get_topics(variant='text')\n",
        "test_qrels = fiqa['test'].get_qrels()\n",
        "\n",
        "train_topics = fiqa['train'].get_topics(variant='text')\n",
        "train_qrels = fiqa['train'].get_qrels()\n",
        "\n",
        "valid_topics = fiqa['valid'].get_topics(variant='text')\n",
        "valid_qrels = fiqa['valid'].get_qrels()"
      ],
      "metadata": {
        "id": "tW-8L-1pOpB9"
      },
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#ltr_feats1\n",
        "print(fiqa)\n",
        "print(train_topics)\n",
        "print(train_qrels)\n",
        "test = pt.text.scorer()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iVYuk3Jxv36J",
        "outputId": "e349b37b-bf65-4422-d886-dbec5c5d6342"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'train': IRDSDataset('beir/fiqa/train'), 'valid': IRDSDataset('beir/fiqa/dev'), 'test': IRDSDataset('beir/fiqa/test')}\n",
            "        qid                                              query\n",
            "0         0  what is considered a business expense on a bus...\n",
            "1         4  business expense car insurance deductible for ...\n",
            "2         5                     starting a new online business\n",
            "3         6                business day and due date for bills\n",
            "4         7  new business owner how do taxes work for the b...\n",
            "...     ...                                                ...\n",
            "5495  11092  is real physical money traded during online tr...\n",
            "5496  11096                  pensions annuities and retirement\n",
            "5497  11097  what does it mean to a life insurance policy h...\n",
            "5498  11099  can capital loss in traditional ira and roth i...\n",
            "5499  11104  selling a stock for gain to offset other stock...\n",
            "\n",
            "[5500 rows x 2 columns]\n",
            "         qid   docno  label iteration\n",
            "0          0   18850      1         0\n",
            "1          4  196463      1         0\n",
            "2          5   69306      1         0\n",
            "3          6  560251      1         0\n",
            "4          6  188530      1         0\n",
            "...      ...     ...    ...       ...\n",
            "14161  11096  407726      1         0\n",
            "14162  11097  131224      1         0\n",
            "14163  11099  424427      1         0\n",
            "14164  11099  150878      1         0\n",
            "14165  11104  518310      1         0\n",
            "\n",
            "[14166 rows x 4 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Creation des fichiers d'indexation\n",
        "!rm -rf ./pd_index_fiqa_train\n",
        "!rm -rf ./pd_index_fiqa_test\n",
        "!rm -rf ./pd_index_fiqa_valid"
      ],
      "metadata": {
        "id": "Lg6CQ12Pu06b"
      },
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def indexingFun(data_list):\n",
        "  pt_index_path_dict = {}\n",
        "  for data in data_list:\n",
        "    pt_index_path = './pd_index_fiqa' + '_' + data\n",
        "    pt_index_path_dict[data] = pt_index_path\n",
        "\n",
        "    if not os.path.exists(pt_index_path + \"/data.properties\"):\n",
        "        # create the index, using the IterDictIndexer indexer \n",
        "        indexer = pt.index.IterDictIndexer(pt_index_path, blocks=True)\n",
        "\n",
        "        # we give the dataset get_corpus_iter() directly to the indexer\n",
        "        # while specifying the fields to index and the metadata to record\n",
        "        index_ref = indexer.index(fiqa[data].get_corpus_iter())\n",
        "\n",
        "    else:\n",
        "        # if you already have the index, use it.\n",
        "        index_ref = pt.IndexRef.of(pt_index_path + \"/data.properties\")\n",
        "  return pt_index_path_dict"
      ],
      "metadata": {
        "id": "o8dcLFX0Pb71"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tpPipeline(index_ref):\n",
        "  tf = pt.BatchRetrieve(index_ref, wmodel=\"Tf\")\n",
        "  tfidf = pt.BatchRetrieve(index_ref, wmodel=\"TF_IDF\")\n",
        "  bm_25 = pt.BatchRetrieve(index_ref, wmodel='BM25')\n",
        "  \n",
        "  pipeline = ((tf %10) | (tfidf % 10)) >> bm_25\n",
        "\n",
        "  return pipeline"
      ],
      "metadata": {
        "id": "qN2pKGnzKUYY"
      },
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#pipeline de caractéristiques \n",
        "def tpCharactPipeline(pipeline, data, index_ref, RANK_CUTOFF=10):\n",
        "  return (pipeline % RANK_CUTOFF) >> pt.text.get_text(data, metadata=['doc_id', 'text'], by_query=True) >> (\n",
        "      pt.transformer.IdentityTransformer()\n",
        "      ** \n",
        "      pt.BatchRetrieve(index_ref, wmodel=\"CoordinateMatch\")\n",
        "      )"
      ],
      "metadata": {
        "id": "_ijwlY7Dt2SM"
      },
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import fastrank\n",
        "\n",
        "\n",
        "def fastRankModel(charact_pipeline, train_topics, train_qrels, validation_topics, validation_qrels):\n",
        "  train_request = fastrank.TrainRequest.coordinate_ascent()\n",
        "\n",
        "  params = train_request.params\n",
        "  params.init_random = True\n",
        "  params.normalize = True\n",
        "  params.seed = 1234567\n",
        "\n",
        "  ca_pipe = charact_pipeline >> pt.ltr.apply_learned_model(train_request, form='fastrank')\n",
        "\n",
        "  %time ca_pipe.fit(train_topics, train_qrels, validation_topics, validation_qrels)\n",
        "\n",
        "  return ca_pipe"
      ],
      "metadata": {
        "id": "jIYQ23d6t68d"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "\n",
        "\n",
        "def randomForestModel(charact_pipeline, train_topics, train_qrels, validation_topics, validation_qrels):\n",
        "  rf = RandomForestRegressor(n_estimators=400, verbose=1, random_state=42, n_jobs=2)\n",
        "\n",
        "  rf_pipe = charact_pipeline >> pt.ltr.apply_learned_model(rf)\n",
        "  %time rf_pipe.fit(train_topics, train_qrels, validation_topics, validation_qrels)\n",
        "  return rf_pipe"
      ],
      "metadata": {
        "id": "ZG-RnYVwuA8l"
      },
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import lightgbm as lgb\n",
        "\n",
        "\n",
        "def lambdaRankModel(charact_pipeline, train_topics, train_qrels, validation_topics, validation_qrels):\n",
        "  # this configures LightGBM as LambdaMART\n",
        "  lmart_l = lgb.LGBMRanker(\n",
        "      task=\"train\",\n",
        "      silent=False,\n",
        "      min_data_in_leaf=1,\n",
        "      min_sum_hessian_in_leaf=1,\n",
        "      max_bin=255,\n",
        "      num_leaves=31,\n",
        "      objective=\"lambdarank\",\n",
        "      metric=\"ndcg\",\n",
        "      ndcg_eval_at=[10],\n",
        "      ndcg_at=[10],\n",
        "      eval_at=[10],\n",
        "      learning_rate= .1,\n",
        "      importance_type=\"gain\",\n",
        "      num_iterations=100,\n",
        "      early_stopping_rounds=5\n",
        "  )\n",
        "\n",
        "  lmart_x_pipe = charact_pipeline >> pt.ltr.apply_learned_model(lmart_l, form=\"ltr\", fit_kwargs={'eval_at':[10]})\n",
        "  %time lmart_x_pipe.fit(train_topics, train_qrels, validation_topics, validation_qrels)\n",
        "\n",
        "  return lmart_x_pipe"
      ],
      "metadata": {
        "id": "vJFaM_X9uIJY"
      },
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyterrier.measures import *\n",
        "\n",
        "def evalPipeline(pipeline, test_topics, test_qrels, name):\n",
        "  return pt.Experiment(\n",
        "      [pipeline],\n",
        "      test_topics,\n",
        "      test_qrels,\n",
        "      eval_metrics=[MAP, nDCG, nDCG@10],\n",
        "\n",
        "      baseline=0,\n",
        "      names=name\n",
        "      )"
      ],
      "metadata": {
        "id": "6CP7V2oCuMoz"
      },
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evalCharctPipeline(charact_pipeline, feature_names, test_topics, test_qrels):\n",
        "  return pt.Experiment(\n",
        "      [charact_pipeline >> pt.ltr.feature_to_score(i) for i in range(len(feature_names))],\n",
        "      test_topics,\n",
        "      test_qrels,  \n",
        "      names=feature_names,\n",
        "      eval_metrics=[\"map\", \"ndcg\", \"ndcg_cut_10\", \"num_rel_ret\"])"
      ],
      "metadata": {
        "id": "HD5v2r0IuQWu"
      },
      "execution_count": 81,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evalModel(pipeline, model_pipelines, test_topics, test_qrels, names, RANK_CUTOFF=10):\n",
        "  return pt.Experiment(\n",
        "      [pipeline % RANK_CUTOFF] + model_pipelines,\n",
        "      test_topics,\n",
        "      test_qrels, \n",
        "      names=names,\n",
        "      baseline=0,\n",
        "      eval_metrics=[\"map\", \"ndcg\", \"ndcg_cut_10\", \"mrt\"])"
      ],
      "metadata": {
        "id": "P8lpne0juWqO"
      },
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evalModelWithAppend(pipeline, model_pipelines, test_topics, test_qrels, names):\n",
        "  return pt.Experiment(\n",
        "      [pipeline] + [append_baseline(x, pipeline) for x in model_pipelines],\n",
        "      test_topics,\n",
        "      test_qrels, \n",
        "      names=names,\n",
        "      baseline=0,\n",
        "      eval_metrics=[\"map\", \"ndcg\", \"ndcg_cut_10\", \"mrt\"])"
      ],
      "metadata": {
        "id": "kbAXAliquZ1f"
      },
      "execution_count": 83,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_list = ['train', 'test', 'valid']\n",
        "\n",
        "names = ['Pipeline', 'FastR', 'RFR', 'Lambda']\n",
        "feature_names = ['CoordinateMatch']\n",
        "\n",
        "# Indexation\n",
        "pt_index_path_dict = indexingFun(data_list)\n",
        "\n",
        "# Creation Pipeline\n",
        "pipeline = tpPipeline(pt.IndexRef.of(pt_index_path_dict['train'] + \"/data.properties\"))\n",
        "charact_pipeline = tpCharactPipeline(pipeline, fiqa['train'], pt_index_path_dict['train'])\n",
        "model_pipelines = [fastRankModel(charact_pipeline, train_topics, train_qrels, valid_topics, valid_qrels),\n",
        "                   randomForestModel(charact_pipeline, train_topics, train_qrels, valid_topics, valid_qrels),\n",
        "                   lambdaRankModel(charact_pipeline, train_topics, train_qrels, valid_topics, valid_qrels)]\n",
        "\n",
        "# Evaluations\n",
        "evaluations_dict = {}\n",
        "print(\"Evaluation : Pipeline\")\n",
        "evaluations_dict['Pipeline'] = evalPipeline(pipeline, test_topics, test_qrels, ['Pipeline'])\n",
        "\n",
        "print(\"Evaluation : Characteristics Pipeline\")\n",
        "evaluations_dict['Charact_Pipeline'] = evalCharctPipeline(charact_pipeline, feature_names, test_topics, test_qrels)\n",
        "\n",
        "print(\"Evaluation : Models\")\n",
        "evaluations_dict['Models'] = evalModel(pipeline, model_pipelines, test_topics, test_qrels, names)\n",
        "\n",
        "print(\"Evaluation : Models with append_baseline\")\n",
        "evaluations_dict['Madels_with_Append'] = evalModelWithAppend(pipeline, model_pipelines, test_topics, test_qrels, names)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386,
          "referenced_widgets": [
            "090f3d2dbace44cda71a0e63d0882d64",
            "445aaa9f42ca40c8bbb95fc0c5c5f7de",
            "ea9c0625dbbc4188a6a9b6c8d3b97f1e",
            "ef0281748ca94d76b052e797fe6c166f",
            "4b9c008d95204f6889dd5c3ac0ad2285",
            "e5223cd1d45c4abcba07f337ddcf840e",
            "843b38a255454bc484a64dab796fe693",
            "35d72040b9e54c9a84909e3bc6a76f7a",
            "74196af47eb44385af7f664fe65698f3",
            "76976a734a4c4d899c091b65f81573e2",
            "fafb87b4be6e49e0948d6aa5502a6614",
            "eee782808d2240a39570c856102372d1",
            "503fd9d4e244409487287c8d67acc8c9",
            "b1d15d78637241cd9c3534c42993309b",
            "84780546219744fe866441023ab0b63f",
            "4dc50292591f41a2a50b00fc38f582cb",
            "e54fdb51b87f419d835c1b3461ffc6e8",
            "bf18a1f423ce4705ac2e4aecadcb5d06",
            "615c489fa6674a06afb386b54c0c0351",
            "5799ca054df1418cbd7c381dd25b6bb1",
            "2e99697c8d574e4ea0552ea424177849",
            "bfb7c0f0cb994ce5aa8774ff0307ff92",
            "6be3c98ae1874a2ba753ffca259b8413",
            "3d125cb328a14addbb12a73581537033",
            "cd6674aa6feb48659a9dfad2b42dc62b",
            "5dfd1115a57643c49fdd954eaadf55b0",
            "efc527a2be9f49f3acd5a0bccbe0e38c",
            "9e6d55f017834827bad2d32180d8a9ef",
            "082d7c65a13042569be15d8a4b3a6130",
            "06b4b85af7604f0687b22b9f42bb2dee",
            "1d6ef861c6b540eb943c529a3529a39e",
            "0240b73b60e24b97a067eab1a00c84aa",
            "4c452bf63d424d94bc514042066d930b"
          ]
        },
        "id": "vgQcKYQ4wtt3",
        "outputId": "426fe5fc-a8d6-4f9f-ad81-f67db5568867"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "beir/fiqa/train documents:   0%|          | 0/57638 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "090f3d2dbace44cda71a0e63d0882d64"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:35:00.701 [ForkJoinPool-4-worker-3] WARN org.terrier.structures.indexing.Indexer - Adding an empty document to the index (7915) - further warnings are suppressed\n",
            "13:35:23.007 [ForkJoinPool-4-worker-3] WARN org.terrier.structures.indexing.Indexer - Indexed 39 empty documents\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "beir/fiqa/test documents:   0%|          | 0/57638 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "eee782808d2240a39570c856102372d1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:35:23.556 [ForkJoinPool-5-worker-3] WARN org.terrier.structures.indexing.Indexer - Adding an empty document to the index (7915) - further warnings are suppressed\n",
            "13:35:44.066 [ForkJoinPool-5-worker-3] WARN org.terrier.structures.indexing.Indexer - Indexed 39 empty documents\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "beir/fiqa/dev documents:   0%|          | 0/57638 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6be3c98ae1874a2ba753ffca259b8413"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13:35:44.456 [ForkJoinPool-6-worker-3] WARN org.terrier.structures.indexing.Indexer - Adding an empty document to the index (7915) - further warnings are suppressed\n",
            "13:36:06.011 [ForkJoinPool-6-worker-3] WARN org.terrier.structures.indexing.Indexer - Indexed 39 empty documents\n",
            "CPU times: user 9min 15s, sys: 10.3 s, total: 9min 26s\n",
            "Wall time: 9min 21s\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=2)]: Using backend ThreadingBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:    4.7s\n",
            "[Parallel(n_jobs=2)]: Done 196 tasks      | elapsed:   19.8s\n",
            "[Parallel(n_jobs=2)]: Done 400 out of 400 | elapsed:   40.4s finished\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 9min 59s, sys: 9.02 s, total: 10min 8s\n",
            "Wall time: 9min 41s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluations_dict['Models']"
      ],
      "metadata": {
        "id": "Yjf46wdvzmnx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "evaluations_dict[\"Madels_with_Append\"]"
      ],
      "metadata": {
        "id": "W-WkffCLxB89"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}