heliokos_repl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05992935-2c6d-4501-b9cb-5e1f4e1c0a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import concurrent.futures\n",
    "import csv\n",
    "import glob\n",
    "import gzip\n",
    "from itertools import permutations, combinations, starmap, repeat\n",
    "import json\n",
    "from math import ceil\n",
    "import operator\n",
    "import os\n",
    "from pprint import pprint\n",
    "import re\n",
    "import string\n",
    "from urllib.parse import quote_plus\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "from toolz import concat, identity, keyfilter, partition_all, assoc, dissoc, unique, merge\n",
    "from pymongo import MongoClient, InsertOne, UpdateOne\n",
    "from rdflib import Graph, URIRef, Literal\n",
    "from rdflib.namespace import SKOS, RDFS, RDF\n",
    "import requests\n",
    "\n",
    "from heliokos.infra.core import Harmonization\n",
    "\n",
    "def pick(whitelist, d):\n",
    "    return keyfilter(lambda k: k in whitelist, d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32addbc8-4f53-4a17-9537-dd9e35ce1367",
   "metadata": {},
   "outputs": [],
   "source": [
    "def part_fn(part, doc_fn):\n",
    "    rv = []\n",
    "    with gzip.open(part, \"rt\") as f:\n",
    "        for line in f:\n",
    "            doc = json.loads(line)\n",
    "            rv.append(doc_fn(doc))\n",
    "    return rv\n",
    "\n",
    "def process_parts(parts, doc_fn):\n",
    "\n",
    "    pbar = tqdm(total=len(parts))\n",
    "    rv = []\n",
    "    with concurrent.futures.ProcessPoolExecutor(max_workers=32) as executor:\n",
    "        future_to_part = {executor.submit(part_fn, part, doc_fn): part for part in parts}\n",
    "        for future in concurrent.futures.as_completed(future_to_part):\n",
    "            part = future_to_part[future]\n",
    "            try:\n",
    "                data = future.result()\n",
    "                rv.extend(data)\n",
    "                pbar.update(1)\n",
    "            except Exception as exc:\n",
    "                print(f\"{part} generated an exception: {exc}\")\n",
    "    return rv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77a85519-2435-43df-96a5-712abbdf3bb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = MongoClient()\n",
    "mdb = client.heliokos"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc64bf37-b68e-4837-93d4-ad92335926d0",
   "metadata": {},
   "source": [
    "# store openalex concepts in mongodb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ab352cd-35cc-44a6-a3e3-f2a583252747",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://docs.openalex.org/download-all-data/openalex-snapshot\n",
    "oax_concept_parts = glob.glob(\"/openalex-snapshot/data/concepts/**/*.gz\")\n",
    "\n",
    "def it(doc):\n",
    "    return doc\n",
    "\n",
    "oax_concepts = process_parts(oax_concept_parts, it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6919680-1928-498a-80c2-5cf4480db913",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(oax_concepts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae10ce5d-8480-4746-9d6f-06ce590beaa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "rv = mdb.oax_concepts.insert_many(oax_concepts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a93ffd1-3e48-4e9e-80f2-b6e54d838107",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(rv.inserted_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20f28ff1-c541-4d1e-bdca-e63c43468b05",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdb.oax_concepts.create_index(\"id\", unique=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f863727-675d-46f8-95bc-1791d52b8feb",
   "metadata": {},
   "source": [
    "# Rig up MongoDB as a serviceable graph database\n",
    "\n",
    "Using: Coupal, Daniel, Pascal Desmarets, and Steve Hoberman. MongoDB Data Modeling and Schema Design. First. Align > Refine > Design. Sedona: Technics Publications, 2023.\n",
    "\n",
    "- single collection pattern (relatedTo -> attribute pattern: outgoing.o, incoming.s)\n",
    "- inheritance pattern (id, type, author.orcid, ads_work.bibcode, ...)\n",
    "- attribute pattern (outgoing: [{p,o}], incoming: [{s,p}])\n",
    "- graph pattern (incoming / outgoing) / tree pattern (parent / ancestors / children)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dde3e861-5b55-4132-b6ee-7a3a5b95b303",
   "metadata": {},
   "source": [
    "# Get unique DOIs from ADS \"snapshot\""
   ]
  },
  {
   "cell_type": "raw",
   "id": "bed44579-8cb8-4a72-a1b5-f21d722b1375",
   "metadata": {},
   "source": [
    "db.getCollection(\"ads_works_doiurls\").aggregate(\n",
    "\n",
    "    // Pipeline\n",
    "    [\n",
    "        // Stage 1\n",
    "        {\n",
    "            $project: {\n",
    "                // specifications\n",
    "                \"_id\": {\"$first\": \"$doi\"}\n",
    "            }\n",
    "        },\n",
    "\n",
    "        // Stage 2\n",
    "        {\n",
    "            $group: {\n",
    "                \"_id\": \"$_id\"\n",
    "            }\n",
    "        },\n",
    "\n",
    "        // Stage 3\n",
    "        {\n",
    "            $out: // Note: must be last stage of pipeline\n",
    "            {\n",
    "                 db: \"helioweb\",\n",
    "                 coll: \"ads_works_only_doiurls\"\n",
    "            }\n",
    "        }\n",
    "    ],\n",
    "\n",
    "    // Options\n",
    "    {\n",
    "        allowDiskUse: true\n",
    "    }\n",
    "\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3f515ec-69c2-4b2b-bdd9-a01264be8a8e",
   "metadata": {},
   "source": [
    "above took 1.5 minutes on laptop, 3.5 minutes on beefy server. Hmmm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c9e22b8-354b-4a8e-bf47-e3dbc718b827",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"{mdb.ads_works_doiurls.estimated_document_count():,}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd555f09-8ad9-4b79-8d93-b6a695ca3dd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"{mdb.ads_works_only_doiurls.estimated_document_count():,}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bebf9f30-be82-48bd-b4f7-823a372ddb24",
   "metadata": {},
   "outputs": [],
   "source": [
    "ads_unique_dois = {d[\"_id\"] for d in mdb.ads_works_only_doiurls.find()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "585494f3-9272-4eb3-b809-462e59ef3a3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "None in ads_unique_dois"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55b0296a-69c3-445b-96d9-377e4995e91a",
   "metadata": {},
   "source": [
    "# Get slim OpenAlex work for every ADS DOI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d4d698e-339c-4924-a5f3-a5c5b0a41ea1",
   "metadata": {},
   "source": [
    "What do I need from an OpenAlex work?\n",
    "\n",
    "- doi\n",
    "- title\n",
    "- publication_date\n",
    "- type\n",
    "- concepts.id, concepts.score\n",
    "- referenced_works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08b15398-ee7c-4e45-9000-364e9a69f9e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def work_essentials(doc):\n",
    "    rv =  pick([\"doi\", \"title\", \"publication_date\", \"type\", \"referenced_works\"], doc)\n",
    "    rv[\"concepts\"] = [pick([\"id\", \"score\"], c) for c in doc[\"concepts\"]]\n",
    "    rv[\"_id\"] = doc[\"id\"]\n",
    "    return rv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ae8f175-4260-46ad-9592-af9cdd2aebcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "oax_works = glob.glob(\"/openalex-snapshot/data/works/**/*.gz\")\n",
    "\n",
    "def openalex_work_doi_in_ads(doc):\n",
    "    return work_essentials(doc) if doc.get(\"doi\") in ads_unique_dois else None\n",
    "\n",
    "openalex_works_with_doi_in_ads = process_parts(oax_works, openalex_work_doi_in_ads)\n",
    "openalex_works_with_doi_in_ads = list(filter(None, openalex_works_with_doi_in_ads))\n",
    "len(openalex_works_with_doi_in_ads)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e4bdc34-f88d-4f31-adc9-61a10232d842",
   "metadata": {},
   "source": [
    "above took 17 minutes on server using 32 workers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f87b7d0-816d-4852-b3fe-1bdb121855ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "chunksize = 1_000\n",
    "\n",
    "n_chunks = (len(openalex_works_with_doi_in_ads) // chunksize) + 1\n",
    "\n",
    "for i, chunk in enumerate(tqdm(partition_all(chunksize, openalex_works_with_doi_in_ads), total=n_chunks)):\n",
    "    requests = [InsertOne(d) for d in chunk]\n",
    "    result = mdb.oax_works_with_doi_in_ads.bulk_write(requests, ordered=False)\n",
    "    assert result.inserted_count == len(requests), f\"chunk {i} failure\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd1c5e16-0880-4516-ba5b-d74ca5bf9a1c",
   "metadata": {},
   "source": [
    "above took 7 minutes. The MongoDB collection is 18G in memory, 7G on disk."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9657dd70-ad32-40db-8134-bc5dcc031a31",
   "metadata": {},
   "source": [
    "# UAT as SKOS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba0ba0d7-bdb4-4adf-830e-43f1cf37ac34",
   "metadata": {},
   "outputs": [],
   "source": [
    "g_uat = Graph()\n",
    "g_uat.parse(location=\"https://raw.githubusercontent.com/astrothesaurus/UAT/v.5.0.0/UAT.rdf\", format=\"xml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "443794d8-3c73-49d2-b0e8-3a3bf88f706f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"saving to disk...\")\n",
    "g_uat.serialize(\"uat.ttl\")\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "801c9a7a-b3b8-45f7-be27-b0ef995a76b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(t[0].n3(g_uat.namespace_manager) for t in g_uat.query(\"SELECT DISTINCT ?p WHERE {?s ?p ?o }\", initNs={\"skos\": SKOS}))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "209e0bed-fee5-464e-b7b9-78ca3579477c",
   "metadata": {},
   "source": [
    "# OpenAlex concepts as skos:ConceptScheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e82d216-0732-4cf6-a1a9-81e68b755219",
   "metadata": {},
   "outputs": [],
   "source": [
    "def concept_essentials(doc):\n",
    "    rv =  pick([\"display_name\", \"description\", \"level\"], doc)\n",
    "    rv[\"ancestors\"] = [pick([\"id\", \"level\"], c) for c in doc[\"ancestors\"]]\n",
    "    rv[\"_id\"] = doc[\"id\"]\n",
    "    return rv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d579f28-6fb1-456e-b03a-f93046a26ed3",
   "metadata": {},
   "outputs": [],
   "source": [
    "oax_concept_parts = glob.glob(\"/openalex-snapshot/data/concepts/**/*.gz\")\n",
    "\n",
    "oax_concepts = process_parts(oax_concept_parts, concept_essentials)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c970c8d0-9b9a-4820-abdb-191fe3069086",
   "metadata": {},
   "source": [
    "above took 30 seconds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de8c600d-0491-4ab7-bfec-303899594833",
   "metadata": {},
   "outputs": [],
   "source": [
    "chunksize = 1_000\n",
    "\n",
    "n_chunks = (len(oax_concepts) // chunksize) + 1\n",
    "\n",
    "for i, chunk in enumerate(tqdm(partition_all(chunksize, oax_concepts), total=n_chunks)):\n",
    "    requests = [InsertOne(d) for d in chunk]\n",
    "    result = mdb.oax_concepts_slim.bulk_write(requests, ordered=False)\n",
    "    assert result.inserted_count == len(requests), f\"chunk {i} failure\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "962cb322-2230-4312-8629-65940e0d9947",
   "metadata": {},
   "source": [
    "10x reduction in collection size in memory, from 380M to 34M."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84ecde88-3696-4bf6-9232-d5e271b6b88f",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = list(mdb.oax_concepts_slim.find())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "205dc3bc-ff9b-4e70-8923-a21aa1b90d39",
   "metadata": {},
   "outputs": [],
   "source": [
    "g_oax = Graph()\n",
    "scheme_id = URIRef(\"https://heliokos.example/OpenAlex\")\n",
    "g_oax.add((scheme_id, RDF.type, SKOS.ConceptScheme))\n",
    "g_oax.add((scheme_id, SKOS.prefLabel, Literal(\"OpenAlex Concept Scheme\")))\n",
    "g_oax.add((scheme_id, RDFS.seeAlso, URIRef(\"https://docs.openalex.org/api-entities/concepts\")))\n",
    "\n",
    "for d in tqdm(docs):\n",
    "    id_ = URIRef(d[\"_id\"])\n",
    "    g_oax.add((id_, RDF.type, SKOS.Concept))\n",
    "    g_oax.add((id_, SKOS.inScheme, scheme_id))\n",
    "    g_oax.add((id_, SKOS.prefLabel, Literal(d[\"display_name\"])))\n",
    "    if d.get(\"description\"):\n",
    "        g_oax.add((id_, SKOS.definition, Literal(d[\"description\"])))\n",
    "    if d[\"level\"] == 0:\n",
    "        g_oax.add((scheme_id, SKOS.hasTopConcept, id_))\n",
    "    for a in d[\"ancestors\"]:\n",
    "        if a[\"level\"] == d[\"level\"] - 1:\n",
    "            g_oax.add((URIRef(a[\"id\"]), SKOS.narrower, id_))\n",
    "\n",
    "print(\"saving to disk...\")\n",
    "g_oax.serialize(\"openalex.ttl\")\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "808a6c03-e9e7-40a8-837c-8216fd78e1c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "g_oax, g_uat = Graph(), Graph()\n",
    "g_oax.parse(\"src/heliokos/infra/static/openalex.ttl\")\n",
    "g_uat.parse(\"src/heliokos/domain/uat.ttl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85de6fab-7b38-476e-ab4c-ee2b6b962210",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(g_uat), len(g_oax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e1325c1-d130-46a9-895a-18fdfd5225d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "len([s for s in g_uat.subjects(RDF.type, SKOS.Concept, unique=True)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ce098e1-9e5f-4834-ad8c-5de90f2e22fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "len([s for s in g_oax.subjects(RDF.type, SKOS.Concept, unique=True)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56e73753-7fbd-4fe7-a1e0-fcf93f04d362",
   "metadata": {},
   "source": [
    "# Pruning skos:ConceptScheme to Cover Corpus Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68f692e9-c177-4be9-ac4b-a612b4077507",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all concepts that tag corpus\n",
    "concepts_in_corpus = [d[\"_id\"] for d in mdb.oax_works_with_doi_in_ads.aggregate([\n",
    "    {\"$project\": {\"_id\": 0, \"concepts.id\": 1}},\n",
    "    {\"$unwind\": {\"path\": \"$concepts\"}},\n",
    "    {\"$group\": {\"_id\": \"$concepts.id\"}},\n",
    "    ], allowDiskUse=True)]"
   ]
  },
  {
   "cell_type": "raw",
   "id": "5e6b1a7a-78d4-45f2-a2c8-8ce6cea477f0",
   "metadata": {},
   "source": [
    "db.getCollection(\"oax_works_with_doi_in_ads\").aggregate(\n",
    "\n",
    "    // Pipeline\n",
    "    [\n",
    "        // Stage 1\n",
    "        {\n",
    "            $project: {\n",
    "                \"_id\": 0,\n",
    "                \"concepts.id\": 1\n",
    "            }\n",
    "        },\n",
    "\n",
    "        // Stage 2\n",
    "        {\n",
    "            $unwind: {\n",
    "                path: \"$concepts\",\n",
    "            }\n",
    "        },\n",
    "\n",
    "        // Stage 3\n",
    "        {\n",
    "            $group: {\n",
    "                _id: \"$concepts.id\",\n",
    "            }\n",
    "        },\n",
    "\n",
    "        // Stage 4\n",
    "        {\n",
    "            $out: // Note: must be last stage of pipeline\n",
    "            {\n",
    "                 db: \"helioweb\",\n",
    "                 coll: \"oax_concepts_in_oax_works_with_doi_in_ads\"\n",
    "            }\n",
    "        }\n",
    "    ],\n",
    "\n",
    "    // Options\n",
    "    {\n",
    "        allowDiskUse: true\n",
    "    }\n",
    "\n",
    ");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "006796a9-8c29-4d95-85fb-db7eb657546d",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_in_ads, n_total = mdb.oax_concepts_in_oax_works_with_doi_in_ads.estimated_document_count(), mdb.oax_concepts.estimated_document_count()\n",
    "print(f\"{n_total=:,}, {n_in_ads=:,}\")\n",
    "f\"{(n_in_ads/n_total):.0%}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f240798a-28ec-49cb-bce7-01e2e95e5051",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all concepts that tag corpus\n",
    "# - (got it above)\n",
    "# make set of all these concepts and their ancestors\n",
    "# - automatic with oax\n",
    "# get complement of above set\n",
    "# - yuck, still 98% (>60k) concepts present. so not going to bother with next step.\n",
    "# for each member of this complement m,\n",
    "#   remove all (m, None, None) and (None, None, m) triples from the scheme.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "596d8d3b-7f5c-4276-91c6-6b146e9c2bd3",
   "metadata": {},
   "source": [
    "# Assisted Harmonization of UAT to OpenAlex\n",
    "UAT as retrieval scheme and OpenAlex as tagging scheme."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66404316-143e-44e1-a5d2-cfd6874fbb06",
   "metadata": {},
   "source": [
    "Approach: from Accidental Taxonomist 3rd ed. (2022), pp 348-9.\n",
    "\n",
    "A series of comparison passes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccc30a36-1e4d-4b56-b812-ffcb9a29ccd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# primary/retrieval taxonomy: uat\n",
    "# merging/tagging taxonomy: oax\n",
    "from heliokos.infra.core import ConceptScheme\n",
    "\n",
    "hzn = Harmonization(\n",
    "    tagging_scheme=ConceptScheme.from_graph(g_oax),\n",
    "    retrieval_scheme=ConceptScheme.from_graph(g_uat)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69ce577d-f4b9-4322-97ac-be6638bce437",
   "metadata": {},
   "outputs": [],
   "source": [
    "all(\"openalex.org\" in str(c) for c in hzn.tagging_scheme.g.subjects(RDF.type, SKOS.Concept))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2c1496e-5070-4fba-b04b-999f25cbc749",
   "metadata": {},
   "outputs": [],
   "source": [
    "all(\"astrothesaurus.org\" in str(c) for c in hzn.retrieval_scheme.g.subjects(RDF.type, SKOS.Concept))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "734089f9-5559-4f1f-a070-97cbc60cec02",
   "metadata": {},
   "source": [
    "1. exact match of merging_preflabel with primary_preflabel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "faf8859d-8761-4920-96b1-2a1a2e6daaf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_en_us_label(label):\n",
    "    return (label.language is None) or (label.language in (\"en\", \"en-US\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f81d2129-227a-4dda-81b2-56541934e098",
   "metadata": {},
   "outputs": [],
   "source": [
    "t_merging = hzn.tagging_scheme.g\n",
    "t_primary = hzn.retrieval_scheme.g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1a27355-670a-4626-89da-010cada87f89",
   "metadata": {},
   "outputs": [],
   "source": [
    "merging_preflabel_concepts = {str(preflabel): c for c, preflabel in t_merging.query(\"\"\"\n",
    "SELECT ?c ?preflabel WHERE {\n",
    "?c a skos:Concept .\n",
    "?c skos:prefLabel ?preflabel .\n",
    "}\"\"\") if is_en_us_label(preflabel)}\n",
    "primary_preflabel_concepts = {str(preflabel): c for c, preflabel in t_primary.query(\"\"\"\n",
    "SELECT ?c ?preflabel WHERE {\n",
    "?c a skos:Concept .\n",
    "?c skos:prefLabel ?preflabel .\n",
    "}\"\"\") if is_en_us_label(preflabel)}\n",
    "\n",
    "exact_match_preflabel_preflabel = {}\n",
    "for merging_preflabel, merging_concept in merging_preflabel_concepts.items():\n",
    "    if merging_preflabel in primary_preflabel_concepts:\n",
    "        exact_match_preflabel_preflabel[merging_concept] = primary_preflabel_concepts[merging_preflabel]\n",
    "len(exact_match_preflabel_preflabel)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13e91897-212c-4140-ae70-f03208558837",
   "metadata": {},
   "source": [
    "2. exact match of merging_preflabel with a primary_altlabel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9f25b3b-5f73-4448-b0e4-1f298fd1da40",
   "metadata": {},
   "outputs": [],
   "source": [
    "primary_altlabel_concepts = {str(altlabel): c for c, altlabel in t_primary.query(\"\"\"\n",
    "SELECT ?c ?altlabel WHERE {\n",
    "?c a skos:Concept .\n",
    "?c skos:altLabel ?altlabel .\n",
    "}\"\"\") if is_en_us_label(altlabel)}\n",
    "\n",
    "exact_match_preflabel_altlabel = {}\n",
    "for merging_preflabel, merging_concept in merging_preflabel_concepts.items():\n",
    "    if merging_preflabel in primary_altlabel_concepts:\n",
    "        exact_match_preflabel_altlabel[merging_concept] = primary_altlabel_concepts[merging_preflabel]\n",
    "len(exact_match_preflabel_altlabel)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1178512e-806e-4b0b-8fc9-e17a5a817b3f",
   "metadata": {},
   "source": [
    "3. exact match of merging_altlabel with primary_preflabel. merging_preflabel should be added as a primary_altlabel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69318a48-bbed-4f6d-b383-a8ff1870bd09",
   "metadata": {},
   "outputs": [],
   "source": [
    "merging_altlabel_concepts = {str(altlabel): c for c, altlabel in t_merging.query(\"\"\"\n",
    "SELECT ?c ?altlabel WHERE {\n",
    "?c a skos:Concept .\n",
    "?c skos:altLabel ?altlabel .\n",
    "}\"\"\") if is_en_us_label(altlabel)}\n",
    "\n",
    "exact_match_altlabel_preflabel = {}\n",
    "for merging_altlabel, merging_concept in merging_altlabel_concepts.items():\n",
    "    if merging_altlabel in primary_preflabel_concepts:\n",
    "        exact_match_altlabel_preflabel[merging_concept] = primary_preflabel_concepts[merging_altlabel]\n",
    "len(exact_match_altlabel_preflabel)\n",
    "# unsuprisingly 0, as oax concepts don't have altlabels."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39b96647-6d5e-4dda-aad4-ebcae32620b4",
   "metadata": {},
   "source": [
    "4. exact match of merging_altlabel with primary_altlabel. merging_preflabel should be added as a primary_altlabel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af56ee08-e94a-463e-8076-3f74354273a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "exact_match_altlabel_altlabel = {}\n",
    "for merging_altlabel, merging_concept in merging_altlabel_concepts.items():\n",
    "    if merging_altlabel in primary_altlabel_concepts:\n",
    "        exact_match_altlabel_altlabel[merging_concept] = primary_altlabel_concepts[merging_altlabel]\n",
    "len(exact_match_altlabel_altlabel)\n",
    "# unsuprisingly 0, as oax concepts don't have altlabels."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9624d84c-784d-4824-b850-5c3ed86c1335",
   "metadata": {},
   "source": [
    "5. close (not exact) match of merging_preflabel with primary_preflabel or primary_altlabel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f9846b0-01b1-4c8a-97bf-1c7cbf9ff050",
   "metadata": {},
   "outputs": [],
   "source": [
    "# use elasticsearch analyzer to get set of tokens for each label.\n",
    "\n",
    "base_url = os.getenv(\"ELASTIC_HOST\")\n",
    "auth = (\"elastic\", os.getenv(\"ELASTIC_PASSWORD\"))\n",
    "\n",
    "def get_tokens(text=None):\n",
    "    if text is None:\n",
    "        raise ValueError(\"no text supplied\")\n",
    "    \n",
    "    rv = requests.post(\n",
    "        base_url + \"/_analyze\",\n",
    "        json={\n",
    "            \"tokenizer\": \"standard\",\n",
    "            \"filter\": [\"snowball\"],\n",
    "            \"text\": text,\n",
    "        },\n",
    "        auth=auth,\n",
    "    )\n",
    "    rv.raise_for_status()\n",
    "    return rv.json()[\"tokens\"]\n",
    "\n",
    "def get_label_tokens_threadpool(label_concepts):\n",
    "    \"\"\"estimated ≈ 7 hours for ≈ 65k concepts done serially. No thanks.\"\"\"\n",
    "    rv = {}\n",
    "    with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor:\n",
    "        future_to_label = {\n",
    "            executor.submit(get_tokens, label): label\n",
    "            for label in label_concepts\n",
    "        }\n",
    "        for future in tqdm(concurrent.futures.as_completed(future_to_label), total=len(label_concepts)):\n",
    "            label = future_to_label[future]\n",
    "            try:\n",
    "                data = future.result()\n",
    "            except Exception as exc:\n",
    "                print('%r generated an exception: %s' % (label, exc))\n",
    "            else:\n",
    "                rv[label] = data\n",
    "    return rv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32435f9e-a8d6-4d41-8e14-6ba4c045203d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def insert_concept_label_tokens(label_concepts, label_tokens, type_=\"preflabel\"):\n",
    "    reqs = []\n",
    "    for label, concept_id in label_concepts.items():\n",
    "        reqs.append(InsertOne({\n",
    "            \"concept\": str(concept_id),\n",
    "            \"label\": label,\n",
    "            \"type\": type_,\n",
    "            \"tokens\": label_tokens[label],\n",
    "        }))\n",
    "    \n",
    "    chunksize = 1_000\n",
    "    n_chunks = (len(reqs) // chunksize) + 1\n",
    "    for i, chunk in enumerate(tqdm(partition_all(chunksize, reqs), total=n_chunks)):\n",
    "        result = mdb.concept_label_tokens.bulk_write([req for req in chunk], ordered=False)\n",
    "        assert result.inserted_count == len(chunk), f\"chunk {i} failure\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "889a4214-4c46-4151-92a2-b942ef4fbf10",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdb.drop_collection(\"concept_label_tokens\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eaaf481-750d-4347-bf42-60f3276b729e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#primary_preflabel_tokens = get_label_tokens_threadpool(primary_preflabel_concepts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87bca691-e41d-4245-bda3-adbc99f18e16",
   "metadata": {},
   "outputs": [],
   "source": [
    "insert_concept_label_tokens(primary_preflabel_concepts, primary_preflabel_tokens, \"preflabel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1876675d-50d8-47ef-895e-151987836229",
   "metadata": {},
   "outputs": [],
   "source": [
    "#merging_preflabel_tokens = get_tokens_threadpool(merging_preflabel_concepts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aed92f08-d41f-409f-9da0-ee33e802b121",
   "metadata": {},
   "source": [
    "above took ≈ 13 minutes with 32 threadpool workers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce337924-7539-42e0-b223-1702d126e732",
   "metadata": {},
   "outputs": [],
   "source": [
    "insert_concept_label_tokens(merging_preflabel_concepts, merging_preflabel_tokens, \"preflabel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9aa4637f-5ace-4f6d-9f59-59858acba88e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#primary_altlabel_tokens = get_label_tokens_threadpool(primary_altlabel_concepts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0819c515-415f-4225-beab-82ef2ad57d62",
   "metadata": {},
   "outputs": [],
   "source": [
    "insert_concept_label_tokens(primary_altlabel_concepts, primary_altlabel_tokens, \"altlabel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c6c844c-748a-4e00-9b01-a3331743dfca",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "len(mdb.concept_label_tokens.distinct(\"concept\", {\"concept\": {\"$regex\": \"openalex\"}}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32ecddbe-27d7-42dc-9c7c-1dd597ffa998",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(merging_preflabel_concepts.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e66f0db8-f27e-4806-8bf7-8ffd0737aa27",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(mdb.concept_label_tokens.distinct(\"concept\", {\"concept\": {\"$regex\": \"astrothesaurus\"}}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b915232b-9ea7-43f4-8cad-dfcb661b84b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdb.concept_label_tokens.create_index({\"concept\": 1, \"type\": 1, \"label\": 1}, unique=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0acb62aa-e25d-4e08-9bcb-c368c8f1ef41",
   "metadata": {},
   "source": [
    "Whew. Okay, finally: 5. close (not exact) match of merging_preflabel with primary_preflabel or primary_altlabel.\n",
    "\n",
    "- first pass: merging_preflabel set-of-tokens is same as primary_preflabel set-of-tokens\n",
    "- second pass: merging_preflabel set-of-tokens is same as primary_altlabel set-of-tokens\n",
    "\n",
    "optional, only if both merging set-of-tokens and primary set-of-tokens have >= 4 members.\n",
    "\n",
    "- third pass: merging_preflabel set-of-tokens has >75% intersection with primary_preflabel set-of-tokens\n",
    "- fourth pass: merging_preflabel set-of-tokens has >75% intersection with primary_altlabel set-of-tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fea90ae-a822-41e0-a1ec-0762aa133bfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "merging_preflabel_filter = {\"concept\": {\"$regex\": \"^https://openalex.org/\"}, \"type\": \"preflabel\"}\n",
    "n_mpl = mdb.concept_label_tokens.count_documents(merging_preflabel_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf03fe0d-0893-4e1f-85eb-aeb3e32143ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "primary_preflabel_filter = {\"concept\": {\"$regex\": \"^http://astrothesaurus.org/\"}, \"type\": \"preflabel\"}\n",
    "n_ppl = mdb.concept_label_tokens.count_documents(primary_preflabel_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5319ff1-9538-41dd-b3fb-37cd2ab0d0c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "primary_altlabel_filter = {\"concept\": {\"$regex\": \"^http://astrothesaurus.org/\"}, \"type\": \"altlabel\"}\n",
    "n_pal = mdb.concept_label_tokens.count_documents(primary_altlabel_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7023989c-b22a-475a-92d9-1de123f54184",
   "metadata": {},
   "outputs": [],
   "source": [
    "def lower_unless_isupper(s):\n",
    "    \"\"\"Lowercase, except likely acronyms.\"\"\"\n",
    "    return s.lower() if not s.isupper() else s\n",
    "\n",
    "# double-for-loop est. 1.5 hrs to complete. no thanks.\n",
    "ppl_tokens_concepts = {\n",
    "    tuple([lower_unless_isupper(i[\"token\"]) for i in ppl[\"tokens\"]]): ppl[\"concept\"]\n",
    "    for ppl in mdb.concept_label_tokens.find(primary_preflabel_filter)\n",
    "}\n",
    "pal_tokens_concepts = {\n",
    "    tuple([lower_unless_isupper(i[\"token\"]) for i in ppl[\"tokens\"]]): ppl[\"concept\"]\n",
    "    for ppl in mdb.concept_label_tokens.find(primary_altlabel_filter)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa9b658c-043f-4046-bc9b-705c3d2b73aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "mpl_matches = {}\n",
    "\n",
    "for mpl in tqdm(mdb.concept_label_tokens.find(merging_preflabel_filter), total=n_mpl):\n",
    "    mpl_tokens = tuple([lower_unless_isupper(i[\"token\"]) for i in mpl[\"tokens\"]])\n",
    "    if mpl_tokens in ppl_tokens_concepts:\n",
    "        # ordered match with preflabel\n",
    "        mpl_matches[mpl[\"concept\"]] = ppl_tokens_concepts[mpl_tokens]\n",
    "    elif mpl_tokens in pal_tokens_concepts:\n",
    "        # ordered match with altlabel\n",
    "        mpl_matches[mpl[\"concept\"]] = pal_tokens_concepts[mpl_tokens]\n",
    "    elif len(mpl_tokens) <= 8:\n",
    "        for p in permutations(mpl_tokens):\n",
    "            if p in ppl_tokens_concepts:\n",
    "                # unordered match with preflabel\n",
    "                mpl_matches[mpl[\"concept\"]] = ppl_tokens_concepts[p]    \n",
    "            if p in pal_tokens_concepts:\n",
    "                # unordered match with altlabel\n",
    "                mpl_matches[mpl[\"concept\"]] = pal_tokens_concepts[p]        \n",
    "        else:\n",
    "            if len(mpl_tokens) in range(4, 7):\n",
    "                for p in permutations(mpl_tokens, r=(len(mpl_tokens) - 1)):\n",
    "                    if p in ppl_tokens_concepts:\n",
    "                        # unordered near-match (>75% intersection) with preflabel\n",
    "                        mpl_matches[mpl[\"concept\"]] = ppl_tokens_concepts[p]\n",
    "                    elif p in pal_tokens_concepts:\n",
    "                        # unordered near-match (>75% intersection) with altlabel\n",
    "                        mpl_matches[mpl[\"concept\"]] = pal_tokens_concepts[p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ba65f92-eeff-4408-ab65-7ed13b18da40",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(mpl_matches)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88f1992c-7da1-4f3c-a2ee-658783944cdf",
   "metadata": {},
   "source": [
    "woohoo!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8a749cb-00f3-4551-964f-6cb9b18a04a2",
   "metadata": {},
   "source": [
    "Now need to generate table interface to y/n the candidate matches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33ae59a1-94a6-44bf-913f-26cd46d36a2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "(len(exact_match_preflabel_preflabel) + \n",
    " len(exact_match_preflabel_altlabel) +\n",
    " len(exact_match_altlabel_preflabel) + \n",
    " len(exact_match_altlabel_altlabel) +\n",
    " len(mpl_matches))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44e64870-81e8-415f-9649-c26ba4a70b9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for c_merging, c_primary in exact_match_preflabel_preflabel.items():\n",
    "    print(c_merging)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8c2f388-4f73-4677-b17e-bd1eb93ebaf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO get a mapping done here, in the notebook, so you can go through the whole\n",
    "#   process in-notebook. And then of course create mapping UI."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0c277f8-3853-4989-bc4a-581e94069cfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "str(t_merging.value(URIRef(\"https://openalex.org/C184779094\"), SKOS.definition))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62d22100-b16f-41e5-86ba-43b2fc7c2853",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mapping_expansion(mapping):\n",
    "    rows = []\n",
    "    for c_merging, (c_primary, nocheck) in mapping.items():\n",
    "        rows.append({\n",
    "            \"c_merging\": str(URIRef(c_merging)),\n",
    "            \"merging\": (f\"{str(t_merging.value(URIRef(c_merging), SKOS.prefLabel))} \" +\n",
    "                        f\"({str(t_merging.value(URIRef(c_merging), SKOS.definition))})\"),\n",
    "            \"action\": \"k\" if nocheck else \"\",\n",
    "            \"c_primary\": str(URIRef(c_primary)),\n",
    "            \"primary\": (f\"{str(t_primary.value(URIRef(c_primary), SKOS.prefLabel))} \" +\n",
    "                        f\"({', '.join(str(o) for o in t_primary.objects(URIRef(c_primary), SKOS.altLabel))})\"),\n",
    "        })\n",
    "    return rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff4f15eb-367c-4610-8e80-2bd123aa4599",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates = {}\n",
    "for candidates_increasing_precedence, nocheck in [\n",
    "    (mpl_matches, False),\n",
    "    (exact_match_altlabel_altlabel, False),\n",
    "    (exact_match_altlabel_preflabel, False),\n",
    "    (exact_match_preflabel_altlabel, True),\n",
    "    (exact_match_preflabel_preflabel, True),\n",
    "]:\n",
    "    for mc, pc in candidates_increasing_precedence.items():\n",
    "        candidates[str(mc)] = (str(pc), nocheck)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e47e21f-2b10-4a71-822b-2cca0584d28c",
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = sorted(mapping_expansion(candidates), key=lambda r: (r[\"action\"], r[\"merging\"].lower()))\n",
    "\n",
    "with open(\"mapping_table.csv\", \"w\") as csvfile:\n",
    "    fieldnames = [\"c_merging\", \"merging\", \"action\", \"primary\", \"c_primary\"]\n",
    "    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "    writer.writeheader()\n",
    "    for row in rows:\n",
    "        writer.writerow(row)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c810cfc1-2ac0-4da5-a8c4-efab62263fc5",
   "metadata": {},
   "source": [
    "Evaluated manually, resulted in `oax_uat_mapping_table.csv`.\n",
    "\n",
    "`n`: not acceptable; `b`: broader (ok fro upward posting); `k`: ok, equivalent.\n",
    "\n",
    "(see p353 of Hedden 2022)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "550a73fe-fa0f-4354-a78d-8481ea47776c",
   "metadata": {},
   "source": [
    "Now, to produce the mapping as RDF:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "041d307c-5d10-430a-ba61-6a3fa5b88793",
   "metadata": {},
   "outputs": [],
   "source": [
    "def printable(s):\n",
    "    return \"\".join(filter(lambda x: x in string.printable, s))\n",
    "\n",
    "g_map_oax_uat = Graph()\n",
    "\n",
    "with open('oax_uat_mapping_table.csv') as f:\n",
    "    reader = csv.DictReader(f)\n",
    "    for d in reader:\n",
    "        d = {printable(k): v for k, v in d.items()}\n",
    "        predicate = None\n",
    "        match d[\"action\"]:\n",
    "            case \"k\":\n",
    "                predicate = SKOS.exactMatch\n",
    "            case \"b\":\n",
    "                predicate = SKOS.broadMatch\n",
    "        if predicate:\n",
    "            g_map_oax_uat.add((URIRef(d[\"c_merging\"]), predicate, URIRef(d[\"c_primary\"])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5176a145-d7e5-4129-bc35-82669c2a3e6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(list(g_map_oax_uat.subject_objects(SKOS.exactMatch)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aecd8f0b-01be-4174-b988-93d1a0f6db75",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(list(g_map_oax_uat.subject_objects(SKOS.broadMatch)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0bc3fb1-78cd-45e8-b003-54759c5e3e58",
   "metadata": {},
   "source": [
    "# Query-Ground-Truth Candidate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b75a650c-e453-42c0-9bdd-6b98b33bfea0",
   "metadata": {},
   "source": [
    "from https://smd-cms.nasa.gov/wp-content/uploads/2023/11/gapanalysisreport-full-final.pdf (https://doi.org/10.1016/j.asr.2023.06.046):\n",
    "\n",
    "review on \"space weather effects of solar flares\": https://doi.org/10.1016/j.jastp.2011.04.004\n",
    "- 33 references in ADS (36 in OAX).\n",
    "\n",
    "Let's do it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0b35d83-82e6-44b3-b524-8029f885c807",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO gather these references and put in a candidate data structure\n",
    "#   for a query-ground-truth candidate\n",
    "\n",
    "review_article = mdb.oax_works_with_doi_in_ads.find_one({\"doi\": \"https://doi.org/10.1016/j.jastp.2011.04.004\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ad3c289-7e69-4039-b4eb-37aca7935f9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "review_references =  list(mdb.oax_works_with_doi_in_ads.find({\"_id\": {\"$in\": review_article[\"referenced_works\"]}}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba6adf91-5043-4983-aee7-fe979d7fab92",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(review_article[\"referenced_works\"]), len(review_references)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc72fb7c-7b08-42ed-87c4-ab2f3b6077b2",
   "metadata": {},
   "source": [
    "huh. Well, P10 is suddenly very interesting..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57fac10a-6557-4f06-8b5e-62238a8da108",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_ground_truth_candidate = {\n",
    "    \"q\": \"space weather effects of solar flares\",\n",
    "    \"relevant\": [d[\"_id\"] for d in review_references],\n",
    "    \"type\": \"binary\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb5a8297-e5b8-479d-87bc-781b73ef8d11",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_ground_truth_candidate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc743bd7-b6d2-45df-9dc5-94e8a2c08265",
   "metadata": {},
   "source": [
    "# Harmonization-Application Strategies\n",
    "\n",
    "Defining a default strategy here.\n",
    "\n",
    "search-box text --> weighted hits on retrieval (primary) scheme --> structured query against corpus using tagging (merging) scheme."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ade0821d-1155-4ae3-814f-6629f9e88805",
   "metadata": {},
   "outputs": [],
   "source": [
    "q = \"space weather effects of solar flares\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ed8cc45-6e4e-47e2-844a-2f35d1a18f2a",
   "metadata": {},
   "source": [
    "## weighted hits on retrieval (primary) scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a6f65d7-b4ec-4c53-ba9e-3d9b4d151616",
   "metadata": {},
   "outputs": [],
   "source": [
    "q_tokens = [i[\"token\"] for i in get_tokens(q)]\n",
    "q_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4e32b4b-5cc3-4e57-b107-9665a22cd258",
   "metadata": {},
   "outputs": [],
   "source": [
    "def subslices(seq):\n",
    "    \"Return all contiguous non-empty subslices of a sequence\"\n",
    "    # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D\n",
    "    slices = starmap(slice, combinations(range(len(seq) + 1), 2))\n",
    "    return map(operator.getitem, repeat(seq), slices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e03fb38-3207-4376-b3e9-301854a92bb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "q_tokens_ngrams = [tuple(ss) for ss in subslices(q_tokens)]\n",
    "q_tokens_ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87bb48f3-b225-4035-b0d5-ea8839784645",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(q_tokens_ngrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d093655-4b39-40e1-9338-3a9c7eed6ea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "primary_scheme_concept_score = defaultdict(int)\n",
    "\n",
    "for q_tokens in tqdm(q_tokens_ngrams):\n",
    "    if q_tokens in ppl_tokens_concepts:\n",
    "        # ordered match with preflabel\n",
    "        primary_scheme_concept_score[ppl_tokens_concepts[q_tokens]] += 1e6\n",
    "    elif q_tokens in pal_tokens_concepts:\n",
    "        # ordered match with altlabel\n",
    "        primary_scheme_concept_score[pal_tokens_concepts[q_tokens]] += 1e6\n",
    "    elif len(q_tokens) <= 8:\n",
    "        for p in permutations(q_tokens):\n",
    "            if p in ppl_tokens_concepts:\n",
    "                # unordered match with preflabel\n",
    "                primary_scheme_concept_score[ppl_tokens_concepts[p]] += 1e5\n",
    "            if p in pal_tokens_concepts:\n",
    "                # unordered match with altlabel\n",
    "                primary_scheme_concept_score[pal_tokens_concepts[p]] += 1e5\n",
    "        else:\n",
    "            if len(q_tokens) in range(4, 7):\n",
    "                for p in permutations(q_tokens, r=(len(q_tokens) - 1)):\n",
    "                    if p in ppl_tokens_concepts:\n",
    "                        # unordered near-match (>75% intersection) with preflabel\n",
    "                        primary_scheme_concept_score[ppl_tokens_concepts[p]] += 1e4\n",
    "                    elif p in pal_tokens_concepts:\n",
    "                        # unordered near-match (>75% intersection) with altlabel\n",
    "                        primary_scheme_concept_score[pal_tokens_concepts[p]] += 1e4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "053101bd-3f88-4475-861c-3a2a8fd33546",
   "metadata": {},
   "outputs": [],
   "source": [
    "primary_scheme_concept_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52407eab-0270-437f-8680-b5145658c039",
   "metadata": {},
   "source": [
    "## structured query against corpus using tagging (merging) scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "461f8a71-0e1a-4d88-af53-6b8645e0ec1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def harmonization_application_strategy(\n",
    "    query: str, cs_retrieval : Graph, cs_tagging: Graph, g_harmonization: Graph, score_threshold=1e4\n",
    "):\n",
    "    q_tokens = [i[\"token\"] for i in get_tokens(query)]\n",
    "    q_tokens_ngrams = [tuple(ss) for ss in subslices(q_tokens)]\n",
    "\n",
    "    retrieval_scheme_concept_score = defaultdict(int) # aka primary_scheme_concept_score\n",
    "    \n",
    "    for q_tokens in tqdm(q_tokens_ngrams):\n",
    "        if q_tokens in ppl_tokens_concepts:\n",
    "            # ordered match with preflabel\n",
    "            retrieval_scheme_concept_score[ppl_tokens_concepts[q_tokens]] += 1e6\n",
    "        elif q_tokens in pal_tokens_concepts:\n",
    "            # ordered match with altlabel\n",
    "            retrieval_scheme_concept_score[pal_tokens_concepts[q_tokens]] += 1e6\n",
    "        elif len(q_tokens) <= 8:\n",
    "            for p in permutations(q_tokens):\n",
    "                if p in ppl_tokens_concepts:\n",
    "                    # unordered match with preflabel\n",
    "                    retrieval_scheme_concept_score[ppl_tokens_concepts[p]] += 1e5\n",
    "                if p in pal_tokens_concepts:\n",
    "                    # unordered match with altlabel\n",
    "                    retrieval_scheme_concept_score[pal_tokens_concepts[p]] += 1e5\n",
    "            else:\n",
    "                if len(q_tokens) in range(4, 7):\n",
    "                    for p in permutations(q_tokens, r=(len(q_tokens) - 1)):\n",
    "                        if p in ppl_tokens_concepts:\n",
    "                            # unordered near-match (>75% intersection) with preflabel\n",
    "                            retrieval_scheme_concept_score[ppl_tokens_concepts[p]] += 1e4\n",
    "                        elif p in pal_tokens_concepts:\n",
    "                            # unordered near-match (>75% intersection) with altlabel\n",
    "                            retrieval_scheme_concept_score[pal_tokens_concepts[p]] += 1e4\n",
    "\n",
    "    retrieval_scheme_query = {\n",
    "        \"filter\": {\"concepts.id\": {\"$in\": [\n",
    "            c for c, score in retrieval_scheme_concept_score.items()\n",
    "            if score > score_threshold\n",
    "        ]}},\n",
    "    }\n",
    "    tagging_scheme_concepts = list(filter(None, [\n",
    "        (\n",
    "            str(g_harmonization.value(predicate=SKOS.exactMatch, object=URIRef(c))) or\n",
    "            str(g_harmonization.value(predicate=SKOS.broadMatch, object=URIRef(c)))\n",
    "        )\n",
    "        for c in retrieval_scheme_query[\"filter\"][\"concepts.id\"][\"$in\"]\n",
    "    ]))\n",
    "    tagging_scheme_query = {\"filter\": {\"concepts.id\": {\"$in\": tagging_scheme_concepts}}}\n",
    "    return retrieval_scheme_query, tagging_scheme_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "510603b5-bfb8-435f-973b-15d506563b94",
   "metadata": {},
   "outputs": [],
   "source": [
    "rsq, tsq = harmonization_application_strategy(\n",
    "    query=q,\n",
    "    cs_retrieval=g_uat,\n",
    "    cs_tagging=g_oax,\n",
    "    g_harmonization=g_map_oax_uat,\n",
    ")\n",
    "rsq, tsq"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60f589ce-1f8b-468d-b2f2-269ca3ebcbfb",
   "metadata": {},
   "source": [
    "# Performance-measuring jobs\n",
    "\n",
    "- PN (e.g. P10, P25, P50)\n",
    "- RN (e.g. R1000, R500, R100)\n",
    "- MRR?\n",
    "- NDCG?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84ef876d-3ec6-460a-a53b-b312a3d1d950",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = {d[\"_id\"] for d in mdb.oax_works_with_doi_in_ads.find({}, [\"_id\"])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e2c33e9-b505-427c-99f9-48ca00c5ff39",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68181516-ceea-4a18-8b40-e552bf829800",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_ground_truth_candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8390751a-1a1f-414e-8aab-96f9338e4aeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_index = mdb.oax_works_with_doi_in_ads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf500746-9298-46c2-8394-ea2af2f25b6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tsq[\"limit\"] = 100\n",
    "results = list(corpus_index.find(**tsq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f36bf5c-ae83-4c4c-9707-546df41e2dc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc53cd4c-5b90-4768-82f7-f7446dbcbcdd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}