dwc-cv/generate_biome_cv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import csv\n",
    "\n",
    "endpoint = 'http://sparql.hegroup.org/sparql'\n",
    "accept_media_type = 'application/json'\n",
    "# Replace this value with your own user agent header string\n",
    "user_agent_header = 'twdgCvTool/0.1 (mailto:steve.baskauf@vanderbilt.edu)'\n",
    "\n",
    "# The column headers of the output will be in the order in which they occur in the dict (usually the order they were added)\n",
    "def write_dicts_to_csv(table, filename, fieldnames):\n",
    "    with open(filename, 'w', newline='', encoding='utf-8') as csv_file_object:\n",
    "        writer = csv.DictWriter(csv_file_object, fieldnames=fieldnames)\n",
    "        writer.writeheader()\n",
    "        for row in table:\n",
    "            writer.writerow(row)\n",
    "\n",
    "\n",
    "def generate_header_dictionary(accept_media_type,user_agent_header):\n",
    "    request_header_dictionary = {\n",
    "        'Accept' : accept_media_type,\n",
    "#        'Content-Type': 'application/sparql-query',\n",
    "        'Content-Type': 'application/x-www-form-urlencoded',\n",
    "        'User-Agent': user_agent_header\n",
    "    }\n",
    "    return request_header_dictionary\n",
    "\n",
    "# The following function requires the request header generated above\n",
    "sparql_request_header = generate_header_dictionary(accept_media_type,user_agent_header)\n",
    "# The query is a valid SPARQL query string\n",
    "\n",
    "# Sends a query to the query service endpoint. \n",
    "# NOTE: request_header and endpoint are global variables defined earlier in the script\n",
    "def send_sparql_query(query_string):\n",
    "    # You can delete the two print statements if the queries are short. However, for large/long queries,\n",
    "    # it's good to let the user know what's going on.\n",
    "    print('querying SPARQL endpoint to acquire item metadata')\n",
    "    #response = requests.post(endpoint, data=query_string.encode('utf-8'), headers=sparql_request_header)\n",
    "    response = requests.post(endpoint, data=dict(query=query_string), headers=sparql_request_header) # use URL-encoded method\n",
    "    #print(response.text) # uncomment to view the raw response, e.g. if you are getting an error\n",
    "    data = response.json()\n",
    "\n",
    "    # Extract the values from the response JSON\n",
    "    results = data['results']['bindings']\n",
    "    \n",
    "    print('done retrieving data')\n",
    "    # print(json.dumps(results, indent=2))\n",
    "    return results\n",
    "\n",
    "query_string = '''prefix xsd: <http://www.w3.org/2001/XMLSchema#>\n",
    "prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n",
    "prefix owl: <http://www.w3.org/2002/07/owl#> \n",
    "SELECT distinct ?subclass ?plainLabel\n",
    "WHERE { \n",
    "bind(<http://purl.obolibrary.org/obo/ENVO_00000428> as ?rootIri)\n",
    "?subclass rdfs:subClassOf+ ?rootIri.\n",
    "?subclass rdfs:label ?label.\n",
    "filter(contains(str(?subclass), \"ENVO\"))\n",
    "bind(lcase(str(?label)) as ?plainLabel)\n",
    "minus {?subclass owl:deprecated \"true\"^^xsd:boolean.}\n",
    "}'''\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = send_sparql_query(query_string)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract IRI and generate lower camelCase controlled value string\n",
    "processed_list = []\n",
    "unique_value_string_test_list = []\n",
    "unique_value_iri_test_list = []\n",
    "for value in result:\n",
    "    iri = value['subclass']['value']\n",
    "    plain_label = value['plainLabel']['value']\n",
    "    pieces = plain_label.split(' ')\n",
    "    # Remove final \"biome\" if it is there\n",
    "    if pieces[len(pieces)-1] == 'biome':\n",
    "        pieces.remove('biome')\n",
    "    # Turn label into lower camelCase\n",
    "    cv_string = pieces[0] # first piece remains lower case\n",
    "    for piece in pieces[1:]:\n",
    "        cv_string += piece.title() # subsequent pieces capitalized\n",
    "    processed_list.append({'iri': iri, 'cv_string': cv_string})\n",
    "    unique_value_string_test_list.append(cv_string)\n",
    "    unique_value_iri_test_list.append(cv_string)\n",
    "\n",
    "unique_value_string_test_list.sort()\n",
    "unique_value_iri_test_list.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check controlled value strings and IRIs to make sure they are actually unique\n",
    "# Compare list to list with removed duplicates to make sure they are the same\n",
    "no_duplicate_strings = list(set(unique_value_string_test_list))\n",
    "no_duplicate_strings.sort()\n",
    "no_duplicate_iris = list(set(unique_value_iri_test_list))\n",
    "no_duplicate_iris.sort()\n",
    "\n",
    "print('No duplicate strings:', no_duplicate_strings == unique_value_string_test_list)\n",
    "print('No duplicate IRIs:', no_duplicate_iris == unique_value_iri_test_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write results to CSV\n",
    "fieldnames = processed_list[0].keys()\n",
    "filename = 'biome_cv_values.csv'\n",
    "write_dicts_to_csv(processed_list, filename, fieldnames)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}