ParsingMetadataMD2JSON.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8737b6b4-a572-46b2-bd22-b70cfccb501d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import re\n",
    "import json\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8373a7f1-49fe-4279-8a35-fadb413acde0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee24ff0d-07b7-4b21-8f38-211783deb980",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Usage:\n",
    "    ./ParsingMetadataMD2JSON [options] <input file>...\n",
    "Options:\n",
    "    --help            Show this screen.\n",
    "\"\"\"\n",
    "\n",
    "def main() -> int:\n",
    "    # Reads in a MarkDown M_datafile.md parsing every section\n",
    "    # into a separate JSON M_datafile.json file\n",
    "\n",
    "    # Get command line arguments for file\n",
    "    args = sys.argv[1:]\n",
    "    \n",
    "    # Get the file to convert\n",
    "    try:\n",
    "        md_path = Path(args[0])\n",
    "        if not md_path.exists():\n",
    "            raise FileNotFoundError(f\"{md_path} does not exist.\")\n",
    "    except IndexError:\n",
    "        print(\"Usage:\")\n",
    "        print(\"./ParsingMetadataMD2JSON [options] <input file>...\")\n",
    "        print(\"Options:\")\n",
    "        print(\"--help            Show this screen.\")\n",
    "        print()\n",
    "        #raise IndexError(f\"Expected file name as first argument\")\n",
    "        print(f\"Expected file name as first argument...EXITING...\")\n",
    "        sys.exit(1) # something went wrong\n",
    "    \n",
    "    with open(str(md_path), \"r\", encoding=\"utf-8\") as f:\n",
    "        contents = f.read()#.replace(\"\\r\", \"\") # Remove Windows style line breaks\n",
    "\n",
    "    parsed_data = get_headings(contents)\n",
    "    \n",
    "    dictionaryJSON = getJSON(parsed_data)\n",
    "    \n",
    "    # Replace file extension from .md to .json and dump .json\n",
    "    with open(Path(args[0][:-len(\".md\")] + \".json\"), 'w', encoding='utf-8') as f:\n",
    "        json.dump(dictionaryJSON, f, ensure_ascii=False, indent=4)\n",
    "    \n",
    "    print(\"SUCCESS: \" + str(Path(args[0]))+\" parsed to \" + str(Path(args[0][:-len(\".md\")] + \".json\")) )\n",
    "\n",
    "    return 0 # success\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "650d3f63-758a-49b6-9663-338dcc47f653",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_headings(content):\n",
    "    # this fct split the input file in sections divided by empty lines\n",
    "    # using regular expression with heading pound # in markdown\n",
    "    pattern = r'(?:^|\\t)*#+\\s*([^#]+)\\s*\\r?'\n",
    "    matches = re.findall(pattern, content)\n",
    "    \n",
    "    heading_values = []\n",
    "    for match in matches:\n",
    "        heading, value = match.split('\\n', maxsplit=1)\n",
    "        heading_values.append({heading: value})\n",
    "        \n",
    "    return heading_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f9f69886-9c77-4017-90bb-6f9e9ee82d42",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getJSON(contentdictionary):\n",
    "    # this fct converts the list to the appropriate JSON style\n",
    "    # change, amend, or replace to your needs\n",
    "\n",
    "    # Initialize variables to store the JSON data\n",
    "    M_title = {}\n",
    "    M_creators = [] # can be a list\n",
    "    M_publisher = {}\n",
    "    M_contributors = [] # can be a list\n",
    "    M_description = {}\n",
    "    M_subjects = [] # a list of keywords\n",
    "    M_date = {}\n",
    "    M_language = [] # a list of languages\n",
    "    M_formats = [] # a list of file formats\n",
    "    M_type = {}\n",
    "    M_coverage = {}\n",
    "    M_identifier = {}\n",
    "    M_methods = []\n",
    "    M_sources = []\n",
    "    M_relations = []\n",
    "    M_rights = []\n",
    "    \n",
    "    \n",
    "    for index, entry in enumerate(contentdictionary):\n",
    "            for heading, value in entry.items():\n",
    "                # parse every heading\n",
    "                if heading == \"Title\":\n",
    "                    M_title[\"Title\"] = value.strip()\n",
    "                \n",
    "                elif heading == \"Creator\":\n",
    "                    name = value\n",
    "                    creator = {}\n",
    "                    creator[\"Name\"] = name.strip()\n",
    "                    M_creators.append(creator)\n",
    "                \n",
    "                elif heading == \"Creator.ORCID\":\n",
    "                    creator = M_creators[-1]\n",
    "                    creator[\"ORCID\"] = value.strip()\n",
    "                    \n",
    "                elif heading == \"Creator.Email\":\n",
    "                    creator = M_creators[-1]\n",
    "                    creator[\"Email\"] = value.strip()\n",
    "                \n",
    "                elif heading == \"Publisher\":\n",
    "                    M_publisher[\"Name\"] = \"<p>\" + value.strip().replace(\"  \\n\", \"<br>\") + \"</p>\"\n",
    "                \n",
    "                elif heading == \"Contributor\":\n",
    "                    name = value\n",
    "                    contributor = {}\n",
    "                    contributor[\"Name\"] = name.strip()\n",
    "                    M_contributors.append(contributor)\n",
    "                \n",
    "                elif heading == \"Contributor.ORCID\":\n",
    "                    contributor = M_contributors[-1]\n",
    "                    contributor[\"ORCID\"] = value.strip()\n",
    "                \n",
    "                elif heading == \"Contributor.Email\":\n",
    "                    contributor = M_contributors[-1]\n",
    "                    contributor[\"Email\"] = value.strip()\n",
    "                \n",
    "                elif heading == \"Description\":\n",
    "                    M_description[\"text\"] = \"<p>\" + value.strip().replace(\"  \\n\", \"<br>\") + \"</p>\"\n",
    "                \n",
    "                elif heading == \"Subject\":\n",
    "                    text = value\n",
    "                    delimiters = ['*','']\n",
    "                    words = [word.strip() for word in value.strip().split('*')]\n",
    "                    result = [word for word in words if word not in ['*','']]\n",
    "                    M_subjects.append(result)\n",
    "                \n",
    "                if heading == \"Date\":\n",
    "                    M_date[\"Date\"] = value.strip()\n",
    "                    \n",
    "                if heading == \"Language\":\n",
    "                    text = value\n",
    "                    delimiters = [';','']\n",
    "                    words = [word.strip() for word in value.strip().split(';')]\n",
    "                    result = [word for word in words if word not in [';','']]\n",
    "                    M_language.append(result)\n",
    "                \n",
    "                if heading == \"Format\":\n",
    "                    text = value\n",
    "                    delimiters = ['*','']\n",
    "                    words = [word.strip() for word in value.strip().split('*')]\n",
    "                    result = [word for word in words if word not in ['*','']]\n",
    "                    M_formats.append(result)\n",
    "                \n",
    "                if heading == \"Type\":\n",
    "                    M_type[\"Type\"] = value.strip()\n",
    "                \n",
    "                if heading == \"Coverage\":\n",
    "                    M_coverage[\"Coverage\"] = value.strip()\n",
    "                \n",
    "                if heading == \"Identifier\":\n",
    "                    M_identifier[\"Identifier\"] = value.strip()\n",
    "                \n",
    "                if heading == \"Method\":\n",
    "                    text = value\n",
    "                    lines = [word.strip() for word in text.strip().split('\\n')]\n",
    "                    for line in lines:\n",
    "                        tokens = [word.strip() for word in line.strip().split()]\n",
    "                        method = {}\n",
    "                        method[\"identifier_source\"]=tokens[0]\n",
    "                        method[\"relation_type\"]=tokens[1]\n",
    "                        method[\"identifier_destination\"]=tokens[2]\n",
    "                        M_methods.append(method)\n",
    "                \n",
    "                if heading == \"Source\":\n",
    "                    text = value\n",
    "                    lines = [word.strip() for word in text.strip().split('\\n')]\n",
    "                    for line in lines:\n",
    "                        tokens = [word.strip() for word in line.strip().split()]\n",
    "                        source = {}\n",
    "                        source[\"identifier_source\"]=tokens[0]\n",
    "                        source[\"relation_type\"]=tokens[1]\n",
    "                        source[\"identifier_destination\"]=tokens[2]\n",
    "                        M_sources.append(source)\n",
    "                \n",
    "                if heading == \"Relation\":\n",
    "                    text = value\n",
    "                    lines = [word.strip() for word in text.strip().split('\\n')]\n",
    "                    for line in lines:\n",
    "                        tokens = [word.strip() for word in line.strip().split()]\n",
    "                        relation = {}\n",
    "                        relation[\"identifier_source\"]=tokens[0]\n",
    "                        relation[\"relation_type\"]=tokens[1]\n",
    "                        relation[\"identifier_destination\"]=tokens[2]\n",
    "                        M_relations.append(relation)\n",
    "                        \n",
    "                if heading == \"Rights\":\n",
    "                    text = value\n",
    "                    lines = [word.strip() for word in text.strip().split('\\n')]\n",
    "                    for line in lines:\n",
    "                        tokens = [word.strip() for word in line.strip().split()]\n",
    "                        right = {}\n",
    "                        right[\"identifier_source\"]=tokens[0]\n",
    "                        right[\"relation_type\"]=tokens[1]\n",
    "                        right[\"identifier_destination\"]=tokens[2]\n",
    "                        M_rights.append(right)\n",
    "\n",
    "    \n",
    "    # Return the resulting JSON object\n",
    "    json_object = { \"metadata\":\n",
    "                   {\n",
    "                    \"title\": M_title[\"Title\"],\n",
    "                    \"creators\": M_creators,\n",
    "                    \"publisher\": M_publisher[\"Name\"],\n",
    "                    \"contributor\": M_contributors,\n",
    "                    \"description\": M_description[\"text\"],\n",
    "                    \"subject\": M_subjects,\n",
    "                    \"date\": M_date[\"Date\"],\n",
    "                    \"language\": M_language,\n",
    "                    \"format\": M_formats,\n",
    "                    \"type\": M_type[\"Type\"],\n",
    "                    \"coverage\": M_coverage[\"Coverage\"],\n",
    "                    \"identifier\": M_identifier[\"Identifier\"],\n",
    "                    \"method\": M_methods,\n",
    "                    \"source\": M_sources,\n",
    "                    \"relation\": M_relations,\n",
    "                    \"rights\": M_rights,\n",
    "                    }\n",
    "    }\n",
    "    \n",
    "    #print(json.dumps(json_object))\n",
    "    return json_object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "32674c9d-4060-4fcb-bfe0-97b922d64227",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SUCCESS: M_Dataset_README_Example.md parsed to M_Dataset_README_Example.json\n"
     ]
    }
   ],
   "source": [
    "%%python3 ParsingMetadataMD2JSON.py ./M_Dataset_README_Example.md\n",
    "# cell magic to fake command line execution\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    sys.exit(main())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0febd75-8b15-4077-b594-4de4747307bc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}