Longhand_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc6cf54a",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Longhand - M. Cook - 2023\n",
    "##Takes text corpus and returns immersive visualization\n",
    "##https://github.com/Cook4986/Longhand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c6c36ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Takes Bag-of-Words and returns json dump of common nouns, Sketchfab models/uids, and relative percentage of occurance\n",
    "import spacy\n",
    "from collections import Counter\n",
    "import json\n",
    "import requests\n",
    "from requests.exceptions import RequestException\n",
    "import time\n",
    "from pathlib import Path\n",
    "\n",
    "#select language model (https://spacy.io/models)\n",
    "model = \"en_core_web_lg\"\n",
    "nlp = spacy.load(model)\n",
    "nlp.max_length = 100000000\n",
    "stopwords = nlp.Defaults.stop_words\n",
    "custom_stopwords = open(\".../longhandStopWords.txt\", \"r\")\n",
    "custom_stopwords = custom_stopwords.read()\n",
    "\n",
    "#I/O\n",
    "BoW = \"...txt\"#plaintext bag-of-words input\n",
    "output = BoW.split(\".txt\")\n",
    "stem = output[0].split(\"Corpora/\")\n",
    "title = stem[1]\n",
    "output = stem[0] + \"Objects/\"+ title + \".txt\"\n",
    "log = stem[0] + \"Objects/\"+ title + \"_log.txt\"\n",
    "\n",
    "#declarations\n",
    "model_size = 5000 #face count\n",
    "SKETCHFAB_API_URL = \"https://api.sketchfab.com/v3/search?type=models&count=24\"\n",
    "API_TOKEN = '...' #from Sketchfab - keep private\n",
    "results = 100 #target number of models, potentially limited by NER outputs\n",
    "start = time.time()\n",
    "\n",
    "#data structures\n",
    "nouns = [] #nouns in Bag-of-Words\n",
    "freqs = [] # noun appearance frequencies\n",
    "objects = {} # key = common nouns; value(s) = [relative percentage of total objects, UID, model name, URL]\n",
    "\n",
    "#parse Bag-of-Words with SpaCy\n",
    "with open(BoW, encoding=\"utf-8\") as file:\n",
    "    print(\"Tokenizing text...\")\n",
    "    print(\"\\n\")\n",
    "    iliad = file.read()\n",
    "document = nlp(iliad)\n",
    "\n",
    "#collate high-frequency nouns in target corpus\n",
    "for token in document:\n",
    "    if (token.text.lower() not in (custom_stopwords)) & (token.text.lower() not in (stopwords)) & (token.pos_ == 'NOUN') & ((len(token.lemma_)) > 2):\n",
    "            nouns.append(token.lemma_)   \n",
    "word_freq = Counter(nouns) \n",
    "common = word_freq.most_common(results)\n",
    "print(\"Common nouns named in the target corpus: \")\n",
    "print(\"\\n\")\n",
    "print(common)\n",
    "print(\"\\n\")\n",
    "\n",
    "#Sketchfab API payload function \n",
    "##From https://sketchfab.com/developers/data-api/v3/python#example-python-model\n",
    "def _get_request_payload(*, data=None, files=None, json_payload=False):\n",
    "    \"\"\"Helper method that returns the authentication token and proper content type depending on\n",
    "    whether or not we use JSON payload.\"\"\"\n",
    "    data = data or {}\n",
    "    files = files or {}\n",
    "    headers = {'Authorization': 'Token {}'.format(API_TOKEN)}\n",
    "    if json_payload:\n",
    "        headers.update({'Content-Type': 'application/json'})\n",
    "        data = json.dumps(data)\n",
    "    return {'data': data, 'files': files, 'headers': headers}\n",
    "\n",
    "#query sketchfabs with tokens and compile object dictionary with results\n",
    "for word in common:\n",
    "    key = str(word[0])\n",
    "    query = \"&q=\"+(key)+\"&tags=\"+(key)+\"&restricted=1&downloadable=true&max_face_count=\"+ str(model_size) #keys must be in name and tag\n",
    "    search_endpoint = f'{SKETCHFAB_API_URL + query}'\n",
    "    payload = _get_request_payload() \n",
    "    response = requests.get(search_endpoint, **payload)\n",
    "    data = response.json()\n",
    "    #print(data)\n",
    "    #model tag/title matching\n",
    "    for item in range(len(data['results'])):\n",
    "        url = (data['results'][item]['uri'])\n",
    "        uid = (data['results'][item]['uid'])\n",
    "        name = (str((data['results'][item]['name'])))\n",
    "        size = int(data['results'][item]['faceCount'])\n",
    "        if (key in name.lower()) & (key not in objects.keys()) & (size != 0):\n",
    "            freqs.append(word[1])\n",
    "            objects[word[0]] = [word[1]]\n",
    "            objects[key] += [name, uid, url, size]\n",
    "print(\"\\n\")      \n",
    "\n",
    "#write object (output) dictionary to disk \n",
    "with open(output, 'w') as file:\n",
    "    file.write(json.dumps(objects)) \n",
    "    file.close()\n",
    "\n",
    "#print hits and relative percentages in target corpus\n",
    "Sum = sum(freqs)\n",
    "for key,value in objects.items():\n",
    "    print(\"Model located for '\" + key + \"':\")\n",
    "    print(value[1].center(24))\n",
    "    flowt = (value[0] / Sum) * 100\n",
    "    percentage = round(flowt, 2)\n",
    "    print(\"Represents \" + str(percentage) + \"% of models identified.\")\n",
    "    if value[4] > 10000:\n",
    "                print(\"Warning: Model size exceeds 10000 faces \" + \"(\" + (str(value[4])) + \" faces)\")\n",
    "    print(\"\\n\")\n",
    "print(\"\\n\")\n",
    "print(str(len(objects)) + \" suitable models located on Sketchfab written to disk\")\n",
    "print(\"\\n\")\n",
    "\n",
    "#write log + terminate program\n",
    "end = time.time()\n",
    "print(str(end - start) + \" seconds elapsed\" )\n",
    "key = \"key\"\n",
    "with open(log, 'w') as log:\n",
    "    log.write(title)\n",
    "    log.write(\"\\n\")\n",
    "    log.write(\"Using \" + model + \" spaCy model.\")\n",
    "    log.write(\"\\n\")\n",
    "    log.write(\"max model size = \" + str(model_size) + \" polys\")\n",
    "    log.write(\"\\n\")\n",
    "    if word_freq == Counter(nouns):\n",
    "        log.write(\"parsing Nouns\")\n",
    "        log.write(\"\\n\")\n",
    "    elif word_freq == Counter(entities):\n",
    "        log.write(\"parsing Named Entities\")\n",
    "        log.write(\"\\n\")\n",
    "    log.write(str(end - start) + \" seconds elapsed\" )\n",
    "    log.write(\"\\n\")\n",
    "    log.write((str(len(objects)) + \" suitable models located on Sketchfab.\"))\n",
    "    log.write(\"\\n\")\n",
    "    log.write(\"SF query used: \" + str(query))\n",
    "    log.write(\"\\n\")\n",
    "print(\"\\n\")\n",
    "print(\"have a nice day\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58b93c85",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "##Launches Blender from terminal and initiates model download script\n",
    "!/Applications/Blender.app/Contents/MacOS/Blender --python .../Longhand_downloader.py\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9ebeaa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "##To-Do##\n",
    "#globals\n",
    "#collision detection\n",
    "#collect/deploy image covers\n",
    "#>100MB automatic decimation\n",
    "#text (key) above models\n",
    "#word2vec\n",
    "#Read plaintext \"title\" line for log and 3d text\n",
    "#Streamlit deployment\n",
    "#text-to-3D (AI)\n",
    "#package README in \"other docs\" on sketchfab\n",
    "#pop-up UI, to (checkbox) include/exclude relevant Sketchfab models\n",
    "#model placement based on focal/peripheral vision science"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}