3_snapshot.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Snapshot\n",
    "\n",
    "This notebook shows how to send a request to run an extraction based on a selection query, monitor the job until it is completed, download the generated files, and starting to analyse data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import requests as req\n",
    "import pandas as pd\n",
    "import fastavro\n",
    "import json\n",
    "from io import BytesIO\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "USER_KEY = os.getenv(\"FSS_USERKEY\")\n",
    "BASE_URL = \"https://api.dowjones.com\"\n",
    "\n",
    "REQ_HEADERS = {\n",
    "    \"user-key\": USER_KEY,\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\"\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_query = {'query': {\n",
    "        'companies': [\n",
    "            'MCROST',\n",
    "            'TSLMI',\n",
    "            'NVDCRP'\n",
    "        ],\n",
    "        'start': '2024-12-08',\n",
    "        'end': '2024-12-16',\n",
    "        'articles': True,\n",
    "        'format': 'avro'\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submit Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "snapshot_post_url = f\"{BASE_URL}/fss/extractions\"\n",
    "payload = json.dumps(s_query)\n",
    "response = req.post(snapshot_post_url, data=payload, headers=REQ_HEADERS)\n",
    "snapshot_response = response.json()\n",
    "short_id = snapshot_response['extraction_id'].split('-')[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "State: JOB_STATE_RUNNING\n",
      "Short ID: 8a5d2cfa53\n"
     ]
    }
   ],
   "source": [
    "print((f\"State: {snapshot_response['state']}\\n\"\n",
    "       f\"Short ID: {short_id}\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Job Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use only when downloading files from an old extraction\n",
    "# For articles = True use '0e2f212bf5' (109 files)\n",
    "# For articles = True (Small) use 'a236f418d8' (1 file)\n",
    "# For articles = False use 'eb26b0186e' (1 file)\n",
    "historic_short_id = ''\n",
    "if len(historic_short_id) == 10:\n",
    "    snapshot_id = f\"dj-synhub-fss-extraction-{USER_KEY}-{historic_short_id}\"\n",
    "else:\n",
    "    snapshot_id = f\"dj-synhub-fss-extraction-{USER_KEY}-{short_id}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "status_post_url = f\"{BASE_URL}/fss/extractions/{snapshot_id}\"\n",
    "response = req.get(status_post_url, headers=REQ_HEADERS)\n",
    "status_response = response.json()\n",
    "check_state = status_response['state']\n",
    "check_id = status_response['extraction_id'].split('-')[-1]\n",
    "check_total_files = len(status_response['destination'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "State: JOB_STATE_DONE\n",
      "Short ID: 8a5d2cfa53\n",
      "Total Files: 1\n"
     ]
    }
   ],
   "source": [
    "print((f\"State: {check_state}\\n\"\n",
    "       f\"Short ID: {check_id}\\n\"\n",
    "       f\"Total Files: {check_total_files}\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 files saved to ./data/8a5d2cfa53/\n"
     ]
    }
   ],
   "source": [
    "files_list = status_response['destination']\n",
    "\n",
    "for file in files_list:\n",
    "    file_resp = req.get(file, headers=REQ_HEADERS)\n",
    "    if file_resp.status_code == 200:\n",
    "        data_dir = f\"./data/{check_id}/\"\n",
    "        file_name = file.split('/')[-1]\n",
    "        file_path = os.path.join(data_dir, file_name)\n",
    "        os.makedirs(data_dir, exist_ok=True)\n",
    "        with open(file_path, 'wb') as f:\n",
    "            f.write(file_resp.content)\n",
    "\n",
    "print(f\"{check_total_files} files saved to ./data/{check_id}/\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fadev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}