USDA_Apr_23.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cpatiffanynguyen/usda/blob/main/USDA_Apr_23.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install PyPDF2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5KW3OZZZTvRA",
        "outputId": "047c4524-4d2b-4fc2-99d6-ea21ad11753c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting PyPDF2\n",
            "  Downloading pypdf2-3.0.1-py3-none-any.whl (232 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m232.6/232.6 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: PyPDF2\n",
            "Successfully installed PyPDF2-3.0.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T0KdPiMOTzt4",
        "outputId": "fdf51508-22fb-4fe9-d3bd-e6ea40b17824"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YvVZRc5YuyGq"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "from requests.exceptions import ConnectionError\n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "from PyPDF2 import PdfReader\n",
        "import io"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#List of PDF URLs\n"
      ],
      "metadata": {
        "id": "fVcDehYfBxeq"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Colorado"
      ],
      "metadata": {
        "id": "m6-WQl7rJlJS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pdf_urls_string = \"\"\"https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2017/04/sam-irr-guide.pdf\n",
        "https://bookstore.ksre.ksu.edu/pubs/MF3523.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/02/Avian-Health-Deliverable-Draft98.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/02/Dead-Bird-Now-What-Infographic-w-Check-List1.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2016/07/forage-guide.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2018/07/GrazingGuide.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/11/FactSheet_E-Colorado-Grassland-Birds-Grazing_12-2022.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2017/08/LivestockManagementintheMountains2017.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/03/Composting-Manual-2023_Web-FINAL.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2018/07/manure-mgt.pdf\n",
        "https://extension.colostate.edu/docs/pubs/crops/xcm178.pdf\n",
        "https://extension.colostate.edu/docs/pubs/crops/xcm178.pdf\n",
        "https://extension.colostate.edu/docs/pubs/crops/xcm178.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://webdoc.agsci.colostate.edu/soiltestinglab/soilsample_horticulture.pdf\n",
        "https://webdoc.agsci.colostate.edu/soiltestinglab/Forage_online_submittal_form.pdf\n",
        "https://agsci.colostate.edu/soiltestinglab/wp-content/uploads/sites/98/2022/01/Soil-Submittal-Form-for-Horticulture-Fillable.pdf\n",
        "https://agsci.colostate.edu/soiltestinglab/wp-content/uploads/sites/98/2022/01/Submittal-Form-for-Water-Fillable.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://nativebeewatch.files.wordpress.com/2018/03/fieldguide_12march2018_lowrez.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/03/BeginnerBeeFieldGuide_LowRezWeb_28Feb2022.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://nativebeewatch.files.wordpress.com/2018/03/fieldguide_12march2018_lowrez.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/03/BeginnerBeeFieldGuide_LowRezWeb_28Feb2022.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://nativebeewatch.files.wordpress.com/2018/03/fieldguide_12march2018_lowrez.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2016/07/forage-guide.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2018/07/GrazingGuide.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/FrontRangeSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/FrontRange.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/MountainsSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/Mountains.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/PrairieSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/Prairie.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/SESm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/SE.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/WestSlopeSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/WestSlope.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/sources-native-plants-2022.pdf\n",
        "https://extension.colostate.edu/docs/pubs/crops/xcm179.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09329.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09341.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/07/Roasting-Chile-Peppers.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09380.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09377.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09367.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09386.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09334.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09318.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/Defenders-of-Wildlife-Conflict-Reduction.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/livestock_and_wolves.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/PC-Tool-Resource-Guide.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/ReducingConflict_WLA-Guide_low-res-1.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/Tool-Kit-Grizzly-Bear-Conflict-Reduction-2017.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/Wilson-et-al.-2017-Learning-to-live-with-wolves-Blackfoot-Valley.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/WolfResourcesGuide.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/bro-livestock-protection-dog-public.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/A-Wyoming-Landowners-Handbook-to-Fences-and-Wildlife_2nd-Edition_-lo-res.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/FladryManual.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/installing-turbo-fladry-guide-ib.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/carcus-mgt.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/Review-of-livestock-compensation-programs-052620.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Breck-Wolf-behavior-and-mgt.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Collins-WLA_RCPP_Extension.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Collins-WLA_WWC_4Cs.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-CPW-Wolf-Livestock-Compensation.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Gary-Cattle-records.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Hoag-Economic-value-of-wolves.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Stanko-Rancher-Community-dialogs.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Vail-Rocky-Mt-Wolf-Project.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Wilson-Blackfoot-Challenge.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2017/04/sam-irr-guide.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/10/Fire-Blight_2.907-2__OConnor-Zentz_10-22.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/07001.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/FrontRangeSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/FrontRange.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/MountainsSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/Mountains.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/PrairieSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/Prairie.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/SESm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/SE.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/WestSlopeSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/WestSlope.pdf\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "tn_NUpE_FyPF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "\n",
        "def download_pdfs(pdf_urls):\n",
        "  \"\"\"Downloads PDFs from a list of URLs and saves them locally.\"\"\"\n",
        "  for url in pdf_urls:\n",
        "    response = requests.get(url)\n",
        "    filename = url.split('/')[-1]  # Extract filename from URL\n",
        "    with open(filename, 'wb') as f:\n",
        "      f.write(response.content)\n",
        "\n",
        "pdf_urls = pdf_urls_string.splitlines()  # Split string into list\n",
        "download_pdfs(pdf_urls)\n"
      ],
      "metadata": {
        "id": "CornsGwikcqg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from PyPDF2 import PdfReader\n",
        "\n",
        "def extract_pdf_text(pdf_path):\n",
        "  \"\"\"Extracts text content from a PDF file.\"\"\"\n",
        "  with open(pdf_path, 'rb') as pdf_file:\n",
        "    pdf_reader = PdfReader(pdf_file)\n",
        "    text = \"\"\n",
        "    for page in pdf_reader.pages:\n",
        "      text += page.extract_text()\n",
        "  return text\n",
        "\n",
        "# Assuming a downloaded PDF named 'ihm-bee.pdf'\n",
        "pdf_text = extract_pdf_text('ihm-bee.pdf')\n",
        "# Process or analyze the extracted text (pdf_text)\n"
      ],
      "metadata": {
        "id": "HAFImZ2_kwgV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from urllib.request import urlretrieve  # For downloading PDFs\n",
        "from PyPDF2 import PdfReader  # For extracting text\n",
        "\n",
        "def download_and_extract_text(pdf_url, filename='ihm-bee.txt'):\n",
        "  \"\"\"Downloads a PDF, extracts text, and saves it to a TXT file.\"\"\"\n",
        "  urlretrieve(pdf_url, filename)  # Download PDF (replace filename if needed)\n",
        "  with open(filename, 'rb') as pdf_file:\n",
        "    pdf_reader = PdfReader(pdf_file)\n",
        "    text = \"\"\n",
        "    for page in pdf_reader.pages:\n",
        "      text += page.extract_text()\n",
        "  return text\n",
        "\n",
        "pdf_url = \"https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\"\n",
        "ihm_bee_text = download_and_extract_text(pdf_url)\n",
        "\n",
        "# Save the extracted text to a separate TXT file (optional):\n",
        "with open('ihm_bee_content.txt', 'w') as f:\n",
        "  f.write(ihm_bee_text)\n"
      ],
      "metadata": {
        "id": "GAUDC-lvk93h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter  # For counting keyword occurrences\n",
        "\n",
        "def analyze_keywords(text, top_n=10):\n",
        "  \"\"\"Analyzes keywords in the text and prints the top 'top_n'.\"\"\"\n",
        "  words = [word.lower() for word in text.split() if word.isalnum()]  # Preprocess text (lowercase, alphanumeric)\n",
        "  keyword_counts = Counter(words)\n",
        "  top_keywords = keyword_counts.most_common(top_n)\n",
        "  print(\"Top\", top_n, \"Keywords:\")\n",
        "  for keyword, count in top_keywords:\n",
        "    print(f\"- '{keyword}': {count} occurrences\")\n",
        "\n",
        "analyze_keywords(ihm_bee_text)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bq5xZy7DlAb6",
        "outputId": "b38bea1d-3f5a-401e-e967-91fb81e5b80e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Top 10 Keywords:\n",
            "- 'the': 724 occurrences\n",
            "- 'and': 434 occurrences\n",
            "- 'to': 412 occurrences\n",
            "- 'of': 348 occurrences\n",
            "- 'in': 282 occurrences\n",
            "- 'a': 253 occurrences\n",
            "- 'is': 193 occurrences\n",
            "- 'or': 153 occurrences\n",
            "- 'are': 151 occurrences\n",
            "- 'bees': 146 occurrences\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "import gspread\n",
        "from google.auth import default\n",
        "creds, _ = default()\n",
        "\n",
        "gc = gspread.authorize(creds)\n",
        "\n",
        "sh = gc.create('A new spreadsheet')\n",
        "\n",
        "# Open our new sheet and add some data.\n",
        "worksheet = gc.open('A new spreadsheet').sheet1\n",
        "\n",
        "cell_list = worksheet.range('A1:C2')\n",
        "\n",
        "import random\n",
        "for cell in cell_list:\n",
        "  cell.value = random.randint(1, 10)\n",
        "\n",
        "worksheet.update_cells(cell_list)\n",
        "# Go to https://sheets.google.com to see your new spreadsheet."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nB1E0oUJVHoR",
        "outputId": "249ce70a-7620-41f6-ec74-3c3fb49bdf3a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'spreadsheetId': '1Tp-veBUjCwnzg_JwvVNwX-z_LhU_WZgIWluaGInWxks',\n",
              " 'updatedRange': 'Sheet1!A1:C2',\n",
              " 'updatedRows': 2,\n",
              " 'updatedColumns': 3,\n",
              " 'updatedCells': 6}"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "After executing the cell above, a new file named 'Sample file.txt' will appear in your [drive.google.com](https://drive.google.com/) file list."
      ],
      "metadata": {
        "id": "sm8ByiEiVZpA"
      }
    }
  ]
}