02_Upload_polygon_mask_to_bucket.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 02. Upload polygon mask to cloud buckets\n",
    "\n",
    "This notebook uploads the polygon mask created for each Sentinel-2 and Landsat tile into a GCP (Google Cloud Platform) cloud bucket.\n",
    "\n",
    "This notebook reads in the output of the notebook [01_Create_polygon_mask.ipynb](https://github.com/ShiruiH/time-series-OFS/blob/main/01_Create_polygon_mask.ipynb), which is located in `/outputs/Sentinel2_tiles_mask` and `/outputs/Landsat_tiles_mask`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "executionInfo": {
     "elapsed": 1214,
     "status": "ok",
     "timestamp": 1723729924896,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "Z4fnKSUtRtyd",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import os\n",
    "import glob\n",
    "import datetime\n",
    "from google.cloud import storage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Authenticate Google Cloud project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 492,
     "status": "ok",
     "timestamp": 1723730053338,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "IifDqGnto8xd",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Your browser has been opened to visit:\n",
      "\n",
      "    https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=32555940559.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Flocalhost%3A8085%2F&scope=openid+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fappengine.admin+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fsqlservice.login+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcompute+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Faccounts.reauth&state=eJJQiV1Bk14GOlVhiM0uVdDd66YoOv&access_type=offline&code_challenge=aXxW7W7dUjv9eGpv70UEduDjYDZxDM878h32LptYADw&code_challenge_method=S256\n",
      "\n",
      "\n",
      "You are now logged in as [kilian.vos@dpie.nsw.gov.au].\n",
      "Your current project is [nsw-dpe-gee-tst].  You can change this setting by running:\n",
      "  $ gcloud config set project PROJECT_ID\n",
      "Updated property [core/project].\n"
     ]
    }
   ],
   "source": [
    "! gcloud auth login\n",
    "! gcloud config set project nsw-dpe-gee-tst"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set bucket and folder names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name of the cloud bucket: label-tiles\n",
      "Name of the folder: Landsat_tiles_mask\n"
     ]
    }
   ],
   "source": [
    "# Set GCS bucket name\n",
    "BUCKET_NAME = f'label-tiles'\n",
    "\n",
    "# Set satellite folder name, choose from: Sentinel2_tiles_mask; Landsat_tiles_mask\n",
    "SATELLITE_FOLDER = 'Landsat_tiles_mask'\n",
    "\n",
    "print(f'Name of the cloud bucket: {BUCKET_NAME}\\nName of the folder: {SATELLITE_FOLDER}')\n",
    "\n",
    "sat_name = SATELLITE_FOLDER.split('_')[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the folder in the bucket to store images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder 'Landsat_tiles_mask' is created in label-tiles\n"
     ]
    }
   ],
   "source": [
    "# Run this cell once to create the folder in the bucket\n",
    "bucket = storage.Client().get_bucket(BUCKET_NAME)\n",
    "blob = bucket.blob(f'{SATELLITE_FOLDER}/')\n",
    "blob.upload_from_string('')\n",
    "print(f\"Folder '{SATELLITE_FOLDER}' is created in {BUCKET_NAME}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Auxiliary function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "executionInfo": {
     "elapsed": 559,
     "status": "ok",
     "timestamp": 1723729930471,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "M1lzhmn9SXWO",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# function to upload a local file to a cloud bucket\n",
    "def upload_blob(bucket_name, source_file_name, destination_blob_name):\n",
    "    storage_client = storage.Client('')\n",
    "    bucket = storage_client.bucket(bucket_name)\n",
    "    blob = bucket.blob(destination_blob_name)\n",
    "    generation_match_precondition = 0\n",
    "    blob.upload_from_filename(source_file_name, if_generation_match=generation_match_precondition)\n",
    "    print(f'File {source_file_name} uploaded to {destination_blob_name}.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Locate the polygon mask in `/outputs` folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1345,
     "status": "ok",
     "timestamp": 1723729990699,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "BPkJWUibSZDN",
    "outputId": "37bb44c1-c7bd-4c1a-8727-5df5e99b4ef3",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c:\\Shirui_Hao\\GitHub\\time-series-OFS\\outputs\\polygon_masks\n",
      "3 Landsat tiles found: ['c:\\\\Shirui_Hao\\\\GitHub\\\\time-series-OFS\\\\outputs\\\\polygon_masks\\\\Landsat_tiles_mask\\\\LC08_091080_20231115.tif', 'c:\\\\Shirui_Hao\\\\GitHub\\\\time-series-OFS\\\\outputs\\\\polygon_masks\\\\Landsat_tiles_mask\\\\LC09_092081_20231115.tif', 'c:\\\\Shirui_Hao\\\\GitHub\\\\time-series-OFS\\\\outputs\\\\polygon_masks\\\\Landsat_tiles_mask\\\\LC09_092083_20231115.tif']\n"
     ]
    }
   ],
   "source": [
    "fp_outputs = os.path.join(os.getcwd(), 'outputs', 'polygon_masks')\n",
    "print(fp_outputs)\n",
    "# load Sentinel-2 tiles\n",
    "fp_tiles = os.path.join(fp_outputs, SATELLITE_FOLDER)\n",
    "fn_tiles = glob.glob(f'{fp_tiles}/*.tif')\n",
    "print(f'{len(fn_tiles)} {sat_name} tiles found: {fn_tiles}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load to the cloud bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 52094,
     "status": "ok",
     "timestamp": 1723730131496,
     "user": {
      "displayName": "Shirui Hao",
      "userId": "16127122271389846236"
     },
     "user_tz": -600
    },
    "id": "pT6kIlCWSbIB",
    "outputId": "ddc0fde8-dfa0-420e-b2c8-c4d7e2ff073f",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File c:\\Shirui_Hao\\GitHub\\time-series-OFS\\outputs\\polygon_masks\\Landsat_tiles_mask\\LC08_091080_20231115.tif uploaded to Landsat_tiles_mask/LC08_091080_20231115.tif.\n",
      "File c:\\Shirui_Hao\\GitHub\\time-series-OFS\\outputs\\polygon_masks\\Landsat_tiles_mask\\LC09_092081_20231115.tif uploaded to Landsat_tiles_mask/LC09_092081_20231115.tif.\n",
      "File c:\\Shirui_Hao\\GitHub\\time-series-OFS\\outputs\\polygon_masks\\Landsat_tiles_mask\\LC09_092083_20231115.tif uploaded to Landsat_tiles_mask/LC09_092083_20231115.tif.\n"
     ]
    }
   ],
   "source": [
    "for fn in fn_tiles:\n",
    "    base_name = os.path.basename(fn)\n",
    "    destination_blob = f'{SATELLITE_FOLDER}/{base_name}'\n",
    "    upload_blob(BUCKET_NAME, fn, destination_blob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***(OPTIONAL) Only run this cell if you need to delete all files in the bucket folder***\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All objects in folder 'Landsat_tiles_mask' have been deleted. Now you can re-upload files.\n"
     ]
    }
   ],
   "source": [
    "# Delete all objects in the folder if needed\n",
    "bucket = storage.Client().get_bucket(BUCKET_NAME)\n",
    "# List all objects in the folder\n",
    "blobs = bucket.list_blobs(prefix=SATELLITE_FOLDER)\n",
    "\n",
    "# Delete all objects in the folder\n",
    "for blob in blobs:\n",
    "    blob.delete()\n",
    "\n",
    "print(f\"All objects in folder '{SATELLITE_FOLDER}' have been deleted. Now you can re-create the folder and re-upload files in the bucket.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyPW46EFKwwBaoadxzo+/Xpt",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}