object_storage_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f8e71d2c-0c4c-4f31-a3f5-8e88f0596067",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-06T13:30:27.700765Z",
     "iopub.status.busy": "2023-05-06T13:30:27.700387Z",
     "iopub.status.idle": "2023-05-06T13:30:27.703781Z",
     "shell.execute_reply": "2023-05-06T13:30:27.703075Z",
     "shell.execute_reply.started": "2023-05-06T13:30:27.700740Z"
    },
    "tags": []
   },
   "source": [
    "# Working with files on object storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "655a6389-e21e-4755-9cce-ffe3333d0254",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import fsspec\n",
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c586954-250d-483b-951e-d39a8dc31214",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-05-06T13:30:09.523433Z",
     "iopub.status.busy": "2023-05-06T13:30:09.523062Z",
     "iopub.status.idle": "2023-05-06T13:30:09.526477Z",
     "shell.execute_reply": "2023-05-06T13:30:09.525879Z",
     "shell.execute_reply.started": "2023-05-06T13:30:09.523410Z"
    },
    "tags": []
   },
   "source": [
    "## Download a netcdf file from [xarray examples](https://github.com/pydata/xarray-data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3247f161-be77-4d45-83aa-548814ffaa99",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "! wget  https://github.com/pydata/xarray-data/raw/master/air_temperature.nc -O air_temperature.nc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47945d26-cce3-4640-b29a-087fce030bc5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "nc_file = 'air_temperature.nc'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7d9f50d-51fa-4dbb-b807-2b39da4ac9f9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ds = xr.open_dataset(nc_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1be57925-0e4b-4443-abc2-367739169193",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fs_s3 = fsspec.filesystem('s3', profile='esip', \n",
    "                          client_kwargs={'endpoint_url':'https://ncsa.osn.xsede.org'},\n",
    "                          skip_instance_cache=True, use_listings_cache=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73a90dea-4dce-43e1-b3a7-a16a5b5d7b1a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "bucket = 'esip'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eff45a97-7e0a-4a94-853e-3c7be7f5d5c2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fs_s3.ls(bucket)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbfdfdd4-3264-4d2a-8851-274978659205",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "username = os.environ['JUPYTERHUB_USER'].split('@')[0]\n",
    "print(username)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3ecbe70-cb4f-408a-a2d6-f554080de8a7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cloud_file = f'{bucket}/workshop/{username}/{nc_file}'\n",
    "cloud_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd4034c-3460-4f02-881c-582246ed0d82",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fs_s3.upload(nc_file, cloud_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28316b58-5fd7-475b-b1b2-751987d3ccc9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fs_s3.ls(f'{bucket}/workshop/{username}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67360bf2-3542-44e1-b95a-b5b64e728d7d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fs_s3.info(cloud_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e70519d-2ea0-4be7-84e9-14d95c1caa8a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ds = xr.open_dataset(fs_s3.open(cloud_file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "011eeb3c-9f2f-48b9-a002-0e8c3b8576de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc86ae6d-f184-46e4-9b62-05f26c452e85",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "users-users-pangeo",
   "language": "python",
   "name": "conda-env-users-users-pangeo-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}