aws_s3_gh.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9310386",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Install once only - python official library from AWS\n",
    "# !pip install boto3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73f7f808",
   "metadata": {},
   "source": [
    "### https://boto3.amazonaws.com/v1/documentation/api/latest/index.html"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b206094",
   "metadata": {},
   "source": [
    "### https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6cae5dd3",
   "metadata": {},
   "source": [
    "### define configuration "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00f099e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "aws_access_key_id = <Your Access Key>\n",
    "aws_secret_access_key = <Your secret access key>\n",
    "myregion= <your region>\n",
    "mybucket = <your bucket name>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64ba582a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05b422da",
   "metadata": {},
   "source": [
    "### List all my buckets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53a6a720",
   "metadata": {},
   "outputs": [],
   "source": [
    "def List_My_Buckets( aws_access,aws_secret, aws_region):\n",
    "    s3_client = boto3.client('s3',aws_access_key_id=aws_access, aws_secret_access_key=aws_secret, region_name =aws_region )\n",
    "    response = s3_client.list_buckets()\n",
    "    # Output the bucket names\n",
    "    print('Existing buckets:')\n",
    "    for bucket in response['Buckets']:\n",
    "        print(f'  {bucket[\"Name\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d609436b",
   "metadata": {},
   "outputs": [],
   "source": [
    "List_My_Buckets(aws_access_key_id,aws_secret_access_key,myregion)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efa5f4b3",
   "metadata": {},
   "source": [
    "### create a new bucket with location constraint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d14eff4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_bucket(aws_access,aws_secret, aws_region, bucket_name):\n",
    "    s3_client = boto3.client('s3',aws_access_key_id=aws_access, aws_secret_access_key=aws_secret, region_name =aws_region)\n",
    "    location = {'LocationConstraint': aws_region}\n",
    "    s3_client.create_bucket(Bucket=bucket_name,CreateBucketConfiguration=location)\n",
    "    return True\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04ee0868",
   "metadata": {},
   "outputs": [],
   "source": [
    "create_bucket(aws_access_key_id,aws_secret_access_key,myregion,mybucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3aac693a",
   "metadata": {},
   "source": [
    "### Upload a new file to the specified buket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ebe98ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "def upload_file(aws_access,aws_secret, aws_region, bucket_name, file_toupload,object_name=None):\n",
    "    s3_client = boto3.client('s3',aws_access_key_id=aws_access, aws_secret_access_key=aws_secret, region_name =aws_region)\n",
    "    # If S3 object_name was not specified, use file_toupload\n",
    "    if object_name is None:\n",
    "        object_name = os.path.basename(file_toupload)\n",
    "    try:\n",
    "        response = s3_client.upload_file(file_toupload, bucket_name, object_name)\n",
    "    except ClientError as e:\n",
    "        logging.error(e)\n",
    "        return False\n",
    "    return True\n",
    " \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9f55392",
   "metadata": {},
   "outputs": [],
   "source": [
    "upload_file(aws_access_key_id,aws_secret_access_key,myregion,mybucket,<upload_filepath>)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79c7d594",
   "metadata": {},
   "source": [
    "### Download a file/object from bucket - Method 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bd9f00d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_file(aws_access,aws_secret, aws_region, bucket_name, file_todownload,object_name):\n",
    "    s3_client = boto3.client('s3',aws_access_key_id=aws_access, aws_secret_access_key=aws_secret, region_name =aws_region)\n",
    "    try:\n",
    "        with open(file_todownload, 'wb') as f:\n",
    "            s3_client.download_fileobj(bucket_name, object_name, f)\n",
    "    except ClientError as e:\n",
    "        logging.error(e)\n",
    "        return False\n",
    "    return True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81e04b6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "download_file(aws_access_key_id,aws_secret_access_key,myregion,mybucket,<download_filepath>)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "202535b5",
   "metadata": {},
   "source": [
    "### Download a file/object from bucket - Method 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9be3cbcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_file_2(aws_access,aws_secret, aws_region, bucket_name, file_todownload,object_name):\n",
    "    s3_client = boto3.client('s3',aws_access_key_id=aws_access, aws_secret_access_key=aws_secret, region_name =aws_region)\n",
    "    try:\n",
    "        s3_client.download_file(Bucket=bucket_name, Key= object_name, Filename = <upload_filepath> + object_name)\n",
    "    except ClientError as e:\n",
    "        logging.error(e)\n",
    "        return False\n",
    "    return True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a382ae0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "download_file_2(aws_access_key_id,aws_secret_access_key,myregion,mybucket,<download_filepath>)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac1b1250",
   "metadata": {},
   "source": [
    "### List objects in a specified bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24a4ab8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_object_v2(aws_access,aws_secret, aws_region, bucket_name):\n",
    "    s3_client = boto3.client('s3',aws_access_key_id=aws_access, aws_secret_access_key=aws_secret, region_name =aws_region)\n",
    "    try:\n",
    "        response = s3_client.list_objects_v2(Bucket=bucket_name)\n",
    "        for content in response['Contents']:\n",
    "            print(f'  {content[\"Key\"]}, {content[\"Size\"]/1024} KB')\n",
    "    except ClientError as e:\n",
    "        logging.error(e)\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9347ff2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_object_v2(aws_access_key_id,aws_secret_access_key,myregion,mybucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c171d67",
   "metadata": {},
   "source": [
    "### Delete a specific file from a bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30432dd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def delete_object(aws_access,aws_secret, aws_region, bucket_name, file_todelete):\n",
    "    s3_client = boto3.client('s3',aws_access_key_id=aws_access, aws_secret_access_key=aws_secret, region_name =aws_region)\n",
    "    try:\n",
    "        response = s3_client.delete_object(Bucket=bucket_name, Key =file_todelete)\n",
    "        print(response)\n",
    "        print(f'  {response[\"ResponseMetadata\"][\"HTTPStatusCode\"]}')\n",
    "    except ClientError as e:\n",
    "        logging.error(e)\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8c99053",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "delete_object(aws_access_key_id,aws_secret_access_key,myregion,mybucket,<download_filepath>)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}