aws_dataset_scores_collection (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get scores from sagemaker aws private workforces\n",
    "\n",
    "from boto3 import client\n",
    "import boto3\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "s3 = boto3.resource('s3')\n",
    "col_names =  ['image', 'label']\n",
    "my_df  = pd.DataFrame(columns = col_names)\n",
    "\n",
    "conn = client('s3')  # again assumes boto.cfg setup, assume AWS S3\n",
    "\n",
    "for key in conn.list_objects(Bucket='workattr',Prefix='iteration-1')['Contents']:\n",
    "    \n",
    "    content_object = s3.Object('workattr', key['Key'])\n",
    "    \n",
    "    body=content_object.get()['Body'].read()\n",
    "    \n",
    "    my_json = body.decode('utf8').replace(\"'\", '\"')\n",
    "    \n",
    "    data = json.loads(my_json)\n",
    "    \n",
    "    for x in data:\n",
    "        \n",
    "        my_df.loc[len(my_df)] = [x['dataObject']['s3Uri'],x['annotations'][0]['annotationData']['content'][36]]\n",
    "\n",
    "my_df.sort_values(\"image\", inplace = True) \n",
    "    \n",
    "my_df.to_excel(\"latino_male_21.xlsx\") \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get scores from sagemaker aws public workforces - AMT\n",
    "\n",
    "from boto3 import client\n",
    "import boto3\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "s3 = boto3.resource('s3')\n",
    "col_names =  ['image','rater','label']\n",
    "my_df  = pd.DataFrame(columns = col_names)\n",
    "\n",
    "conn = client('s3')  # again assumes boto.cfg setup, assume AWS S3\n",
    "\n",
    "for key in conn.list_objects(Bucket='workattr',Prefix='iteration-2')['Contents']:\n",
    "    \n",
    "    content_object = s3.Object('workattr', key['Key'])\n",
    "    \n",
    "    body=content_object.get()['Body'].read()\n",
    "    \n",
    "    my_json = body.decode('utf8').replace(\"'\", '\"')\n",
    "    \n",
    "    data = json.loads(my_json)\n",
    "    \n",
    "    for x in data:\n",
    "         \n",
    "        for y in x['annotations']:\n",
    "            my_df.loc[len(my_df)] = [x['dataObject']['s3Uri'],y['workerId'],y['annotationData']['content']]\n",
    "\n",
    "my_df.sort_values(\"image\", inplace = True) \n",
    "\n",
    "#my_df['image'] = my_df['image'].str.replace(r'\\D', '') \n",
    "my_df['label']= my_df['label'].str.replace(r'{\"crowd-image-classifier\":{\"label\":\"', '') \n",
    "my_df['image']= my_df['image'].str.replace(r's3://ecust-fbp-male/', '')  \n",
    "my_df['rater']= my_df['rater'].str.replace(r'public.us-east-1.', '')  \n",
    "my_df['label']= my_df['label'].str.replace(r'\"}}', '') \n",
    "my_df.to_excel(\"public_date_all5_2.xlsx\")\n",
    "#my_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ndf = pd.read_excel(\"gen_guys1.xlsx\")\\ndf2 = pd.read_excel(\"gen_guys2.xlsx\")\\ndf3 = pd.read_excel(\"gen_guys3.xlsx\")\\ndf4 = pd.read_excel(\"gen_guys4.xlsx\")\\nmatrix = df.pivot_table(columns=\\'rater\\', index=\\'image\\', values=\\'label\\').reset_index()\\nmatrix2 = df2.pivot_table(columns=\\'rater\\', index=\\'image\\', values=\\'label\\').reset_index()\\nmatrix3 = df3.pivot_table(columns=\\'rater\\', index=\\'image\\', values=\\'label\\').reset_index()\\nmatrix4 = df4.pivot_table(columns=\\'rater\\', index=\\'image\\', values=\\'label\\').reset_index()\\n#matrix.columns.intersection(matrix2.columns)\\nresult = pd.merge(matrix, matrix2, on=\"image\")\\nresult = pd.merge(result, matrix3, on=\"image\")\\nresult = pd.merge(result, matrix4, on=\"image\")\\n\\n\\nresult[\\'mean\\'] = result.mean(axis=1)\\n\\nresult.to_excel(\"guys_mean.xlsx\")\\n'"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#create score matrix - private scores\n",
    "import os\n",
    "import pandas as pd\n",
    "DIR ='/home/ec2-user/SageMaker/private_generic')\n",
    "files = os.listdir(DIR)\n",
    "files_xls = [f for f in files if f[-4:] == 'xlsx']\n",
    "\n",
    "df=pd.DataFrame(columns = ['image','label'])\n",
    "prefix_x=''\n",
    "\n",
    "for f in files_xls:\n",
    "    prefix_y=f\n",
    "    data = pd.read_excel(f, 'Sheet1',usecols = \"B:C\")    \n",
    "    df=pd.merge(df,data,on = \"image\", how='outer', suffixes=(prefix_x, prefix_y))\n",
    "    prefix_x=f\n",
    "    \n",
    "#df['mean'] = df.mean(axis=1)    \n",
    "df.to_excel(\"private_generic.xlsx\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#common image names\n",
    "female = pd.read_excel('private_date_female.xlsx') \n",
    "male = pd.read_excel('private_date_male.xlsx')\n",
    "s1 = pd.merge(female, male, how='inner', on=['image'])\n",
    "s1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "#os.chdir('/home/ec2-user/SageMaker')\n",
    "with zipfile.ZipFile(\"images.zip\",\"r\") as zip_ref:\n",
    "    zip_ref.extractall(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "542"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "list = os.listdir(\"/home/ubuntu/mtcnn_aligned_images/female/caucasian/\") # dir is your directory path\n",
    "number_files = len(list)\n",
    "number_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#images cropping\n",
    "from PIL import Image\n",
    "from autocrop import Cropper\n",
    "import os\n",
    "cropper = Cropper()\n",
    "files=os.listdir('4-5')\n",
    "for file in files:\n",
    "    print(file)\n",
    "    img_path = os.path.join('4-5', file)\n",
    "    \n",
    "    try:    \n",
    "        cropped_array = cropper.crop(img_path)\n",
    "        cropped_image = Image.fromarray(cropped_array)\n",
    "        width, height = cropped_image.size\n",
    "        print(width, height)\n",
    "        cropped_image.save('./4_crop/'+file)\n",
    "    except:\n",
    "        print('no face',file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/ec2-user/SageMaker/scores.zip'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import shutil\n",
    "shutil.make_archive('scores', 'zip', './scores/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create score matrix - public AMT scores\n",
    "import os\n",
    "import pandas as pd\n",
    "DIR = '/home/ec2-user/SageMaker/public_date/'\n",
    "files = os.listdir(DIR)\n",
    "files_xls = [f for f in files if f[-4:] == 'xlsx']\n",
    "df=pd.DataFrame(columns = ['image','rater'])\n",
    "prefix_x=''\n",
    "\n",
    "for f in files_xls:\n",
    "   # if '_male' in f:\n",
    "       print(f) \n",
    "       data = pd.read_excel(DIR+f, 'Sheet1',usecols = \"B:D\") \n",
    "       data['image']= data['image'].str.replace(r's3://allattrimages/', '') \n",
    "       matrix = data.pivot_table(columns='rater', index='image', values='label').reset_index()\n",
    "       df=pd.merge(df,matrix,on = \"image\", how='outer')\n",
    "\n",
    "df1=df['image']\n",
    "df = df.drop(['image', 'rater'], axis=1)\n",
    "df3 = df.groupby(df.columns.str.split(\"_\").str[0],axis=1).mean()\n",
    "df3 = df3.join(df1) \n",
    "print(df3)\n",
    "df3['mean'] = df3.mean(axis=1)\n",
    "print(df3)\n",
    "df3.to_excel(\"public_date.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#join generic scores\n",
    "\n",
    "import pandas as pd\n",
    "private = pd.read_excel('') \n",
    "public = pd.read_excel('/home/ubuntu/ECUST_FBP/scores/public_generic_all.xlsx') \n",
    "private['image'] = private['image'].astype(str)+'.jpg'\n",
    "all_=pd.merge(private,public,on = \"image\", how='outer')\n",
    "all_.to_excel(\"/home/ubuntu/ECUST_FBP/scores/generic_all.xlsx\")\n",
    "#print(all_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#add file path \n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "df['path'] = ''      \n",
    "        \n",
    "        \n",
    "        \n",
    "df = pd.read_excel('/home/ubuntu/ECUST_FBP/scores/generic_all_deleted_small_rows_columns.xlsx') \n",
    "for index,row in df.iterrows():\n",
    "        for root, dirs, files in os.walk('/home/ubuntu/ECUST_FBP/images/'):\n",
    "            if row['image'] in files:\n",
    "                df.loc[index,'path'] = os.path.join(root, row['image'])\n",
    "\n",
    "            \n",
    "#df['path'] = find(df['image'],'/home/ubuntu/ECUST_FBP/images/')\n",
    "#df['path']\n",
    "df.to_excel(\"/home/ubuntu/ECUST_FBP/scores/generic_all_deleted_small_rows_columns_path.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "df = pd.read_excel('/home/ubuntu/ECUST_FBP/scores/generic_all_deleted_small_rows_columns_path.xlsx')\n",
    "df1 = df['image']\n",
    "df2 = df['path']\n",
    "df_t = df.T\n",
    "df_t = df_t.drop(['image','path','Unnamed: 0'], axis = 0)\n",
    "for column in df_t:\n",
    "\n",
    "df=df_t.T\n",
    "df=df.join(df1)\n",
    "df=df.join(df2)\n",
    "df.to_excel(\"/home/ubuntu/ECUST_FBP/scores/test.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_pytorch_p36)",
   "language": "python",
   "name": "conda_pytorch_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}