clear_data_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_static_feature = pd.read_table(\"./ad_static_feature.out\",header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "500000\n",
      "600000\n",
      "700000\n"
     ]
    }
   ],
   "source": [
    "adStaticFeature_data = []\n",
    "with open('./ad_static_feature.out','r') as f:\n",
    "    attributeName = ['aid','createTime','auid','pid','pType','aIndustryId','size']\n",
    "    cnt = 0\n",
    "    for i, line in enumerate(f):\n",
    "        i = i + 1\n",
    "        line = line.strip().split('\\t')\n",
    "        adStaticFeature_dict = {}\n",
    "        \n",
    "#         if (line[1]=='0') or (len(line)<7):#creat time is 0 and size is null is dirty data\n",
    "#             continue\n",
    "#         if ',' in line[5]:# mutilty aIndustryId is dirty data \n",
    "#             continue\n",
    "            \n",
    "        j = 0\n",
    "        for each in line:\n",
    "            each_list = each.split(',')\n",
    "            adStaticFeature_dict[attributeName[j]] = ' '.join(each_list[0:])\n",
    "#             adStaticFeature_dict[attributeName[j]] = each\n",
    "            j += 1\n",
    "        adStaticFeature_data.append(adStaticFeature_dict)\n",
    "        # print(adStaticFeature_data)\n",
    "        if i % 100000 == 0:\n",
    "            print(i)\n",
    "        if i % 1000000 == 0:\n",
    "            ad_static_feature = pd.DataFrame(adStaticFeature_data)\n",
    "            ad_static_feature.to_csv('./adStaticFeature_' + str(cnt) + '.csv', index=False)\n",
    "            cnt += 1\n",
    "            del adStaticFeature_data,ad_static_feature\n",
    "            adStaticFeature_data = []\n",
    "    ad_static_feature = pd.DataFrame(adStaticFeature_data)\n",
    "    ad_static_feature.to_csv('./adStaticFeature_' + str(cnt) + '.csv', index=False)\n",
    "    del adStaticFeature_data,ad_static_feature\n",
    "    ad_static_feature = pd.concat([pd.read_csv('./adStaticFeature_' + str(k) + '.csv', low_memory=False) for k in range(cnt+1)]).reset_index(drop=True)\n",
    "    ad_static_feature.to_csv('./adStaticFeature.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_operation=pd.read_csv('./adOperation.csv',header=0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_operation = pd.read_table(\"/Users/hanmin/Desktop/testA/ad_operation.dat\",header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "500000\n",
      "600000\n",
      "700000\n"
     ]
    }
   ],
   "source": [
    "adOperation_data = []\n",
    "with open('/Users/hanmin/Desktop/testA/ad_operation.dat','r') as f:\n",
    "    attributeName = ['aid','updateTime','operationType','changeType','newValue']\n",
    "    cnt = 0\n",
    "    for i, line in enumerate(f):\n",
    "        i = i + 1\n",
    "        line = line.strip().split('\\t')\n",
    "        adOperation_dict = {}\n",
    "        j = 0\n",
    "#         print(i)\n",
    "#         if int(line[0]) in not_contain:#the aid if is not appear in ad static ad feature is dirty data\n",
    "#             continue\n",
    "#         if line[1] == '20190230000000':#the 2.30 is not correct\n",
    "#             continue\n",
    "            \n",
    "        for each in line:\n",
    "            adOperation_dict[attributeName[j]] = each\n",
    "            j += 1\n",
    "        adOperation_data.append(adOperation_dict)\n",
    "        # print(adOperation_data)\n",
    "        if i % 100000 == 0:\n",
    "            print(i)\n",
    "        if i % 1000000 == 0:\n",
    "            ad_operation = pd.DataFrame(adOperation_data)\n",
    "            ad_operation.to_csv('/Users/hanmin/Desktop/NewData/adOperation_' + str(cnt) + '.csv', index=False)\n",
    "            cnt += 1\n",
    "            del adOperation_data,ad_operation\n",
    "            adOperation_data = []\n",
    "    ad_operation = pd.DataFrame(adOperation_data)\n",
    "    ad_operation.to_csv('/Users/hanmin/Desktop/NewData/adOperation_' + str(cnt) + '.csv', index=False)\n",
    "    del adOperation_data,ad_operation\n",
    "    ad_operation = pd.concat([pd.read_csv('/Users/hanmin/Desktop/NewData/adOperation_' + str(k) + '.csv', low_memory=False) for k in range(cnt+1)]).reset_index(drop=True)\n",
    "    ad_operation.to_csv('/Users/hanmin/Desktop/NewData/adOperation.csv', index=False)\n",
    "# df_test.aid = df_test.aid.astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ad_operation=pd.read_csv('./adOperation.csv',header=0) \n",
    "df_ad_operation.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_operation=df_ad_operation[(df_ad_operation.updateTime%1000000)==0]\n",
    "ad_bid=ad_operation[ad_operation.changeType==2][[\"aid\",\"newValue\"]]#bid\n",
    "ad_bid.columns = ['aid','bid']\n",
    "ad_people=ad_operation[ad_operation.changeType==3][[\"aid\",\"newValue\"]]#people\n",
    "ad_people.columns = ['aid','people']\n",
    "ad_time=ad_operation[ad_operation.changeType==4][[\"aid\",\"newValue\"]]#time\n",
    "ad_time.columns = ['aid','time']\n",
    "ad_static_operation=pd.merge(ad_static_feature,ad_bid,on='aid',how='left')\n",
    "ad_static_operation=pd.merge(ad_static_operation,ad_people,on='aid',how='left')\n",
    "ad_static_operation=pd.merge(ad_static_operation,ad_time,on='aid',how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_static_operation=pd.merge(ad_static_feature,ad_operation,on='aid',how='inner')\n",
    "ad_static_operation.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aIndustryId</th>\n",
       "      <th>aid</th>\n",
       "      <th>auid</th>\n",
       "      <th>createTime</th>\n",
       "      <th>pType</th>\n",
       "      <th>pid</th>\n",
       "      <th>size</th>\n",
       "      <th>changeType</th>\n",
       "      <th>newValue</th>\n",
       "      <th>operationType</th>\n",
       "      <th>updateTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>172</td>\n",
       "      <td>457009</td>\n",
       "      <td>23614</td>\n",
       "      <td>1550439402</td>\n",
       "      <td>13</td>\n",
       "      <td>7447</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>150</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>172</td>\n",
       "      <td>457009</td>\n",
       "      <td>23614</td>\n",
       "      <td>1550439402</td>\n",
       "      <td>13</td>\n",
       "      <td>7447</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>area:11442</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>172</td>\n",
       "      <td>457009</td>\n",
       "      <td>23614</td>\n",
       "      <td>1550439402</td>\n",
       "      <td>13</td>\n",
       "      <td>7447</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>281474976694272,281474976694272,28147497669427...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>172</td>\n",
       "      <td>457009</td>\n",
       "      <td>23614</td>\n",
       "      <td>1550439402</td>\n",
       "      <td>13</td>\n",
       "      <td>7447</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>20190223000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12</td>\n",
       "      <td>347637</td>\n",
       "      <td>18529</td>\n",
       "      <td>1550790222</td>\n",
       "      <td>13</td>\n",
       "      <td>28775</td>\n",
       "      <td>34</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>347637</td>\n",
       "      <td>18529</td>\n",
       "      <td>1550790222</td>\n",
       "      <td>13</td>\n",
       "      <td>28775</td>\n",
       "      <td>34</td>\n",
       "      <td>3</td>\n",
       "      <td>age:217,601,79,202,837,942,638,394,347,731,739...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12</td>\n",
       "      <td>347637</td>\n",
       "      <td>18529</td>\n",
       "      <td>1550790222</td>\n",
       "      <td>13</td>\n",
       "      <td>28775</td>\n",
       "      <td>34</td>\n",
       "      <td>4</td>\n",
       "      <td>281474976694272,281474976694272,28147497669427...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>84</td>\n",
       "      <td>288207</td>\n",
       "      <td>27840</td>\n",
       "      <td>1543437023</td>\n",
       "      <td>13</td>\n",
       "      <td>17911</td>\n",
       "      <td>34</td>\n",
       "      <td>2</td>\n",
       "      <td>200</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>84</td>\n",
       "      <td>288207</td>\n",
       "      <td>27840</td>\n",
       "      <td>1543437023</td>\n",
       "      <td>13</td>\n",
       "      <td>17911</td>\n",
       "      <td>34</td>\n",
       "      <td>3</td>\n",
       "      <td>age:217,601,79,202,837,942,638,394,347,731,739...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>84</td>\n",
       "      <td>288207</td>\n",
       "      <td>27840</td>\n",
       "      <td>1543437023</td>\n",
       "      <td>13</td>\n",
       "      <td>17911</td>\n",
       "      <td>34</td>\n",
       "      <td>4</td>\n",
       "      <td>70364516302848,70364516302848,70364516302848,7...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  aIndustryId     aid   auid  createTime  pType    pid size  changeType  \\\n",
       "0         172  457009  23614  1550439402     13   7447  NaN           2   \n",
       "1         172  457009  23614  1550439402     13   7447  NaN           3   \n",
       "2         172  457009  23614  1550439402     13   7447  NaN           4   \n",
       "3         172  457009  23614  1550439402     13   7447  NaN           1   \n",
       "4          12  347637  18529  1550790222     13  28775   34           2   \n",
       "5          12  347637  18529  1550790222     13  28775   34           3   \n",
       "6          12  347637  18529  1550790222     13  28775   34           4   \n",
       "7          84  288207  27840  1543437023     13  17911   34           2   \n",
       "8          84  288207  27840  1543437023     13  17911   34           3   \n",
       "9          84  288207  27840  1543437023     13  17911   34           4   \n",
       "\n",
       "                                            newValue  operationType  \\\n",
       "0                                                150              2   \n",
       "1                                         area:11442              2   \n",
       "2  281474976694272,281474976694272,28147497669427...              2   \n",
       "3                                                  1              1   \n",
       "4                                                100              2   \n",
       "5  age:217,601,79,202,837,942,638,394,347,731,739...              2   \n",
       "6  281474976694272,281474976694272,28147497669427...              2   \n",
       "7                                                200              2   \n",
       "8  age:217,601,79,202,837,942,638,394,347,731,739...              2   \n",
       "9  70364516302848,70364516302848,70364516302848,7...              2   \n",
       "\n",
       "       updateTime  \n",
       "0               0  \n",
       "1               0  \n",
       "2               0  \n",
       "3  20190223000000  \n",
       "4               0  \n",
       "5               0  \n",
       "6               0  \n",
       "7               0  \n",
       "8               0  \n",
       "9               0  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adExposure = pd.read_csv('./adExposure.csv')\n",
    "adExposure =adExposure.drop_duplicates(['aRequestId','adLocationId','aid'])\n",
    "adExposure\n",
    "adExposure =adExposure.drop_duplicates(['aRequestId','adLocationId','aid'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import time\n",
    "import numpy as np\n",
    "adExposure = pd.read_csv('./adExposureNew.csv')\n",
    "adExposure['data']=adExposure.requestTime.apply(lambda x:time.strftime('%m%d',time.localtime(x)))\n",
    "aid_label = []\n",
    "i=0\n",
    "# adExposure2=adExposure.groupby([adExposure.aid, adExposure.data])\n",
    "for (k1,k2), group in adExposure.groupby(['aid','data']):\n",
    "    i+=1\n",
    "    print(i)\n",
    "    aid_label_dict = {}\n",
    "    aid_label_dict[\"aid\"]=k1\n",
    "    aid_label_dict[\"data\"]=k2\n",
    "    aid_label_dict[\"label\"]=len(group)\n",
    "    aid_label.append(aid_label_dict)\n",
    "#     print(aid_label)\n",
    "print(aid_label)\n",
    "ad_label_pd = pd.DataFrame(aid_label)\n",
    "ad_label_pd.to_csv('./ad_label.csv', index=False)\n",
    "ad_label_pd= pd.read_csv('./ad_label.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.merge(ad_static_operation,ad_label_pd,on='aid',how='right')\n",
    "ad_static_operation\n",
    "data=pd.merge(ad_static_operation,ad_label_pd,on='aid',how='left')\n",
    "data2=data[(data.time).notnull()]\n",
    "data3=data2[(data.label).notnull()]\n",
    "data3.to_csv('/Users/hanmin/Desktop/NewData/training.csv', index=False)\n",
    "label_train=data3.label\n",
    "label_train=data3.label\n",
    "feature_train=data3.drop(columns=['label'])\n",
    "# test= ['aIndustryId', 'aid ','auid', 'createTime ', 'crowd', 'exposureTime', 'pType', 'pid', \n",
    "#         'price', 'size', 'tid']---->label\n",
    "feature_train.rename(columns={'people':'crowd', 'bid':'price', 'time':'exposureTime'}, inplace=True)\n",
    "extime=feature_train2.exposureTime.str.split(\",\", expand = True).astype(\"int64\")\n",
    "frames=[extime,feature_train2]\n",
    "feature_train = pd.concat(frames, axis=1)\n",
    "feature_train.rename(columns={0:'exposureTime0', 1:'exposureTime1', 2:'exposureTime2',3:'exposureTime3',4:'exposureTime4',5:'exposureTime5',6:'exposureTime6',}, inplace=True)\n",
    "feature_train=feature_train.drop(columns=['exposureTime'])\n",
    "feature_train['exposureTime6']=feature_train.exposureTime6.apply(lambda x:time.strftime('%m%d',time.localtime(x)))\n",
    "feature_train.to_csv('/Users/hanmin/Desktop/NewData/feature_train1.csv', index=False)\n",
    "feature_train['crowd']=feature_train.crowd.str.split(',').str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import warnings\n",
    "from sklearn.preprocessing import scale\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "import lightgbm as lgb\n",
    "lr = LogisticRegression(random_state=2018,tol=1e-6)\n",
    "tree = DecisionTreeClassifier(random_state=2018) #决策树模型\n",
    "svm = SVC(probability=True,random_state=2018,tol=1e-6)  # SVM模型\n",
    "forest=RandomForestClassifier(n_estimators=100,random_state=2018) #　随机森林\n",
    "Gbdt=GradientBoostingClassifier(random_state=2018) #CBDT\n",
    "Xgbc=XGBClassifier(random_state=2018)  #Xgbc\n",
    "gbm=lgb.LGBMClassifier(random_state=2018)  #lgb\n",
    "\n",
    "def muti_score(model):\n",
    "    warnings.filterwarnings('ignore')\n",
    "    accuracy = cross_val_score(model, feature_train, label_train, scoring='accuracy', cv=5)\n",
    "    precision = cross_val_score(model, feature_train, label_train, scoring='precision', cv=5)\n",
    "    recall = cross_val_score(model, feature_train, label_train, scoring='recall', cv=5)\n",
    "    f1_score = cross_val_score(model, feature_train, label_train, scoring='f1', cv=5)\n",
    "    auc = cross_val_score(model, feature_train, label_train, scoring='roc_auc', cv=5)\n",
    "    print(\"准确率:\",accuracy.mean())\n",
    "    print(\"精确率:\",precision.mean())\n",
    "    print(\"召回率:\",recall.mean())\n",
    "    print(\"F1_score:\",f1_score.mean())\n",
    "    print(\"AUC:\",auc.mean())\n",
    "model_name=[\"lr\"]#,\"tree\",\"svm\",\"forest\",\"Gbdt\",\"Xgbc\",\"gbm\"\n",
    "for name in model_name:\n",
    "    model=eval(name)\n",
    "    print(name)\n",
    "    muti_score(model)\n",
    "    import pandas as pd\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import OneHotEncoder,LabelEncoder\n",
    "from scipy import sparse\n",
    "import os\n",
    "\n",
    "# ad_feature=pd.read_csv('../data/adFeature.csv')\n",
    "# if os.path.exists('../data/userFeature.csv'):\n",
    "#     user_feature=pd.read_csv('../data/userFeature.csv')\n",
    "# else:\n",
    "#     userFeature_data = []\n",
    "#     with open('../data/userFeature.data', 'r') as f:\n",
    "#         cnt = 0\n",
    "#         for i, line in enumerate(f):\n",
    "#             line = line.strip().split('|')\n",
    "#             userFeature_dict = {}\n",
    "#             for each in line:\n",
    "#                 each_list = each.split(' ')\n",
    "#                 userFeature_dict[each_list[0]] = ' '.join(each_list[1:])\n",
    "#             userFeature_data.append(userFeature_dict)\n",
    "#             if i % 100000 == 0:\n",
    "#                 print(i)\n",
    "#             if i % 1000000 == 0:\n",
    "#                 user_feature = pd.DataFrame(userFeature_data)\n",
    "#                 user_feature.to_csv('../data/userFeature_' + str(cnt) + '.csv', index=False)\n",
    "#                 cnt += 1\n",
    "#                 del userFeature_data, user_feature\n",
    "#                 userFeature_data = []\n",
    "#         user_feature = pd.DataFrame(userFeature_data)\n",
    "#         user_feature.to_csv('../data/userFeature_' + str(cnt) + '.csv', index=False)\n",
    "#         del userFeature_data, user_feature\n",
    "#         user_feature = pd.concat([pd.read_csv('../data/userFeature_' + str(i) + '.csv') for i in range(cnt + 1)]).reset_index(drop=True)\n",
    "#         user_feature.to_csv('../data/userFeature.csv', index=False)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# train=pd.read_csv('../data/train.csv')\n",
    "# predict=pd.read_csv('../data/test1.csv')\n",
    "# train.loc[train['label']==-1,'label']=0\n",
    "# predict['label']=-1\n",
    "# data=pd.concat([train,predict])\n",
    "# data=pd.merge(data,ad_feature,on='aid',how='left')\n",
    "# data=pd.merge(data,user_feature,on='uid',how='left')\n",
    "# data=data.fillna('-1')\n",
    "# one_hot_feature=['LBS','age','carrier','consumptionAbility','education','gender','house','os','ct','marriageStatus','advertiserId','campaignId', 'creativeId',\n",
    "#        'adCategoryId', 'productId', 'productType']\n",
    "# vector_feature=['appIdAction','appIdInstall','interest1','interest2','interest3','interest4','interest5','kw1','kw2','kw3','topic1','topic2','topic3']\n",
    "# for feature in one_hot_feature:\n",
    "#     try:\n",
    "#         data[feature] = LabelEncoder().fit_transform(data[feature].apply(int))\n",
    "#     except:\n",
    "#         data[feature] = LabelEncoder().fit_transform(data[feature])\n",
    "\n",
    "# train=data[data.label!=-1]\n",
    "# train_y=train.pop('label')\n",
    "# # train, test, train_y, test_y = train_test_split(train,train_y,test_size=0.2, random_state=2018)\n",
    "# test=data[data.label==-1]\n",
    "# res=test[['aid','uid']]\n",
    "# test=test.drop('label',axis=1)\n",
    "# enc = OneHotEncoder()\n",
    "# train_x=train[['creativeSize']]\n",
    "# test_x=test[['creativeSize']]\n",
    "\n",
    "# for feature in one_hot_feature:\n",
    "#     enc.fit(data[feature].values.reshape(-1, 1))\n",
    "#     train_a=enc.transform(train[feature].values.reshape(-1, 1))\n",
    "#     test_a = enc.transform(test[feature].values.reshape(-1, 1))\n",
    "#     train_x= sparse.hstack((train_x, train_a))\n",
    "#     test_x = sparse.hstack((test_x, test_a))\n",
    "# print('one-hot prepared !')\n",
    "\n",
    "# cv=CountVectorizer()\n",
    "# for feature in vector_feature:\n",
    "#     cv.fit(data[feature])\n",
    "#     train_a = cv.transform(train[feature])\n",
    "#     test_a = cv.transform(test[feature])\n",
    "#     train_x = sparse.hstack((train_x, train_a))\n",
    "#     test_x = sparse.hstack((test_x, test_a))\n",
    "# print('cv prepared !')\n",
    "\n",
    "def LGB_test(train_x,train_y,test_x,test_y):\n",
    "    from multiprocessing import cpu_count\n",
    "    print(\"LGB test\")\n",
    "    clf = lgb.LGBMClassifier(\n",
    "        boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,\n",
    "        max_depth=-1, n_estimators=1000, objective='binary',\n",
    "        subsample=0.7, colsample_bytree=0.7, subsample_freq=1,\n",
    "        learning_rate=0.05, min_child_weight=50,random_state=2018,n_jobs=cpu_count()-1\n",
    "    )\n",
    "    clf.fit(train_x, train_y,eval_set=[(train_x, train_y),(test_x,test_y)],eval_metric='auc',early_stopping_rounds=100)\n",
    "    # print(clf.feature_importances_)\n",
    "    return clf,clf.best_score_[ 'valid_1']['auc']\n",
    "\n",
    "def LGB_predict(train_x,train_y,test_x,res):\n",
    "    print(\"LGB test\")\n",
    "    clf = lgb.LGBMClassifier(\n",
    "        boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,\n",
    "        max_depth=-1, n_estimators=1500, objective='binary',\n",
    "        subsample=0.7, colsample_bytree=0.7, subsample_freq=1,\n",
    "        learning_rate=0.05, min_child_weight=50, random_state=2018, n_jobs=100\n",
    "    )\n",
    "    clf.fit(train_x, train_y, eval_set=[(train_x, train_y)], eval_metric='auc',early_stopping_rounds=100)\n",
    "    res['score'] = clf.predict_proba(test_x)[:,1]\n",
    "    res['score'] = res['score'].apply(lambda x: float('%.6f' % x))\n",
    "    res.to_csv('../data/submission.csv', index=False)\n",
    "    os.system('zip baseline.zip ../data/submission.csv')\n",
    "    return clf\n",
    "\n",
    "model=LGB_predict(train_x,train_y,test_x,res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}