ksb_lgb&nn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "import random\n",
    "import os\n",
    "import gc\n",
    "import math\n",
    "from multiprocessing import Pool,cpu_count\n",
    "from sklearn.preprocessing import Normalizer,LabelEncoder,OneHotEncoder,MinMaxScaler\n",
    "from sklearn.decomposition import PCA,TruncatedSVD\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.model_selection import StratifiedKFold,train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.utils import resample\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from scipy import sparse\n",
    "import matplotlib.pyplot as plt\n",
    "from gensim.models import word2vec\n",
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "from gensim.test.utils import datapath as dpath, get_tmpfile\n",
    "import torch \n",
    "import codecs\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "import catboost as catb\n",
    "import pickle\n",
    "import time\n",
    "import datetime\n",
    "import math\n",
    "import scipy.special as special\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import tqdm\n",
    "from optimizer import Lookahead\n",
    "from optimizer import RAdam\n",
    "import torch.utils.data as Data\n",
    "import codecs\n",
    "import sys\n",
    "import jieba.posseg\n",
    "import jieba.analyse\n",
    "import re\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "datapath = '.....'\n",
    "\n",
    "t0_train = 3838\n",
    "t1_train = 3867\n",
    "t0_eval = 3868\n",
    "t1_eval = 3874\n",
    "t0_a = 3807\n",
    "t1_a = 3867\n",
    "evalday = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def memoryOptimization(data,floattype):\n",
    "    subdata = data.select_dtypes(include = 'int')\n",
    "    for col in subdata.columns:\n",
    "        m = subdata[col].max()\n",
    "        n = subdata[col].min()\n",
    "        if m < np.power(2,31)-1 and n >=  -np.power(2,31):\n",
    "            if m < np.power(2,15)-1 and n >=  -np.power(2,15):\n",
    "                if m < np.power(2,7)-1 and n >=  -np.power(2,7):\n",
    "                    subdata[col] = subdata[col].astype(np.int8)\n",
    "                else:\n",
    "                    subdata[col] = subdata[col].astype(np.int16)\n",
    "            else:\n",
    "                subdata[col] = subdata[col].astype(np.int32)\n",
    "    data[subdata.columns] = subdata\n",
    "    subdata = data.select_dtypes(include = 'float')\n",
    "    data[subdata.columns] = data[subdata.columns].astype(floattype)\n",
    "#     subdata = data.select_dtypes(include = 'object')\n",
    "#     data[subdata.columns] = data[subdata.columns].astype('category')\n",
    "    gc.collect()\n",
    "    return data\n",
    "\n",
    "# def down_sample(df,df_feat,rate):#对目标特征下采样，通过给定随机数种子保证每个特征组抽样的负样本是同样的\n",
    "#     df_majority = df_feat[df['label']==0]\n",
    "#     df_minority = df_feat[df['label']==1]\n",
    "#     positive_num = df_minority.shape[0]\n",
    "#     df_majority_downsampled = resample(df_majority,\n",
    "#                                      replace=False,  # sample without replacement\n",
    "#                                      n_samples=positive_num*rate,  # to match minority class\n",
    "#                                      random_state=7)  # reproducible results\n",
    "#     df_downsampled = pd.concat([df_majority_downsampled, df_minority],axis = 0,ignore_index = True)\n",
    "#     del df_majority, df_minority, df_majority_downsampled\n",
    "#     return df_downsampled\n",
    "\n",
    "def lgb_train_pre1(train_x,train_y,test_x,categoryfeas,dropfeas,one,save_model):    \n",
    "    train_x = train_x.drop(dropfeas+['inviteday','inviteallhour'],axis = 1)\n",
    "    test_x = test_x.drop(dropfeas+['inviteday','inviteallhour'],axis = 1)\n",
    "    pickle.dump(list(train_x.columns), open(datapath+'data/lgb_fea.pkl', 'wb'))\n",
    "    params_lgbc ={\n",
    "        'boosting_type': 'gbdt',\n",
    "        'objective': 'binary',  \n",
    "        'num_leaves': 41, \n",
    "        'learning_rate': 0.1,\n",
    "        'feature_fraction': 0.8,\n",
    "        'bagging_fraction': 0.8,\n",
    "        'bagging_freq': 1,\n",
    "        'min_sum_hessian_in_leaf': 10,\n",
    "        'num_threads': cpu_count() - 1,\n",
    "        'seed': 7, \n",
    "        'n_estimators':50000,\n",
    "        'max_depth': 6,\n",
    "        'subsample':0.9,\n",
    "        'subsample_freq':2,\n",
    "        'reg_alpha':0, \n",
    "        'reg_lambda':2\n",
    "        # 'device': 'gpu',\n",
    "    }\n",
    "    \n",
    "    pre_train = pd.Series(np.zeros(len(train_y)))\n",
    "    pre_test = []\n",
    "    kf = StratifiedKFold(n_splits = 5,shuffle = True,random_state = 2019)\n",
    "    fold = 1\n",
    "    for train_index,eval_index in kf.split(train_x,train_y):\n",
    "        dtrain_x = train_x.loc[train_index,:]\n",
    "        deval_x = train_x.loc[eval_index,:]\n",
    "        dtrain_y = train_y[train_index]\n",
    "        deval_y = train_y[eval_index]\n",
    "        if flag_weight:\n",
    "            sample_weight = ((dtrain_x['inviteday']-3500)/(dtrain_x['inviteday']-3500).mean()).values\n",
    "        else:\n",
    "            sample_weight = None\n",
    "        lgbc = lgb.LGBMClassifier(random_state = 2020,**params_lgbc) # np.random.randint(1,3000)\n",
    "        lgbc.fit(dtrain_x,dtrain_y,eval_set = [(deval_x,deval_y)],eval_names = ['eval'],eval_metric = 'auc',\n",
    "                 early_stopping_rounds = 50,sample_weight = sample_weight,verbose = 100,categorical_feature = categoryfeas)\n",
    "        pre_train[eval_index] = lgbc.predict_proba(deval_x,num_iteration = lgbc.best_iteration_)[:,1]\n",
    "        pre_test.append(list(lgbc.predict_proba(test_x,num_iteration = lgbc.best_iteration_)[:,1]))\n",
    "        if save_model:\n",
    "            joblib.dump(lgbc, open(datapath+'data/lgb_'+str(params_lgbc['learning_rate'])+'_'+str(fold)+'.pkl', 'wb'))\n",
    "        fold += 1\n",
    "        if one:\n",
    "            break\n",
    "    pre_test = np.array(pre_test)\n",
    "    pre_test = np.mean(pre_test,axis = 0)\n",
    "    \n",
    "    score = roc_auc_score(train_y,pre_train)\n",
    "    feas = train_x.columns\n",
    "    imps = lgbc.feature_importances_\n",
    "    fea_imp = pd.DataFrame(pd.Series(feas),columns = ['feas'])\n",
    "    fea_imp['imp'] = imps\n",
    "    fea_imp = fea_imp.sort_values(by = 'imp',ascending = False)\n",
    "    del dtrain_x\n",
    "    del deval_x\n",
    "    del dtrain_y\n",
    "    del deval_y\n",
    "    gc.collect()\n",
    "    return pre_test,pre_train,score,fea_imp,lgbc.best_iteration_\n",
    "def xgb_train_pre1(train_x,train_y,test_x,dropfeas,one,save_model):\n",
    "    train_x = train_x.drop(dropfeas+['inviteday','inviteallhour'],axis = 1)\n",
    "    test_x = test_x.drop(dropfeas+['inviteday','inviteallhour'],axis = 1)\n",
    "    pickle.dump(list(train_x.columns), open(datapath+'data/xgb_fea.pkl', 'wb'))\n",
    "    params_lgbc ={        \n",
    "        'booster':'gbtree',\n",
    "        'learning_rate':0.1,\n",
    "        'n_estimators':50000,\n",
    "        'max_depth':6,\n",
    "        'min_child_weight':3,\n",
    "        'gamma':0.1,\n",
    "        'subsample':0.9,\n",
    "        'colsample_bytree':0.8,\n",
    "        'reg_alpha':0, \n",
    "        'reg_lambda':2,\n",
    "        'objective':'binary:logistic',\n",
    "        'nthread':cpu_count() - 1,\n",
    "        'scale_pos_weight':1,\n",
    "        'seed':7,\n",
    "#         'tree_method':'gpu_hist'\n",
    "    }\n",
    "    \n",
    "    pre_train = pd.Series(np.zeros(len(train_y)))\n",
    "    pre_test = []\n",
    "    kf = StratifiedKFold(n_splits = 5,shuffle = True,random_state = 2019)\n",
    "    fold = 1\n",
    "    for train_index,eval_index in kf.split(train_x,train_y):\n",
    "        dtrain_x = train_x.loc[train_index,:]\n",
    "        deval_x = train_x.loc[eval_index,:]\n",
    "        dtrain_y = train_y[train_index]\n",
    "        deval_y = train_y[eval_index]\n",
    "        if flag_weight:\n",
    "            sample_weight = ((dtrain_x['inviteday']-3500)/(dtrain_x['inviteday']-3500).mean()).values\n",
    "        else:\n",
    "            sample_weight = None\n",
    "        xgbc = xgb.XGBClassifier(random_state = 2019,**params_lgbc)\n",
    "        xgbc.fit(dtrain_x,dtrain_y,eval_set = [(deval_x,deval_y)],eval_metric = 'auc',\n",
    "                 early_stopping_rounds = 50,sample_weight = sample_weight,verbose = 100)\n",
    "        pre_train[eval_index] = xgbc.predict_proba(deval_x, ntree_limit=xgbc.best_ntree_limit)[:,1]\n",
    "        pre_test.append(list(xgbc.predict_proba(test_x, ntree_limit=xgbc.best_ntree_limit)[:,1]))\n",
    "        if save_model:\n",
    "            joblib.dump(xgbc, open(datapath+'data/xgb_'+str(params_lgbc['learning_rate'])+'_'+str(fold)+'.pkl', 'wb'))\n",
    "        fold += 1\n",
    "        if one:\n",
    "            break\n",
    "    pre_test = np.array(pre_test)\n",
    "    pre_test = np.mean(pre_test,axis = 0)\n",
    "    \n",
    "    score = roc_auc_score(train_y,pre_train)\n",
    "    feas = train_x.columns\n",
    "    imps = xgbc.feature_importances_\n",
    "    fea_imp = pd.DataFrame(pd.Series(feas),columns = ['feas'])\n",
    "    fea_imp['imp'] = imps\n",
    "    fea_imp = fea_imp.sort_values(by = 'imp',ascending = False)\n",
    "    del dtrain_x\n",
    "    del deval_x\n",
    "    del dtrain_y\n",
    "    del deval_y\n",
    "    gc.collect()\n",
    "    return pre_test,pre_train,score,fea_imp,xgbc.best_iteration\n",
    "def cat_train_pre1(train_x,train_y,test_x,categoryfeas,dropfeas,one,save_model):\n",
    "    train_x = train_x.drop(dropfeas+['inviteday','inviteallhour'],axis = 1)\n",
    "    test_x = test_x.drop(dropfeas+['inviteday','inviteallhour'],axis = 1)\n",
    "    pickle.dump(list(train_x.columns), open(datapath+'data/cat_fea.pkl', 'wb'))\n",
    "    params_lgbc ={        \n",
    "        'learning_rate':0.1,\n",
    "        'n_estimators':50000,\n",
    "        'max_depth':6,\n",
    "#         'subsample':0.9,\n",
    "        'l2_leaf_reg':2,\n",
    "        'objective':'Logloss',\n",
    "        'scale_pos_weight':1,\n",
    "        'eval_metric':'AUC',\n",
    "        'colsample_bylevel':0.8\n",
    "    }\n",
    "    \n",
    "    pre_train = pd.Series(np.zeros(len(train_y)))\n",
    "    pre_test = []\n",
    "    kf = StratifiedKFold(n_splits = 5,shuffle = True,random_state = 2019)\n",
    "    fold = 1\n",
    "    for train_index,eval_index in kf.split(train_x,train_y):\n",
    "        dtrain_x = train_x.loc[train_index,:]\n",
    "        deval_x = train_x.loc[eval_index,:]\n",
    "        dtrain_y = train_y[train_index]\n",
    "        deval_y = train_y[eval_index]\n",
    "        catbc = catb.CatBoostClassifier(random_state = 2019,**params_lgbc)\n",
    "        catbc.fit(dtrain_x,dtrain_y,eval_set = [(deval_x,deval_y)],cat_features = categoryfeas, # eval_metric = 'auc',\n",
    "                 early_stopping_rounds = 50,sample_weight = sample_weight,verbose = 100)\n",
    "        pre_train[eval_index] = catbc.predict_proba(deval_x)[:,1]\n",
    "        pre_test.append(list(catbc.predict_proba(test_x)[:,1]))\n",
    "        if save_model:\n",
    "            joblib.dump(catbc, open(datapath+'data/cat_'+str(params_lgbc['learning_rate'])+'_'+str(fold)+'.pkl', 'wb'))\n",
    "        fold += 1\n",
    "        if one:\n",
    "            break\n",
    "    pre_test = np.array(pre_test)\n",
    "    pre_test = np.mean(pre_test,axis = 0)\n",
    "    \n",
    "    score = roc_auc_score(train_y,pre_train)\n",
    "    feas = train_x.columns\n",
    "    imps = catbc.get_feature_importance()\n",
    "    fea_imp = pd.DataFrame(pd.Series(feas),columns = ['feas'])\n",
    "    fea_imp['imp'] = imps\n",
    "    fea_imp = fea_imp.sort_values(by = 'imp',ascending = False)\n",
    "    del dtrain_x\n",
    "    del deval_x\n",
    "    del dtrain_y\n",
    "    del deval_y\n",
    "    gc.collect()\n",
    "    return pre_test,pre_train,score,fea_imp,catbc.get_best_iteration()\n",
    "\n",
    "def parallelize_dataframe(df,func):\n",
    "    df_split = np.array_split(df,20)#cpu_count()\n",
    "    pool = Pool(20)#cpu_count()\n",
    "    df = pd.concat(pool.map(func, df_split))\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    return df\n",
    "\n",
    "def mostliketheme(x):\n",
    "    if x == '-1':\n",
    "        return '-1'\n",
    "    for theme in iter(x.strip().split(',')):\n",
    "        theme = theme.strip().split(':')\n",
    "        try:\n",
    "            if float(theme[1])>biggestlike:\n",
    "                biggestlike = theme[1]\n",
    "                mostliketheme = theme[0]\n",
    "        except:\n",
    "            biggestlike = theme[1]\n",
    "            mostliketheme = theme[0]\n",
    "    return mostliketheme\n",
    "\n",
    "def getweekday(data):\n",
    "    return data%7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# .数据处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一、预处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 词向量\n",
    "\n",
    "处理词向量文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_word = pd.read_csv(datapath+'data/word_vectors_64d.txt',sep = ' ',header = None,\n",
    "                        names = ['word_'+str(i) for i in range(64)])\n",
    "\n",
    "d = data_word['word_0'].apply(lambda x:x.split('\\t'))\n",
    "data_word['wordId'] = d.apply(lambda x:x[0])\n",
    "data_word['word_0'] = d.apply(lambda x:x[1]).astype(np.float32)\n",
    "data_word = memoryOptimization(data_word,np.float32)\n",
    "\n",
    "data_word.to_csv(datapath+'data/word_vector.csv',header = True,index = False)\n",
    "del data_word\n",
    "del d\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 单字向量\n",
    "\n",
    "处理单字向量文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_letter = pd.read_csv(datapath+'data/single_word_vectors_64d.txt',sep = ' ',header = None,\n",
    "                        names = ['letter_'+str(i) for i in range(64)])\n",
    "\n",
    "d = data_letter['letter_0'].apply(lambda x:x.split('\\t'))\n",
    "data_letter['letterId'] = d.apply(lambda x:x[0])\n",
    "data_letter['letter_0'] = d.apply(lambda x:x[1]).astype(np.float32)\n",
    "data_letter = memoryOptimization(data_letter,np.float32)\n",
    "\n",
    "data_letter.to_csv(datapath+'data/letter_vector.csv',header = True,index = False)\n",
    "del data_letter\n",
    "del d\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 话题向量\n",
    "\n",
    "处理话题向量文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_theme = pd.read_csv(datapath+'data/topic_vectors_64d.txt',sep = ' ',header = None,\n",
    "                        names = ['theme_'+str(i) for i in range(64)])\n",
    "\n",
    "d = data_theme['theme_0'].apply(lambda x:x.split('\\t'))\n",
    "data_theme['themeId'] = d.apply(lambda x:x[0])\n",
    "data_theme['theme_0'] = d.apply(lambda x:x[1]).astype(np.float32)\n",
    "data_theme = memoryOptimization(data_theme,np.float32)\n",
    "data_theme.to_csv(datapath+'data/theme_vector.csv',header = True,index = False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "利用pca将64维的话题向量进行压缩为22维，以进一步去除噪声"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "normalizer = Normalizer(copy = False)\n",
    "data_pca = data_theme[['theme_'+str(i) for i in range(64)]]\n",
    "data_pca = normalizer.fit_transform(data_pca)\n",
    "n = int(64*0.35)\n",
    "svd = TruncatedSVD(n_components = n)\n",
    "data_pca = svd.fit_transform(data_pca)\n",
    "data_pca= pd.DataFrame(data_pca,columns = ['theme_'+str(i) for i in range(n)])\n",
    "data_pca['themeId'] = data_theme['themeId']\n",
    "\n",
    "data_pca.to_csv(datapath+'data/theme_vector_pca.csv',header = True,index = False)\n",
    "del data_theme\n",
    "del data_pca\n",
    "del d\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 回答记录文件\n",
    "\n",
    "处理回答记录文件，根据回答时间字段生成回答的天、时字段"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns = ['answerId','qId','writerId','answertime','content_letters','content_words','good_bool','recommend_bool','yuanzhuo_bool',\n",
    "           'picture_bool','vedio_bool','wordnum','likenum','cancellikenum','commentnum','collectnum','3qnum','jubaonum','unhelpnum',\n",
    "           'unlikenum']\n",
    "data_answer = pd.read_csv(datapath+'data/answer_info.txt',sep = '\\t',header = None,names = columns)\n",
    "data_answer['answerday'] = np.nan\n",
    "data_answer['answerhour'] = np.nan\n",
    "data_answer['answerday'] = data_answer['answertime'].apply(lambda x: int(x.split('-')[0][1:]))\n",
    "data_answer['answerhour'] = data_answer['answertime'].apply(lambda x: int(x.split('-')[1][1:]))\n",
    "data_answer = memoryOptimization(data_answer,np.float32)\n",
    "\n",
    "data_answer.to_csv(datapath+'data/data_answer.csv',header = True,index = False)\n",
    "del data_answer\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 问题文件\n",
    "\n",
    "处理问题信息文件，根据问题创建时间字段生成问题创建的的天、时字段  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns = ['qId','createtime','title_letters','title_words','describe_letters','describe_words','themeId']\n",
    "data_question = pd.read_csv(datapath+'data/question_info.txt',sep = '\\t',header = None,names = columns)\n",
    "\n",
    "data_question['createday'] = np.nan\n",
    "data_question['createhour'] = np.nan\n",
    "data_question['createday'] = data_question['createtime'].apply(lambda x: int(x.split('-')[0][1:]))\n",
    "data_question['createhour'] = data_question['createtime'].apply(lambda x: int(x.split('-')[0][1:]))\n",
    "data_question = memoryOptimization(data_question,np.float32)\n",
    "\n",
    "data_question.to_csv(datapath+'data/data_question.csv',header = True,index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "问题绑定的话题相当于问题的tag字段，直接做countvector成类似onehot的形式维度过大，所以这里采用利用给定话题64维向量pca降维压缩并聚合的方式表示问题tag，利用pca后得到的话题的22维embedding，得到问题关于话题的一个embdeding表示，具体方法是：  \n",
    "1）将问题绑定的多个话题的对应22维em加和取平均  \n",
    "2）如果问题绑定的话题为缺失，即‘-1’，则对应的emb为nan  \n",
    "代码如下："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t2v_pca = pd.read_csv(datapath+'data/theme_vector_pca.csv')\n",
    "cols = list(t2v_pca.columns)\n",
    "cols.remove('themeId')\n",
    "cols = ['themeId']+cols\n",
    "t2v_pca = t2v_pca[cols]\n",
    "dic_t2v_pca = {}\n",
    "for row in iter(t2v_pca.values):\n",
    "    dic_t2v_pca[row[0]] = row[1:]\n",
    "\n",
    "def get_data_themeembs(data):\n",
    "    result = []\n",
    "    for themes in iter(data['themeId'].values):\n",
    "        if themes == '-1':\n",
    "            result.append([])\n",
    "            continue\n",
    "        cur = np.zeros(int(64*0.35))\n",
    "        themes = themes.split(',')\n",
    "        for theme in iter(themes):\n",
    "            cur = cur + dic_t2v_pca[theme]\n",
    "        cur = cur/len(themes)\n",
    "        result.append(list(cur))\n",
    "    return pd.DataFrame(result,columns = ['themeId'+str(i) for i in range(int(64*0.35))])\n",
    "\n",
    "q_theme = parallelize_dataframe(data_question,get_data_themeembs)\n",
    "q_theme.to_csv(datapath+'data/question_theme.csv',header = True,index = False)\n",
    "\n",
    "del data_question\n",
    "del q_theme\n",
    "del t2w_pca\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对问题title跟描述利用TFIDF进行过滤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_idf(corpus,outputfile):\n",
    "    ignored = {'', ' ', '', '。', '：', '，', '）', '（', '！', '?', '”', '“'}\n",
    "    id_freq = {}\n",
    "    i = 0\n",
    "    for doc in corpus:\n",
    "        doc = set(x for x in doc if x not in ignored)\n",
    "        for x in doc:\n",
    "            id_freq[x] = id_freq.get(x, 0) + 1\n",
    "        i += 1\n",
    "    with open(outputfile, 'w', encoding='utf-8') as f:\n",
    "        for key, value in id_freq.items():\n",
    "            f.write(key + ' ' + str(math.log(i / value, 2)) + '\\n')\n",
    "            \n",
    "class IDFLoader(object):\n",
    "    def __init__(self, idf_path):\n",
    "        self.idf_path = idf_path\n",
    "        self.idf_freq = {}     # idf\n",
    "        self.mean_idf = 0.0    # 均值\n",
    "        self.load_idf()\n",
    "\n",
    "    def load_idf(self):       # 从文件中载入idf\n",
    "        cnt = 0\n",
    "        with open(self.idf_path, 'r', encoding='utf-8') as f:\n",
    "            for line in f:\n",
    "                try:\n",
    "                    word, freq = line.strip().split(' ')\n",
    "                    cnt += 1\n",
    "                except Exception as e:\n",
    "                    pass\n",
    "                self.idf_freq[word] = float(freq)\n",
    "\n",
    "        print('Vocabularies loaded: %d' % cnt)\n",
    "        self.mean_idf = sum(self.idf_freq.values()) / cnt\n",
    "        \n",
    "class TFIDF(object):\n",
    "    def __init__(self, idf_path):\n",
    "        self.idf_loader = IDFLoader(idf_path)\n",
    "        self.idf_freq = self.idf_loader.idf_freq\n",
    "        self.mean_idf = self.idf_loader.mean_idf\n",
    "\n",
    "    def extract_sentence_keywords(self, sentence,filter_word=None,topK=None,all_tfidf = False):    # 提取关键词\n",
    "        # 过滤\n",
    "        #seg_list = segment(sentence)\n",
    "        seg_list = [x for x in sentence if len(x)>1]\n",
    "        freq = {}\n",
    "        for w in seg_list:\n",
    "            freq[w] = freq.get(w, 0.0) + 1.0\n",
    "        total = sum(freq.values())\n",
    "\n",
    "        for k in freq:   # 计算 TF-IDF\n",
    "            freq[k] *= self.idf_freq.get(k, self.mean_idf) / total\n",
    "\n",
    "\n",
    "        tags = sorted(freq, key=freq.__getitem__, reverse=True)  # 排序\n",
    "        if filter_word!=None:\n",
    "            tags = [x for x in tags if x not in filter_word]\n",
    "        if topK!=None:\n",
    "            if all_tfidf:\n",
    "                return tags[:topK],freq\n",
    "            else:\n",
    "                return tags[:topK]\n",
    "        else:\n",
    "            if all_tfidf:\n",
    "                return tags,freq\n",
    "            else:\n",
    "                return tags\n",
    "            \n",
    "    def extract_corpus_keywords(self, corpus, filter_word=None,topK=None,all_tfidf = False):    # 提取关键词\n",
    "        # 过滤\n",
    "        #seg_list = segment(sentence)\n",
    "        all_tags = []\n",
    "        all_freq = []\n",
    "        for sentence in corpus:\n",
    "            seg_list = [x for x in sentence if len(x)>1]\n",
    "            freq = {}\n",
    "            for w in seg_list:\n",
    "                freq[w] = freq.get(w, 0.0) + 1.0\n",
    "            total = sum(freq.values())\n",
    "\n",
    "            for k in freq:   # 计算 TF-IDF\n",
    "                freq[k] *= self.idf_freq.get(k, self.mean_idf) / total\n",
    "            if all_tfidf:\n",
    "                all_freq.append(freq)\n",
    "            tags = sorted(freq, key=freq.__getitem__, reverse=True)  # 排序\n",
    "            if filter_word!=None:\n",
    "                tags = [x for x in tags if x not in filter_word]\n",
    "            if topK!=None:\n",
    "                 all_tags.append(tags[:topK])\n",
    "            else:\n",
    "                all_tags.append(tags)\n",
    "        if all_tfidf:            \n",
    "            return all_tags,all_freq\n",
    "        else:\n",
    "            return all_tags\n",
    "\n",
    "\n",
    "question_info = pd.read_csv(datapath+'data/question_info.txt', header=None, sep='\\t')\n",
    "question_info.columns = ['问题id','问题创建时间','问题标题单字编码','问题标题切词编码','问题描述单字编码','问题描述切词编码','问题绑定话题']\n",
    "question_info['len'] = question_info['问题标题切词编码'].apply(lambda x:len(x.split(',')))\n",
    "question_info['len'].max()\n",
    "\n",
    "def text(row):\n",
    "    text = row.问题标题切词编码.split(',')\n",
    "    if row.问题描述切词编码!=str(-1):\n",
    "        text.extend(row.问题描述切词编码.split(','))\n",
    "    return text\n",
    "question_info['text'] = question_info.apply(lambda row:text(row),axis=1)\n",
    "question_info['title'] =  question_info['问题标题切词编码'].apply(lambda x:x.split(','))\n",
    "\n",
    "out_file = datapath+'data/问题标题.idf.txt'\n",
    "if not os.path.exists(out_file):\n",
    "    corpus = question_info.title.values.tolist()\n",
    "    gen_idf(corpus,out_file)\n",
    "tdidf = TFIDF(out_file)\n",
    "x = question_info.title.values.tolist()\n",
    "tags = tdidf.extract_corpus_keywords(x)\n",
    "question_info['tfidf_title'] = tags\n",
    "del x,tags\n",
    "gc.collect()\n",
    "\n",
    "out_file = datapath+'data/问题标题描述.idf.txt'\n",
    "if not os.path.exists(out_file):\n",
    "    corpus = question_info.text.values.tolist()\n",
    "    gen_idf(corpus,out_file)\n",
    "tdidf = TFIDF(out_file)\n",
    "x = question_info.text.values.tolist()\n",
    "tags = tdidf.extract_corpus_keywords(x)\n",
    "question_info['tfidf_text'] = tags\n",
    "demo_data = question_info[['问题id','len','tfidf_text','tfidf_title','问题描述切词编码']]\n",
    "\n",
    "def get_topk(row):\n",
    "    ### title\n",
    "#     if row.问题标题切词编码==str(-1):\n",
    "#         return\n",
    "    n = round(row.len/3)\n",
    "    if n==0:\n",
    "        n=1\n",
    "    row['title_topk'] = row.tfidf_title[:n]\n",
    "    if row.问题描述切词编码==str(-1):\n",
    "        row['text_topk'] = row.tfidf_text[:n]\n",
    "    else:\n",
    "        if row.tfidf_title[0]=='-1':\n",
    "            m = min(4,round(2*len(row.tfidf_text)/3))\n",
    "            row['text_topk'] = row.tfidf_text[:m]\n",
    "        else:\n",
    "            row['text_topk'] = row.tfidf_text[:round(2*row.len/3)]\n",
    "    return row\n",
    "\n",
    "\n",
    "def parallelize_dataframe(df, func):\n",
    "    df_split = np.array_split(df, 16)\n",
    "    pool = Pool(16)\n",
    "    df = pd.concat(pool.map(func, df_split))\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    return df\n",
    "def get_topk_all(df):\n",
    "    df = df.apply(lambda row:get_topk(row),axis=1)\n",
    "    return df\n",
    "\n",
    "demo_data = parallelize_dataframe(demo_data, get_topk_all)\n",
    "demo_data['问题标题切词编码']=question_info['问题标题切词编码']\n",
    "\n",
    "def clean_1(ls):\n",
    "    if len(ls)==1 and ls[0]=='-1':\n",
    "        return ls\n",
    "    ls = [x for x in ls if x!='-1']\n",
    "    return ls\n",
    "\n",
    "x = demo_data[[\"问题id\",'title_topk','text_topk']]\n",
    "x['text_topk'] = x['text_topk'].apply(lambda x:clean_1(x))\n",
    "x['title_topk'] = x['title_topk'].apply(lambda s:','.join(s))\n",
    "x['text_topk'] = x['text_topk'].apply(lambda s:','.join(s))\n",
    "x.to_csv(datapath+'data/data_q_title_tfidf.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用户文件\n",
    "\n",
    "处理用户信息文件,  \n",
    "drop掉keywords等没有意义的列；  \n",
    "将类别特征中的‘unknown’替换为nan；  \n",
    "构建用户最感兴趣的主题；"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns = ['writerId','sex','keywords','publishrank','heatrank','registertype','platform','activity','bool_A','bool_B','bool_C',\n",
    "          'bool_D','bool_E','category_A','category_B','category_C','category_D','category_E','yanzhi','attentionthemes','likethemes']\n",
    "data_writer = pd.read_csv(datapath+'data/member_info.txt',sep = '\\t',names = columns)\n",
    "data_writer.drop(['keywords','publishrank', 'heatrank', 'registertype', 'platform'],axis = 1)\n",
    "data_writer[data_writer.select_dtypes(include = 'object').columns] = data_writer.select_dtypes(include = 'object').applymap(lambda x: float('nan') if x == 'unknown' else x)\n",
    "\n",
    "def mostliketheme(x):\n",
    "    if x == '-1':\n",
    "        return '-1'\n",
    "    for theme in iter(x.strip().split(',')):\n",
    "        theme = theme.strip().split(':')\n",
    "        try:\n",
    "            if float(theme[1])>biggestlike:\n",
    "                biggestlike = theme[1]\n",
    "                mostliketheme = theme[0]\n",
    "        except:\n",
    "            biggestlike = theme[1]\n",
    "            mostliketheme = theme[0]\n",
    "    return mostliketheme\n",
    "\n",
    "data_writer['mostliketheme'] = data_writer['likethemes'].apply(mostliketheme)\n",
    "data_writer = memoryOptimization(data_writer,np.float32)\n",
    "data_writer.to_csv(datapath+'data/data_writer.csv',header = True,index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "与问题文件类似的，利用话题的22维emb分别得到用户关于用户关注话题的emb以及关于用户感兴趣的话题的emb，  \n",
    "其中感兴趣话题的聚合方式是加权平均，权重是用户对该话题感兴趣程度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t2v_pca = pd.read_csv(datapath+'data/theme_vector_pca.csv')\n",
    "cols = list(t2v_pca.columns)\n",
    "cols.remove('themeId')\n",
    "cols = ['themeId']+cols\n",
    "t2v_pca = t2v_pca[cols]\n",
    "dic_t2v_pca = {}\n",
    "for row in iter(t2v_pca.values):\n",
    "    dic_t2v_pca[row[0]] = row[1:]\n",
    "\n",
    "def get_data_themeembs(data):\n",
    "    result = []\n",
    "    for themes in iter(data['attentionthemes'].values):\n",
    "        if themes == '-1':\n",
    "            result.append([])\n",
    "            continue\n",
    "        cur = np.zeros(int(64*0.35))\n",
    "        themes = themes.split(',')\n",
    "        for theme in iter(themes):\n",
    "            cur = cur + dic_t2v_pca[theme]\n",
    "        cur = cur/len(themes)\n",
    "        result.append(list(cur))\n",
    "    return pd.DataFrame(result,columns = ['attentionthemes'+str(i) for i in range(int(64*0.35))])\n",
    "\n",
    "writer_attentionthemes = parallelize_dataframe(data_writer,get_data_themeembs)\n",
    "writer_attentionthemes.to_csv(datapath+'data/writer_attentiontheme.csv',header = True,index = False)\n",
    "\n",
    "def get_data_themeembs_weight(data):\n",
    "    def f(s,data_v,leix):\n",
    "        result = np.zeros(22)\n",
    "        if s == '-1':\n",
    "            return result\n",
    "        s = s.strip().split(',')\n",
    "        for t in iter(s):\n",
    "            t = t.strip().split(':')\n",
    "            result = result+data_v.loc[data_v[leix+'Id'] == t[0],[leix+'_'+str(i) for i in range(22)]].values[0]*float(t[1])\n",
    "        try:\n",
    "            result = result/len(s)\n",
    "        except:\n",
    "            pass\n",
    "        return result\n",
    "    data_v = t2v_pca\n",
    "    col = 'likethemes'\n",
    "    leix = 'theme'\n",
    "    return pd.DataFrame(list(data.apply(lambda x:f(x,data_v,leix))),columns = [col+str(i) for i in range(int(64*0.35))])\n",
    "\n",
    "writer_likethemes = parallelize_dataframe(data_writer['likethemes'],get_data_themeembs_weight)\n",
    "writer_likethemes.to_csv(datapath+'data/writer_liketheme.csv',header = True,index = False)\n",
    "\n",
    "del data_writer\n",
    "del writer_likethemes\n",
    "del writer_attentionthemes\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练集\n",
    "处理训练集，根据邀请时间字段生成邀请的天、时字段 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_invite = pd.read_csv(datapath+'data/invite_info.txt',sep = '\\t',header = None,names = ['qId','writerId','invitetime','label'])\n",
    "data_invite['inviteday'] = np.nan\n",
    "data_invite['invitehour'] = np.nan\n",
    "data_invite['inviteday'] = data_invite.invitetime.apply(lambda x: int(x.split('-')[0][1:]))\n",
    "data_invite['invitehour'] = data_invite.invitetime.apply(lambda x: int(x.split('-')[1][1:]))\n",
    "data_invite = memoryOptimization(data_invite,np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "根据官方的说法一个问题只会对同一个用户邀请一次，训练集中存在2k个左右重复的样本，将这部分样本去重，保留邀请时间最早的样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_invite['index'] = data_invite.index\n",
    "data_invite = data_invite.sort_values(by = ['inviteday','invitehour']).reset_index(drop = True).drop_duplicates(['qId','writerId'],keep = 'first')\n",
    "data_invite = data_invite.sort_values(by = 'index').reset_index(drop = True)\n",
    "del data_invite['index']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "删除回答和邀请时间>10天的正样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_answer = pd.read_csv(datapath+'data/data_answer.csv')[['qId','writerId','answerday','answerhour']]\n",
    "data_invite = data_invite.merge(data_answer[['qId','writerId','answerday','answerhour']],on = ['writerId','qId'],how = 'left')\n",
    "data_invite['deltday'] = data_invite['answerday']-data_invite['inviteday']\n",
    "data_invite = data_invite[~(data_invite['deltday']>10)].reset_index(drop = True)\n",
    "del data_invite['answerday']\n",
    "del data_invite['answerhour']\n",
    "del data_invite['deltday']\n",
    "del data_answer\n",
    "data_invite.to_csv(datapath+'data/data_invite.csv',header = True,index = False)\n",
    "del data_invite\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试集1&2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "处理测试集1和测试集2，根据邀请时间字段生成邀请的天、时字段"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_eval = pd.read_csv(datapath+'data/invite_info_evaluate_1.txt',sep = '\\t',header = None,names = ['qId','writerId','invitetime','label'])\n",
    "data_eval['inviteday'] = np.nan\n",
    "data_eval['invitehour'] = np.nan\n",
    "data_eval['inviteday'] = data_eval.invitetime.apply(lambda x: int(x.split('-')[0][1:]))\n",
    "data_eval['invitehour'] = data_eval.invitetime.apply(lambda x: int(x.split('-')[1][1:]))\n",
    "data_eval = memoryOptimization(data_eval,np.float64)\n",
    "data_eval.to_csv(datapath+'data/data_invite_eval.csv',header = True,index = False)\n",
    "del data_eval\n",
    "\n",
    "data_test = pd.read_csv(datapath+'data/invite_info_evaluate_2_0926.txt',sep = '\\t',header = None,names = ['qId','writerId','invitetime','label'])\n",
    "data_test['inviteday'] = np.nan\n",
    "data_test['invitehour'] = np.nan\n",
    "data_test['inviteday'] = data_test.invitetime.apply(lambda x: int(x.split('-')[0][1:]))\n",
    "data_test['invitehour'] = data_test.invitetime.apply(lambda x: int(x.split('-')[1][1:]))\n",
    "data_test = memoryOptimization(data_test,np.float64)\n",
    "data_test.to_csv(datapath+'data/data_invite_test.csv',header = True,index = False)\n",
    "del data_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二、数据拼接  \n",
    "拼接train、test1、test2得到完整数据data，将用户信息和问题信息merge到data上"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_train = pd.read_csv(datapath+'data/data_invite.csv')#train\n",
    "data_test1 = pd.read_csv(datapath+'data/data_invite_eval.csv')#test1\n",
    "data_test2 = pd.read_csv(datapath+'data/data_invite_test.csv')#test2\n",
    "data_train['type'] = 'train'\n",
    "data_test1['type'] = 'test1'\n",
    "data_test2['type'] = 'test2'\n",
    "data = pd.concat([data_train,data_test1,data_test2],axis = 0,ignore_index = True)\n",
    "data['label'] = data['label'].fillna(-1)\n",
    "data = data.drop(['invitetime'],axis = 1)\n",
    "data = memoryOptimization(data,np.float32)\n",
    "del data_test1\n",
    "del data_test2\n",
    "del data_train\n",
    "gc.collect()\n",
    "\n",
    "data_question = pd.read_csv(datapath+'data/data_question.csv')\n",
    "question_theme = pd.read_csv(datapath+'data/question_theme.csv')\n",
    "data_question = pd.concat([data_question,question_theme],axis = 1)\n",
    "data_question = data_question.drop(['createtime','title_letters','title_words','describe_letters','describe_words'],axis = 1)\n",
    "data_question = memoryOptimization(data_question,np.float32)\n",
    "del question_theme\n",
    "\n",
    "data_writer = pd.read_csv(datapath+'data/data_writer.csv')\n",
    "writer_attentiontheme = pd.read_csv(datapath+'data/writer_attentiontheme.csv')\n",
    "data_writer = pd.concat([data_writer,writer_attentiontheme],axis = 1)\n",
    "data_writer = memoryOptimization(data_writer,np.float32)\n",
    "del writer_attentiontheme\n",
    "gc.collect()\n",
    "\n",
    "data = pd.merge(data,data_question,how = 'left',on = 'qId')\n",
    "data = pd.merge(data,data_writer,how = 'left',on = 'writerId')\n",
    "data['inviteallhour'] = (data['inviteday']-3800)*24+data['invitehour']\n",
    "data['inviteweekday'] = getweekday(data['inviteday'])\n",
    "data['createweekday'] = getweekday(data['createday'])\n",
    "del data_question\n",
    "del data_writer\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# .特征工程\n",
    "\n",
    "## 一、单一侧特征（用户侧、问题侧）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1、计数类特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1）滑窗统计特征，对id及类别特征统计过去7天的邀请数，反应过去一周的邀请情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_crossfeas_inv(data,fea1,fea2):\n",
    "    dataf = data[[fea1,fea2]].copy()\n",
    "    bool_s = (~dataf[fea1].isna())&(~dataf[fea2].isna())\n",
    "    dataf['cross'] = np.nan\n",
    "    dataf.loc[bool_s,'cross'] = dataf.loc[bool_s,fea1].apply(str)+'_'+dataf.loc[bool_s,fea2].apply(str)\n",
    "    return dataf['cross'].values\n",
    "\n",
    "def lastndayinvite(dataf,n,feas,use_weight):\n",
    "    dicfea = feas[0]\n",
    "    if len(feas)>1:\n",
    "        fea = dicfea\n",
    "        for i in feas[1:]:\n",
    "            fea = fea+'_'+i\n",
    "        dataf[fea] = get_crossfeas_inv(dataf,feas[0],feas[1])\n",
    "    else:\n",
    "        fea = dicfea\n",
    "    \n",
    "    if use_weight:\n",
    "        gps = dataf.groupby(['inviteday'])\n",
    "        dic = {}\n",
    "        for gp_id in iter(dataf['inviteday'].unique()):\n",
    "            gp = gps.get_group(gp_id)\n",
    "            dic[gp_id] = gp.shape[0]/gp[dicfea].nunique()\n",
    "        dic = pd.Series(dic)\n",
    "        dic = (dic.mean()/dic).round(3).to_dict()\n",
    "    \n",
    "    data_gps = dataf.groupby([fea,'inviteday']).size().astype(float).reset_index()\n",
    "    if use_weight:\n",
    "        values_0 = []\n",
    "        for row in iter(data_gps[['inviteday',0]].values):\n",
    "            values_0.append(dic[row[0]]*row[1])\n",
    "\n",
    "        data_gps[0] = values_0\n",
    "    data_gps = data_gps.rename(columns = {0:'size'})\n",
    "    \n",
    "    result = dataf[[]]\n",
    "    result['val'] = np.nan\n",
    "    for day in iter(dataf['inviteday'].unique()):\n",
    "        result.loc[dataf['inviteday'] == day,'val'] = get_invite_count(data_gps[(data_gps['inviteday']<=day)&(data_gps['inviteday']>day-n)],dataf[dataf['inviteday'] == day],fea)\n",
    "    result.loc[(~dataf[fea].isna())&(result['val'].isna()),'val'] = 0\n",
    "    \n",
    "    if len(feas)>1:\n",
    "        del dataf[fea]\n",
    "    gc.collect()\n",
    "    \n",
    "    return result['val'].values\n",
    "\n",
    "def get_invite_count(df_train,df_test,fea):\n",
    "    df_train = df_train[[fea,'size']].groupby(fea).sum()['size'].reset_index()\n",
    "    new_fea_name = fea+'_count'\n",
    "    df_train.columns = [fea,new_fea_name]\n",
    "    df_test = df_test[[fea]].merge(df_train,on = fea,how = 'left')\n",
    "    \n",
    "    return df_test[new_fea_name].values\n",
    "\n",
    "#7天邀请数，代表用户被邀请的频率\n",
    "n = 7\n",
    "for fea in ['qId','writerId']+['category_C','sex','activity']:#'invitehour','createhour','activity','bool_A','bool_B','bool_C','bool_D','bool_E','category_A','yanzhi'\n",
    "    df[fea+'_last%sday_count' %n] = lastndayinvite(data,n,[fea],use_weight = False)\n",
    "    print(fea+' is ok')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对盐值等频分箱后作为类别变量处理，也可以等间距分箱、卡方分箱等，或者直接取整后作为类别变量；  \n",
    "另外invitehour等时间类数值特征也可以分箱，这里只考虑了盐值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def split_data(dataf,col,split_num):\n",
    "#     dataf = dataf.copy()\n",
    "    count = dataf.shape[0]\n",
    "    n = math.ceil(count/split_num)\n",
    "    split_index = [i*n for i in range(1,split_num)]\n",
    "    values = sorted(list(dataf[col]))\n",
    "    split_point = [values[i] for i in split_index]\n",
    "    split_point = sorted(list(set(split_point)))\n",
    "    return split_point\n",
    "\n",
    "def get_group(x,split_bin):\n",
    "    n = len(split_bin)\n",
    "    if x <= min(split_bin):\n",
    "        return min(split_bin)\n",
    "    elif x> max(split_bin):\n",
    "        return max(split_bin)+max(split_bin)/n\n",
    "    else:\n",
    "        for i in range(n-1):\n",
    "            if split_bin[i] < x <= split_bin[i+1]:\n",
    "                return split_bin[i+1]\n",
    "\n",
    "points = split_data(data,'yanzhi',split_num = 10)\n",
    "data['yanzhi_d'] = data['yanzhi'].apply(lambda x:get_group(x,points)).apply(int)\n",
    "df['yanzhi_d_last%sday_count' %n] = lastndayinvite(data,n,['yanzhi_d'],use_weight = False)\n",
    "print('yanzhi_d is ok')\n",
    "del data['yanzhi_d']\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2）滑窗统计特征，过去三天每天分别的邀请量计数统计，反应的是对应用户或问题近期的邀请情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_last3dayinvitenum(dataf,df,fea,use_weight):\n",
    "    if use_weight:\n",
    "        gps = dataf.groupby(['inviteday'])\n",
    "        dic = {}\n",
    "        for gp_id in iter(dataf['inviteday'].unique()):\n",
    "            gp = gps.get_group(gp_id)\n",
    "            dic[gp_id] = gp.shape[0]/gp[fea].nunique()\n",
    "        dic = pd.Series(dic)\n",
    "        dic = (dic.mean()/dic).round(3).to_dict()\n",
    "        \n",
    "    data_gps = dataf.groupby(['inviteday',fea]).size().astype(float)\n",
    "    if use_weight:\n",
    "        for day in iter(dataf['inviteday'].unique()):#range(t0_eval,t1_eval+1):\n",
    "            data_gps[day] = data_gps[day]*dic[day]\n",
    "    data_gps = data_gps.reset_index().rename(columns = {0:'size'})\n",
    "    \n",
    "    for i in [fea+'_last3invnum'+str(i) for i in range(3)]+[fea+'_curdayinvnum']:\n",
    "        df[i] = np.nan\n",
    "    dic_result = {}\n",
    "    pool = Pool(10)\n",
    "    for day in iter(dataf['inviteday'].unique()):\n",
    "        dic_result[day] = pool.apply_async(func = get_last3invnum,args = (data_gps[(data_gps['inviteday']<=day)&(data_gps['inviteday']>=day-3)],dataf[dataf['inviteday'] == day],fea,day,))\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    \n",
    "    for day in iter(dataf['inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,[fea+'_last3invnum'+str(i) for i in range(3)]+[fea+'_curdayinvnum']] = dic_result[day].get()\n",
    "    \n",
    "    return df\n",
    "\n",
    "def get_last3invnum(data_train,data_test,fea,day):\n",
    "    data_train = data_train[data_train[fea].isin(data_test[fea].unique())]\n",
    "    gps = data_train.groupby(fea)\n",
    "    dic_fea = {}\n",
    "    daylist = [day-3,day-2,day-1,day]\n",
    "    for val in iter(data_train.reset_index()[fea].unique()):\n",
    "        gp = gps.get_group(val)\n",
    "        dic_val = gp['size']\n",
    "        dic_val.index = gp['inviteday'].values\n",
    "        dic_fea[val] = []\n",
    "        for day in iter(daylist):\n",
    "            try:\n",
    "                dic_fea[val].append(dic_val[day])\n",
    "            except:\n",
    "                dic_fea[val].append(0)\n",
    "        \n",
    "    dic_fea = pd.DataFrame(dic_fea).T.reset_index()\n",
    "    dic_fea.columns = [fea]+[fea+'_last3invnum'+str(i) for i in range(3)]+[fea+'_curdayinvnum']\n",
    "    data_test = data_test.merge(dic_fea,on = fea,how = 'left')\n",
    "    return data_test[[fea+'_last3invnum'+str(i) for i in range(3)]+[fea+'_curdayinvnum']].values\n",
    "\n",
    "for  fea in ['qId','writerId']:#,\n",
    "    df = get_last3dayinvitenum(data,df,fea,use_weight = False)\n",
    "    print(fea+' is ok')\n",
    "\n",
    "df['qId_last3+1invnum_mean'] = df[['qId_last3invnum0','qId_last3invnum1','qId_last3invnum2','qId_curdayinvnum']].mean(axis = 1)\n",
    "df['qId_last3+1invnum_std'] = df[['qId_last3invnum0','qId_last3invnum1','qId_last3invnum2','qId_curdayinvnum']].std(axis = 1)\n",
    "df['writerId_last3+1invnum_mean'] = df[['writerId_last3invnum0','writerId_last3invnum1','writerId_last3invnum2','writerId_curdayinvnum']].mean(axis = 1)\n",
    "df['writerId_last3+1invnum_std'] = df[['writerId_last3invnum0','writerId_last3invnum1','writerId_last3invnum2','writerId_curdayinvnum']].std(axis = 1)\n",
    "df = df.drop(['qId_curdayinvnum','writerId_curdayinvnum'],axis = 1)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3）当天的邀请数计数特征，当天的邀请数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_curdayinvitenum(dataf,fea,use_weight):\n",
    "    if use_weight:\n",
    "        gps = dataf.groupby(['inviteday'])\n",
    "        dic = {}\n",
    "        for gp_id in iter(dataf['inviteday'].unique()):\n",
    "            gp = gps.get_group(gp_id)\n",
    "            dic[gp_id] = gp.shape[0]/gp[fea].nunique()\n",
    "        dic = pd.Series(dic)\n",
    "        dic = (dic.mean()/dic).round(3).to_dict()\n",
    "    \n",
    "    data_gps = dataf.groupby(['inviteday',fea]).size().astype('float')\n",
    "    if use_weight:\n",
    "        for day in iter(dataf['inviteday'].unique()):#range(t0_eval,t1_eval+1):\n",
    "            data_gps[day] = data_gps[day]*dic[day]\n",
    "    data_gps = data_gps.reset_index().rename(columns = {0:'size'})\n",
    "    \n",
    "    result = data.merge(data_gps,on = [fea,'inviteday'],how = 'left')['size'].values\n",
    "    \n",
    "    return result\n",
    "\n",
    "for fea in ['qId','writerId','invitehour','createhour','createday','createweekday',\n",
    "            'sex','activity','bool_D','category_C','category_E','yanzhi','mostliketheme']:\n",
    "    df[fea+'_curdayinv_count'] = get_curdayinvitenum(data,fea,use_weight = False)\n",
    "    print(fea+' is ok')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4）全局计数特征：对类别特征，包括id，统计整个数据集的邀请数；虽然已经构造了历史计数特征，但全局的统计仍然有意义"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_alldata_count(data, fea, new_column_name,use_weight):#构造类别特征数量统计特征\n",
    "    if use_weight:\n",
    "        gps = data.groupby('inviteday')\n",
    "        dic = {}\n",
    "        for gp_id in iter(data['inviteday'].unique()):\n",
    "            gp = gps.get_group(gp_id)\n",
    "            dic[gp_id] = gp.shape[0]/gp[fea].nunique()\n",
    "        dic = pd.Series(dic)\n",
    "        dic = (dic.mean()/dic).round(3).to_dict()\n",
    "        \n",
    "    dataf = data[[fea,'inviteday']].groupby([fea,'inviteday']).size().astype(float).reset_index()\n",
    "    if use_weight:\n",
    "        values_0 = []\n",
    "        for row in iter(dataf[['inviteday',0]].values):\n",
    "            values_0.append(dic[row[0]]*row[1])\n",
    "        dataf[0] = values_0\n",
    "    dataf = dataf[[fea,0]].groupby(fea).sum()[0].reset_index()\n",
    "    \n",
    "    dataf = dataf.rename(columns = {0:new_column_name})\n",
    "    dataf = data.merge(dataf, on = fea, how = \"left\") \n",
    "    return dataf[new_column_name]\n",
    "\n",
    "for fea in ['qId','createday','createhour','invitehour','writerId','yanzhi','mostliketheme']:\n",
    "    df['%s_count' % fea] = get_alldata_count(data,fea,'%s_count' % fea,use_weight = False)\n",
    "    print(fea+' is ok')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5）用户或问题id的历史统计特征关于问题或用户id的平均值  \n",
    "实际该部分属于用户跟问题的交叉，而不是单一侧"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#过去7天的统计聚合\n",
    "df['writerId_last7count_gp_qId'] = df['writerId_last7day_count'].groupby(data['qId']).transform(np.mean)\n",
    "df['qId_last7count_gp_writerId'] = df['qId_last7day_count'].groupby(data['writerId']).transform(np.mean)\n",
    "\n",
    "#全局的统计集合\n",
    "df['writerId_count_gp_qId'] = df['writerId_count'].groupby(data['qId']).transform(np.mean)\n",
    "df['qId_count_gp_writerId'] = df['qId_count'].groupby(data['writerId']).transform(np.mean)\n",
    "\n",
    "#当天的统计聚合\n",
    "df['writerId_curcount_gp_qId'] = df['writerId_curdayinv_count'].groupby(data['qId']).transform(np.mean)\n",
    "df['qId_curcount_gp_writerId'] = df['qId_curdayinv_count'].groupby(data['writerId']).transform(np.mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_1.1.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2、目标编码特征\n",
    "  因为回答记录文件的记录时长是两个月，所以该部分利用回答记录文件统计历史的回答计数，利用训练集的标签统计历史点击率特征；  \n",
    "  构造该部分特征时注意数据时间穿越（泄露）问题；"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_answer = pd.read_csv(datapath+'data/data_answer.csv')\n",
    "# data_answer = data_answer[['qId','writerId','answerday','answerhour']]\n",
    "data_question = pd.read_csv(datapath+'data/data_question.csv')\n",
    "data_writer = pd.read_csv(datapath+'data/data_writer.csv')\n",
    "data_answer = pd.merge(data_answer,data_question,on = 'qId',how = 'left')\n",
    "data_answer = pd.merge(data_answer,data_writer,on = 'writerId',how = 'left')\n",
    "\n",
    "data_answer = data_answer.rename(columns = {'answerhour':'invitehour','answerday':'inviteday'})\n",
    "#为了便于构造特征，将问题回答记录中的回答时间名称改为邀请时间\n",
    "data_answer['inviteweekday'] = getweekday(data_answer['inviteday'])\n",
    "data_answer['createweekday'] = getweekday(data_answer['createday'])\n",
    "data_answer['label'] = 1\n",
    "del data_writer\n",
    "del data_question\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1）历史第七天用户和问题的目标编码特征，滑窗统计特征，包括回答数量以及邀请的接受率，id类特征构造的特征结果特别稀疏"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def slidewindow(dataf,data_dic,dayfea,func):\n",
    "    dataf = data.copy()\n",
    "    dataf['day_lastweek'] = dataf[dayfea[0]]-7\n",
    "    dic = data_dic.groupby(dayfea)['label'].agg(func).reset_index()\n",
    "    dic = dic.rename(columns = {dayfea[0]:'day_lastweek','label':'count'})\n",
    "    dataf = pd.merge(dataf,dic,on = ['day_lastweek']+dayfea[1:],how = 'left')\n",
    "    dataf.loc[(~dataf[datafea[1]].isna())&(dataf['count'].isna()),'count'] = 0\n",
    "    return dataf['count'].values\n",
    "\n",
    "for fea in ['qId','writerId']+['activity','category_A','category_D']:\n",
    "    for dayfea in ['inviteday','createday']:\n",
    "        str_week = dayfea.split('day')[0]+'weekday'\n",
    "        df['%s_%slastweek2label_count' % (fea,dayfea)] = slidewindow(data,data_answer,[dayfea,fea],func = np.sum)\n",
    "        df['%s_%slastweek2label_rate' % (fea,dayfea)] = slidewindow(data,data[data['label'] != -1],[dayfea,fea],func = np.mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2）历史目标编码特征，滑窗统计特征，labelcount选取时间窗口为3周，labelctr选取为整个历史训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_crossfeas_tar(data,data_asw,fea1,fea2):\n",
    "    dataf = pd.concat([data[[fea1,fea2]],data_asw[[fea1,fea2]]],axis = 0).reset_index(drop = True)\n",
    "    bool_s = (~dataf[fea1].isna())&(~dataf[fea2].isna())\n",
    "    dataf['cross'] = np.nan\n",
    "    dataf.loc[bool_s,'cross'] = dataf.loc[bool_s,fea1].apply(str)+'_'+dataf.loc[bool_s,fea2].apply(str)\n",
    "    cross = dataf['cross'].values\n",
    "    c1 = cross[:data.shape[0]]\n",
    "    c2 = cross[data.shape[0]:]\n",
    "    return c1,c2\n",
    "\n",
    "def targetencoder(dataf,data_asw,df,feas):\n",
    "    dicfea = feas[0]\n",
    "    if len(feas)>1:\n",
    "        fea = dicfea\n",
    "        for i in feas[1:]:\n",
    "            fea = fea+'_'+i\n",
    "        dataf[fea],data_asw[fea] = get_crossfeas_tar(dataf,data_asw,feas[0],feas[1])\n",
    "    else:\n",
    "        fea = dicfea\n",
    "\n",
    "#     dataf = dataf.copy()\n",
    "    gps_asw = data_asw.groupby('inviteday')\n",
    "    dic_asw = {}\n",
    "    for gp_id in iter(data_asw['inviteday'].unique()):\n",
    "        gp = gps_asw.get_group(gp_id)\n",
    "        dic_asw[gp_id] = gp.shape[0]/gp[dicfea].nunique()\n",
    "    dic_asw = pd.Series(dic_asw)\n",
    "    dic_asw = (dic_asw.mean()/dic_asw).round(3).to_dict()\n",
    "\n",
    "    data_asw = data_asw[[fea,'inviteday']].groupby([fea,'inviteday']).size().reset_index()\n",
    "    values_0 = []\n",
    "    for row in iter(data_asw[['inviteday',0]].values):\n",
    "        values_0.append(dic_asw[row[0]]*row[1])\n",
    "    data_asw[0] = values_0\n",
    "    data_asw = data_asw.rename(columns = {0:'size'})\n",
    "    \n",
    "    gps_inv = dataf.groupby('inviteday')\n",
    "    dic_inv = {}\n",
    "    for gp_id in iter(dataf['inviteday'].unique()):\n",
    "        gp = gps_inv.get_group(gp_id)\n",
    "        dic_inv[gp_id] = gp.shape[0]/gp[dicfea].nunique()\n",
    "    dic_inv = pd.Series(dic_inv)\n",
    "    dic_inv = (dic_inv.mean()/dic_inv).round(3).to_dict()\n",
    "    \n",
    "    data_gps = dataf[[fea,'inviteday','label']].groupby([fea,\n",
    "                'inviteday']).agg(['sum','count'])['label'].reset_index()\n",
    "    values_sum = []\n",
    "    values_count = []\n",
    "    for row in iter(data_gps[['inviteday','sum','count']].values):\n",
    "        values_sum.append(dic_inv[row[0]]*row[1])\n",
    "        values_count.append(dic_inv[row[0]]*row[2])\n",
    "    data_gps['sum'] = values_sum\n",
    "    data_gps['count'] = values_count\n",
    "    \n",
    "    dataf['timegroup'] = (dataf['inviteday']/3).apply(int)\n",
    "    data_gps['timegroup'] = (data_gps['inviteday']/3).apply(int)\n",
    "    data_train = dataf[dataf['label'] != -1]\n",
    "    data_test = dataf[dataf['label'] == -1]  \n",
    "    \n",
    "    df[fea+'_label_count'] = np.nan#np.zeros(dataf.shape[0])\n",
    "    daylen_asw = 21\n",
    "    for day in iter(data_train['inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,fea+'_label_count'] = get_label_count(data_asw[(data_asw['inviteday']<day)&(data_asw['inviteday']>=day-daylen_asw)],dataf[dataf['inviteday'] == day],fea)\n",
    "    df.loc[data_test.index,fea+'_label_count'] = get_label_count(data_asw[(data_asw['inviteday']<t0_eval)&(data_asw['inviteday']>=t0_eval-daylen_asw)],data_test,fea)\n",
    "    df.loc[(~dataf[fea].isna())&(df[fea+'_label_count'].isna()),fea+'_label_count'] = 0\n",
    "    \n",
    "    df[fea+'_label_ctr'] = np.nan#np.zeros(dataf.shape[0])\n",
    "    for gp in iter(data_train['timegroup'].unique()):\n",
    "        df.loc[(dataf['timegroup'] == gp)&(dataf['label'] != -1),fea+'_label_ctr'] = get_label_ctr(data_gps[data_gps['timegroup'] < gp],data_train[data_train['timegroup'] == gp],fea)\n",
    "    df.loc[data_test.index,fea+'_label_ctr'] = get_label_ctr(data_gps[data_gps['inviteday']<t0_eval],data_test,fea)\n",
    "    df.loc[(~dataf[fea].isna())&(df[fea+'_label_ctr'].isna()),fea+'_label_ctr'] = 0\n",
    "\n",
    "    if len(feas)>1:\n",
    "        del dataf[fea]\n",
    "        del data_asw[fea]\n",
    "    del dataf['timegroup']\n",
    "    del data_train\n",
    "    del data_test\n",
    "    gc.collect()\n",
    "    \n",
    "    return df\n",
    "\n",
    "def get_label_count(df_train,df_test,fea):\n",
    "    df_train = df_train[[fea,'size']].groupby(fea).sum()['size'].reset_index()\n",
    "    new_fea_name = fea+'_label_count'\n",
    "    df_train = df_train[[fea,'size']].rename(columns = {'size':new_fea_name})\n",
    "    df_test = df_test.merge(df_train,on = fea,how = 'left')\n",
    "    \n",
    "    return df_test[new_fea_name].values\n",
    "\n",
    "def get_label_ctr(df_train,df_test,fea):\n",
    "    df_train = df_train[[fea,'sum','count']].groupby(fea).sum()[['sum','count']].reset_index()\n",
    "    new_fea_name = fea+'_label_ctr'\n",
    "    df_train[new_fea_name] = (df_train['sum'] + 1) / (df_train['count'] + 1)\n",
    "    df_train = df_train[[fea,new_fea_name]]\n",
    "    df_test = df_test.merge(df_train,on = fea,how = 'left')\n",
    "    \n",
    "    return df_test[new_fea_name].values   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for fea in ['qId','createhour','invitehour','mostliketheme','writerId','yanzhi','category_C','category_D','activity']:\n",
    "    df = targetencoder(data,data_answer,df,[fea])\n",
    "    print(fea+' is ok')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3）多标签类别特征的目标编码  \n",
    "对多个tag的统计结果进行排序，取topk个作为k列特征，这样处理的目的在于过滤标签，许多标签是冗余的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def targetencoder_multi(dataf,data_asw,df,fea):\n",
    "    dataf['timegroup'] = (dataf['inviteday']/3).apply(int)\n",
    "    data_train = dataf[dataf['label'] != -1]\n",
    "    data_test = dataf[dataf['label'] == -1]\n",
    "    daylen_asw = 21\n",
    "    for i in range(3):\n",
    "        df[fea+'_label_count%s' %i] = np.nan#np.zeros(data.shape[0])\n",
    "    for day in iter(data_train['inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,[fea+'_label_count'+str(i) for i in range(3)]] = get_multi_label_count(data_asw[(data_asw['inviteday']<day)&(data_asw['inviteday']>=day-daylen_asw)],dataf[dataf['inviteday'] == day],fea)#\n",
    "    df.loc[data_test.index,[fea+'_label_count'+str(i) for i in range(3)]] = get_multi_label_count(data_asw[(data_asw['inviteday']<t0_eval)&(data_asw['inviteday']>=t0_eval-daylen_asw)],data_test,fea)#\n",
    "\n",
    "    for i in range(3):\n",
    "        df[fea+'_label_ctr%s' %i] = np.nan#np.zeros(data.shape[0])\n",
    "    for gp in iter(data_train['timegroup'].unique()):\n",
    "        try:\n",
    "            df.loc[(dataf['timegroup'] == gp)&(dataf['label'] != -1),[fea+'_label_ctr'+str(i) for i in range(3)]] = get_multi_label_ctr(data_train[data_train['timegroup'] < gp],data_train[data_train['timegroup'] == gp],fea)\n",
    "        except:\n",
    "            pass\n",
    "    df.loc[data_test.index,[fea+'_label_ctr'+str(i) for i in range(3)]] = get_multi_label_ctr(data_train,data_test,fea)\n",
    "    del dataf['timegroup']\n",
    "    \n",
    "    return df\n",
    "\n",
    "def get_multi_label_count(df_train,df_test,fea):\n",
    "    countall = {}\n",
    "    for row in iter(df_train[fea].values):\n",
    "        for theme in iter(row.strip().split(',')):\n",
    "            try:\n",
    "                countall[theme] = countall[theme] + 1\n",
    "            except:\n",
    "                countall[theme] = 1\n",
    "\n",
    "    result_count = []\n",
    "    for row in iter(df_test[fea].values):\n",
    "        row_count = []\n",
    "        for theme in iter(row.strip().split(',')):\n",
    "            try:\n",
    "                row_count.append(countall[theme])\n",
    "            except:\n",
    "                pass\n",
    "        row_count.sort(reverse = True)       \n",
    "        result_count.append(row_count[:3])\n",
    "        \n",
    "    result_count = pd.DataFrame(result_count)\n",
    "    for i in range(3):\n",
    "        if i not in result_count.columns:\n",
    "            result_count[i] = np.nan\n",
    "    return result_count.values\n",
    "\n",
    "def get_multi_label_ctr(df_train,df_test,fea):\n",
    "    count1 = {}\n",
    "    countall = {}\n",
    "    for row in iter(df_train[[fea,'label']].values):\n",
    "        for theme in iter(row[0].strip().split(',')):\n",
    "            try:\n",
    "                countall[theme] = countall[theme] + 1\n",
    "            except:\n",
    "                countall[theme] = 1\n",
    "            if row[1] == 1:\n",
    "                try:\n",
    "                    count1[theme] = count1[theme] + 1\n",
    "                except:\n",
    "                    count1[theme] = 1\n",
    "\n",
    "    result_ctr = []\n",
    "    for row in iter(df_test[fea].values):\n",
    "        row_ctr = []\n",
    "        for theme in iter(row.strip().split(',')):\n",
    "            try:\n",
    "                row_ctr.append((count1[theme]+1)/(countall[theme]+1))\n",
    "            except:\n",
    "                pass\n",
    "        row_ctr.sort(reverse = True)\n",
    "        result_ctr.append(row_ctr[:3])\n",
    "        \n",
    "    result_ctr = pd.DataFrame(result_ctr)\n",
    "    for i in range(3):\n",
    "        if i not in result_ctr.columns:\n",
    "            result_ctr[i] = np.nan\n",
    "    return result_ctr.values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for fea in ['themeId','attentionthemes']:\n",
    "    df = targetencoder_multi(data,data_answer,df,fea)\n",
    "    print(fea+' is ok')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对问题的标题利用tdidf过滤的结果作为问题的多标签类别特征，进行多标签目标编码："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_question = pd.read_csv(datapath+'data_q_title_tfidf.csv')\n",
    "data_question = data_question[['问题id','title_topk','text_topk']].rename(columns = {'问题id':'qId','title_topk':'title_words_tfidf0','text_topk':'title_words_tfidf1'})\n",
    "\n",
    "data = data.merge(data_question[['qId','title_words_tfidf1']],on = 'qId',how = 'left')\n",
    "data_answer = data_answer.merge(data_question[['qId','title_words_tfidf1']],on = 'qId',how = 'left')\n",
    "del data_question\n",
    "\n",
    "df = targetencoder_multi(data,data_answer,df,'title_words_tfidf1')\n",
    "del data['title_words_tfidf1']\n",
    "del data_answer['title_words_tfidf1']\n",
    "\n",
    "df['title_words_tfidf1_label_count_mean'] = df[['title_words_tfidf1_label_count0','title_words_tfidf1_label_count1','title_words_tfidf1_label_count2']].mean(axis = 1)\n",
    "df['title_words_tfidf1_label_ctr_mean'] = df[['title_words_tfidf1_label_ctr0','title_words_tfidf1_label_ctr1','title_words_tfidf1_label_ctr2']].mean(axis = 1)\n",
    "\n",
    "cols = ['title_words_tfidf1_label_ctr0','title_words_tfidf1_label_ctr1', 'title_words_tfidf1_label_ctr2','title_words_tfidf1_label_ctr_mean']\n",
    "df[cols] = df[cols]*1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4）分组统计特征，用户或问题id的目标编码特征关于问题或用户id的平均值 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['writerId_label_count_gp_qId'] = df['writerId_label_count'].groupby(data['qId']).transform(np.mean)\n",
    "df['qId_label_count_gp_writerId'] = df['qId_label_count'].groupby(data['writerId']).transform(np.mean)\n",
    "\n",
    "df['writerId_label_ctr_gp_qId'] = df['writerId_label_ctr'].groupby(data['qId']).transform(np.mean)\n",
    "df['qId_label_ctr_gp_writerId'] = df['qId_label_ctr'].groupby(data['writerId']).transform(np.mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_1.2.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df\n",
    "del data_answer\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3、其它特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1）邀请间隔时长：构造每条样本的问题id最近一次发出邀请到现在的时间间隔以及用户id最近一次被邀请到现在的时间间隔（单位小时）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_hourlenfromlastinv(dataf,df,fea):\n",
    "    result = []\n",
    "    last_invite = {}\n",
    "    for row in iter(dataf[[fea,'inviteallhour']].sort_values(by = 'inviteallhour',ascending = True).values):\n",
    "        try:\n",
    "            result.append(last_invite[row[0]])\n",
    "        except:\n",
    "            result.append(-1)\n",
    "        last_invite[row[0]] = row[1]\n",
    "\n",
    "    df1 = dataf[[]]\n",
    "    df1['seq'] = dataf.sort_values(by = 'inviteallhour').index\n",
    "    df1['lastinvitehour_'+fea] = result\n",
    "    df1 = df1.sort_values(by = 'seq')\n",
    "\n",
    "    df1 = df1.reset_index(drop = True)\n",
    "    del df1['seq']\n",
    "    df = pd.concat([df,df1],axis = 1)\n",
    "    del df1\n",
    "    gc.collect()\n",
    "\n",
    "    df.loc[df['lastinvitehour_'+fea] == -1,'lastinvitehour_'+fea] = None\n",
    "    df['hourlenfromlastinvite_'+fea] = dataf['inviteallhour'] - df['lastinvitehour_'+fea]\n",
    "    del df['lastinvitehour_'+fea]\n",
    "    \n",
    "    return df\n",
    "\n",
    "df = get_hourlenfromlastinv(data,df,'qId')\n",
    "df = get_hourlenfromlastinv(data,df,'writerId')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2）时效性：问题从创建当前邀请的存在时长，反应问题的时效性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['q_life'] = data['inviteday'] - data['createday']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3）活跃度：问题或用户近期的活跃度，用id近期有活动的天数表示，分为邀请活跃度以及回答活跃度，对于问题来说是问题近期是否有被邀请或者被回答，对于用户来说代表了用户近期是否更愿意回答或接收邀请"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_activeday(dataf,data_asw,df,fea):\n",
    "    df[fea+'_activeday_inv'] = np.nan\n",
    "    for day in iter(dataf['inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,fea+'_activeday_inv'] = get_daynum(dataf[dataf['inviteday']<day],dataf[dataf['inviteday'] == day],fea)\n",
    "    \n",
    "    df[fea+'_activeday_asw'] = np.nan\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1,'inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,fea+'_activeday_asw'] = get_daynum(data_asw[data_asw['inviteday']<day],dataf[dataf['inviteday'] == day],fea)\n",
    "    df.loc[dataf['label'] == -1,fea+'_activeday_asw'] = get_daynum(data_asw[data_asw['inviteday']<t0_eval],dataf[dataf['label'] == -1],fea)\n",
    "    \n",
    "    return df\n",
    "        \n",
    "def get_daynum(data_train,data_test,fea):\n",
    "    data_train = data_train[data_train[fea].isin(data_test[fea].unique())]\n",
    "    data_train = data_train.groupby([fea,'inviteday']).size().reset_index()[[fea,'inviteday']].groupby(fea).size().reset_index()\n",
    "    new_fea_name = fea+'_activeday_inv'\n",
    "    data_train = data_train.rename(columns = {0:new_fea_name})\n",
    "    data_test = data_test.merge(data_train,on = fea,how = 'left')\n",
    "    \n",
    "    return data_test[new_fea_name].values\n",
    "\n",
    "#活跃天数\n",
    "for fea in ['qId','writerId']:\n",
    "    df = get_activeday(data,data_answer,df,fea)\n",
    "    print(fea+' is ok')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_1.3.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二、交叉特征\n",
    "\n",
    "该部分主要用到协同过滤的思想，关键在于用户跟问题的交叉组合"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1、计数类特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1)id跟类别特征交叉计数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features=['sex','activity','bool_D','category_C','mostliketheme','invitehour','bool_A','bool_B','bool_C','bool_E',\n",
    "          'yanzhi','category_A','category_B','category_D','category_E']\n",
    "n = 7\n",
    "for fea in features:\n",
    "    df['qId'+'_'+fea+'_last%sday_count' %n] = lastndayinvite(data[['qId',fea,'inviteday']],n,['qId',fea],use_weight = False)\n",
    "    print(fea+' is ok')\n",
    "\n",
    "features=['invitehour','createday','createhour','createweekday']\n",
    "n = 7\n",
    "for fea in features:\n",
    "    df['writerId'+'_'+fea+'_last%sday_count' %n] = lastndayinvite(data[['writerId',fea,'inviteday']],n,['writerId',fea],use_weight = False)\n",
    "    print(fea+' is ok')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2）交叉计数分组统计特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features1 = ['qId_sex_last7day_count', 'qId_activity_last7day_count',\n",
    "       'qId_bool_D_last7day_count', 'qId_category_C_last7day_count',\n",
    "       'qId_mostliketheme_last7day_count', 'qId_invitehour_last7day_count',\n",
    "       'qId_bool_A_last7day_count', 'qId_bool_B_last7day_count',\n",
    "       'qId_bool_C_last7day_count', 'qId_bool_E_last7day_count',\n",
    "       'qId_yanzhi_last7day_count', 'qId_category_A_last7day_count',\n",
    "       'qId_category_B_last7day_count', 'qId_category_D_last7day_count',\n",
    "       'qId_category_E_last7day_count']\n",
    "features2 = ['writerId_invitehour_last7day_count','writerId_createday_last7day_count',\n",
    "             'writerId_createhour_last7day_count','writerId_createweekday_last7day_count']\n",
    "\n",
    "for fea in features1:\n",
    "    df[fea+'_gp_writerId'] = df[fea].groupby(data['writerId']).transform(np.mean)\n",
    "for fea in features2:\n",
    "    df[fea+'_gp_qId'] = df[fea].groupby(data['qId']).transform(np.mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_2.1.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2、目标编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_answer = pd.read_csv(datapath+'data/data_answer.csv')\n",
    "# data_answer = data_answer[['qId','writerId','answerday','answerhour']]\n",
    "data_question = pd.read_csv(datapath+'data/data_question.csv')\n",
    "data_writer = pd.read_csv(datapath+'data/data_writer.csv')\n",
    "data_answer = pd.merge(data_answer,data_question,on = 'qId',how = 'left')\n",
    "data_answer = pd.merge(data_answer,data_writer,on = 'writerId',how = 'left')\n",
    "\n",
    "data_answer = data_answer.rename(columns = {'answerhour':'invitehour','answerday':'inviteday'})\n",
    "#为了便于构造特征，将问题回答记录中的回答时间名称改为邀请时间\n",
    "data_answer['inviteweekday'] = getweekday(data_answer['inviteday'])\n",
    "data_answer['createweekday'] = getweekday(data_answer['createday'])\n",
    "del data_writer\n",
    "del data_question\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1）Id类与类别特征交叉的历史目标编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = ['bool_B','bool_C','bool_D','bool_E','invitehour','inviteweekday']\n",
    "#'qId_sex','qId_activity','qId_bool_A','qId_yanzhi','qId_category_A','qId_category_B','qId_category_C','qId_category_D','qId_category_E',\n",
    "#'qId_mostliketheme','writerId_createday',\n",
    "for fea in features:\n",
    "    df = targetencoder(data,data_answer,df,['qId',fea])\n",
    "    print(fea+' is ok')\n",
    "\n",
    "features = ['invitehour','inviteweekday','createhour','createweekday']\n",
    "#'qId_sex','qId_activity','qId_bool_A','qId_yanzhi','qId_category_A','qId_category_B','qId_category_C','qId_category_D','qId_category_E',\n",
    "#'qId_mostliketheme','writerId_createday',\n",
    "for fea in features:\n",
    "    df = targetencoder(data,data_answer,df,['writerId',fea])\n",
    "    print(fea+' is ok')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2）Id类与多标签类别特征交叉的历史目标编码，将count/ctr结果从大到小排序取topk，目的也是为了一定程度上的过滤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lastndaylabelcount_multi(dataf,data_asw,idfea,fea):  \n",
    "    gps = data_asw.groupby(['inviteday'])\n",
    "    dic = {}\n",
    "    for gp_id in iter(data_asw['inviteday'].unique()):\n",
    "        gp = gps.get_group(gp_id)\n",
    "        dic[gp_id] = gp.shape[0]#/gp[idfea].nunique()\n",
    "    dic = pd.Series(dic)\n",
    "    dic = (dic.mean()/dic).round(3).to_dict()\n",
    "    \n",
    "    result = dataf[[]]\n",
    "    result['val0'] = np.nan\n",
    "    result['val1'] = np.nan\n",
    "    result['val2'] = np.nan\n",
    "    result['val3'] = np.nan\n",
    "    result['val4'] = np.nan\n",
    "    daylen_asw = 21\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        result[dataf['inviteday'] == day] = get_multi_count(data_asw[(data_asw['inviteday']<day)&(data_asw['inviteday']>=day-daylen_asw)],dataf[dataf['inviteday'] == day],idfea,fea,dic)           \n",
    "    result[dataf['label'] == -1] = get_multi_count(data_asw[(data_asw['inviteday']<t0_eval)&(data_asw['inviteday']>=day-daylen_asw)],dataf[dataf['label'] == -1],idfea,fea,dic)\n",
    "    \n",
    "    result.columns = [idfea+'_'+fea+'_lastnday_labelcount' +str(i) for i in range(5)]\n",
    "    \n",
    "    return result\n",
    "\n",
    "def get_multi_count(df_train,df_test,idfea,fea,dic):\n",
    "    \n",
    "    day_max = df_test['inviteday'].min()\n",
    "    countall = {}\n",
    "    for row in iter(df_train[[idfea,fea,'inviteday']].values):\n",
    "        if row[1] == '-1':\n",
    "            continue\n",
    "        t = (1.5-0.3*np.floor((day_max-row[2])/3))#*dic[row[2]]#1#\n",
    "        if t<0.5:\n",
    "            t = 0.6\n",
    "        for theme in iter(row[1].strip().split(',')):\n",
    "            theme = row[0]+'_'+theme\n",
    "            try:\n",
    "                countall[theme] = countall[theme] + t\n",
    "            except:\n",
    "                countall[theme] = t\n",
    "    \n",
    "    result_count = []\n",
    "    for row in iter(df_test[[idfea,fea]].values):\n",
    "        row_count = []\n",
    "        if row[1] == '-1':\n",
    "            result_count.append([np.nan])\n",
    "            continue\n",
    "        for theme in iter(row[1].strip().split(',')):\n",
    "            theme = row[0]+'_'+theme\n",
    "            try:\n",
    "                row_count.append(countall[theme])\n",
    "            except:\n",
    "                row_count.append(0)\n",
    "        row_count.sort(reverse = True)     \n",
    "        result_count.append(row_count[:5])\n",
    "        \n",
    "    result_count = pd.DataFrame(result_count)\n",
    "    for i in range(5):\n",
    "        if i not in result_count.columns:\n",
    "            result_count[i] = np.nan\n",
    "    del countall\n",
    "    return result_count.values\n",
    "\n",
    "def lastndaylabelctr_multi(dataf,data_asw,idfea,fea):   \n",
    "\n",
    "    result = dataf[[]]\n",
    "    result['val0'] = np.nan\n",
    "    result['val1'] = np.nan\n",
    "    result['val2'] = np.nan\n",
    "    result['val3'] = np.nan\n",
    "    result['val4'] = np.nan\n",
    "\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        result[dataf['inviteday'] == day] = get_multi_ctr(data_asw[(data_asw['inviteday']<day)],dataf[dataf['inviteday'] == day],idfea,fea)\n",
    "    result[dataf['label'] == -1] = get_multi_ctr(data_asw[(data_asw['inviteday']<t0_eval)],dataf[dataf['label'] == -1],idfea,fea)\n",
    "    \n",
    "    result.columns = [idfea+'_'+fea+'_lastnday_labelctr' +str(i) for i in range(5)]\n",
    "    return result\n",
    "\n",
    "def get_multi_ctr(df_train,df_test,idfea,fea):\n",
    "    count1 = {}\n",
    "    countall = {}\n",
    "    day_max = df_test['inviteday'].min()\n",
    "    for row in iter(df_train[[idfea,fea,'inviteday','label']].values):\n",
    "        if row[1] == '-1':\n",
    "            continue\n",
    "        t = 1#dic[row[2]]#1.5-0.3*np.floor((day_max-row[2])/3)\n",
    "#         if t<0.5:\n",
    "#             t = 0.6\n",
    "        for theme in iter(row[1].strip().split(',')):\n",
    "            theme = row[0]+'_'+theme\n",
    "            try:\n",
    "                countall[theme] = countall[theme] + t\n",
    "            except:\n",
    "                countall[theme] = t\n",
    "            if row[3] == 1:\n",
    "                try:\n",
    "                    count1[theme] = count1[theme] + t\n",
    "                except:\n",
    "                    count1[theme] = t\n",
    "\n",
    "    result_ctr = []\n",
    "    for row in iter(df_test[[idfea,fea]].values):\n",
    "        row_ctr = []\n",
    "        if fea == '-1':\n",
    "            result_ctr.append([np.nan])\n",
    "            continue\n",
    "        \n",
    "        for theme in iter(row[1].strip().split(',')):\n",
    "            theme = row[0]+'_'+theme\n",
    "            try:\n",
    "                row_ctr.append((count1[theme]+0.1)/(countall[theme]+0.1))\n",
    "            except:\n",
    "                row_ctr.append(0)\n",
    "        row_ctr.sort(reverse = True)\n",
    "        result_ctr.append(row_ctr[:5])\n",
    "        \n",
    "    result_ctr = pd.DataFrame(result_ctr)\n",
    "    for i in range(5):\n",
    "        if i not in result_ctr.columns:\n",
    "            result_ctr[i] = np.nan\n",
    "    return result_ctr.values    \n",
    "\n",
    "\n",
    "\n",
    "df_this = data[[]]\n",
    "df_this = lastndaylabelcount_multi(data,data_answer,'writerId','themeId')\n",
    "df = pd.concat([df,df_this],axis = 1)\n",
    "df['writerId_themeId_lastnday_labelcount_mean'] = df[['writerId_themeId_lastnday_labelcount0', 'writerId_themeId_lastnday_labelcount1',\n",
    "                                           'writerId_themeId_lastnday_labelcount2', 'writerId_themeId_lastnday_labelcount3',\n",
    "                                           'writerId_themeId_lastnday_labelcount4']].mean(axis = 1)\n",
    "df['writerId_themeId_lastnday_labelcount_sum'] = df[['writerId_themeId_lastnday_labelcount0', 'writerId_themeId_lastnday_labelcount1',\n",
    "                                           'writerId_themeId_lastnday_labelcount2', 'writerId_themeId_lastnday_labelcount3',\n",
    "                                           'writerId_themeId_lastnday_labelcount4']].sum(axis = 1)\n",
    "\n",
    "df_this = data[[]]\n",
    "df_this = lastndaylabelctr_multi(data,data,'writerId','themeId')\n",
    "df = pd.concat([df,df_this],axis = 1)\n",
    "df['writerId_themeId_lastnday_labelctr_mean'] = df[['writerId_themeId_lastnday_labelctr0', 'writerId_themeId_lastnday_labelctr1',\n",
    "                                           'writerId_themeId_lastnday_labelctr2', 'writerId_themeId_lastnday_labelctr3',\n",
    "                                           'writerId_themeId_lastnday_labelctr4']].mean(axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对于问题标题的切词序列，采用过滤高频词的方式过滤后作为多标签类别特征进行交叉目标编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_lowfreq_word(data_q,del_words):\n",
    "    titles = []\n",
    "    for words in iter(data_q['title_words'].values):\n",
    "        if words == '-1':\n",
    "            titles.append('-1')\n",
    "            continue\n",
    "        words = words.split(',')\n",
    "        for word in iter(words):\n",
    "            if word in del_words:\n",
    "                words.remove(word)\n",
    "        words = ','.join(words)\n",
    "        titles.append(words)\n",
    "    return titles\n",
    "\n",
    "data_question = pd.read_csv(datapath+'data/data_question.csv')\n",
    "word_num = {}\n",
    "for words in iter(data_question['title_words'].values):\n",
    "    if words == '-1':\n",
    "        continue\n",
    "    words = words.split(',')\n",
    "    for word in iter(words):\n",
    "        try:\n",
    "            word_num[word] += 1\n",
    "        except:\n",
    "            word_num[word] = 1\n",
    "        \n",
    "word_num = pd.Series(word_num)\n",
    "# word_top100 = list(word_num.sort_values(ascending = False)[:100].index)\n",
    "# word_top50 = list(word_num.sort_values(ascending = False)[:50].index)\n",
    "word_top150 = list(word_num.sort_values(ascending = False)[:150].index)\n",
    "# word_top200 = list(word_num.sort_values(ascending = False)[:200].index)\n",
    "\n",
    "# data_question['title_words_deltop50'] = remove_lowfreq_word(data_question[['title_words']],word_top50)\n",
    "# data_question['title_words_deltop100'] = remove_lowfreq_word(data_question[['title_words']],word_top100)\n",
    "data_question['title_words_deltop150'] = remove_lowfreq_word(data_question[['title_words']],word_top150)\n",
    "# data_question['title_words_deltop200'] = remove_lowfreq_word(data_question[['title_words']],word_top200)\n",
    "\n",
    "data_question = data_question[['qId','title_words_deltop150']]\n",
    "data_question.loc[data_question['title_words_deltop150'] == '','title_words_deltop150'] = '-1'\n",
    "data = data.merge(data_question[['qId','title_words_deltop150']],on = 'qId',how = 'left')\n",
    "data_answer = data_answer.merge(data_question[['qId','title_words_deltop150']],on = 'qId',how = 'left')\n",
    "del data_question\n",
    "\n",
    "\n",
    "df_this = data[[]]\n",
    "df_this = lastndaylabelcount_multi(data,data_answer,'writerId','title_words_deltop150')\n",
    "df = pd.concat([df,df_this],axis = 1)\n",
    "\n",
    "df_this = data[[]]\n",
    "df_this = lastndaylabelctr_multi(data,data,'writerId','title_words_deltop150')\n",
    "df = pd.concat([df,df_this],axis = 1)\n",
    "del data['title_words_deltop150']\n",
    "del data_answer['title_words_deltop150']\n",
    "\n",
    "cols = ['writerId_title_words_deltop150_lastnday_labelctr0',\n",
    "       'writerId_title_words_deltop150_lastnday_labelctr1',\n",
    "       'writerId_title_words_deltop150_lastnday_labelctr2',\n",
    "       'writerId_title_words_deltop150_lastnday_labelctr3',\n",
    "       'writerId_title_words_deltop150_lastnday_labelctr4']\n",
    "df[cols] = df[cols]*1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_2.2.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3、问题聚类&目标编码"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对问题绑定话题的embedding加和取平均后得到问题的embedding，根据问题的embedding对问题进行聚类，得到问题的单标签话题类别特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t2v = {}\n",
    "with open(datapath+'data/topic_vectors_64d.txt') as f:\n",
    "    for line in f.readlines():\n",
    "        line = line.strip().split('\\t')\n",
    "        t2v[int(line[0][1:])] = np.array(list(map(float, line[1].strip().split(' '))))\n",
    "q_info = pd.read_csv(datapath+'data_question.csv')\n",
    "\n",
    "q2vec = {}\n",
    "for ind,row in q_info.iterrows():\n",
    "    t = row['topic'].split(',')\n",
    "    vec = np.zeros(64)\n",
    "    n = 0\n",
    "    for i in t:\n",
    "        i = int(i[1:])\n",
    "        if i in t2v:\n",
    "            vec += t2v[i]\n",
    "            n += 1\n",
    "    if n == 0:\n",
    "        q2vec[row['qid']] = vec\n",
    "    else:\n",
    "        q2vec[row['qid']] = vec/n\n",
    "\n",
    "# 使用kmean聚类\n",
    "clusters = 300 # 类别个数\n",
    "q2class = {}\n",
    "vectors_list = np.array([q2vec[qid] for qid in q2vec])\n",
    "kmean_model = KMeans(random_state = 2019, n_clusters=clusters)\n",
    "kmean_model.fit(vectors_list)\n",
    "q = list(q2vec.keys())\n",
    "cla = kmean_model.predict(vectors_list.reshape(-1,64))\n",
    "class2vec = kmean_model.cluster_centers_  # clusters*300 代表每个簇的中心\n",
    "\n",
    "qc_df = pd.DataFrame({'qid':q, 'class':list(cla)})\n",
    "qc_df.rename(columns = {'qid':'qId'}, inplace=True)\n",
    "qc_df['qId'] = qc_df['qId'].astype(str)\n",
    "qc_df['qId'] = 'Q'+qc_df['qId']\n",
    "qc_df.rename(columns={'class':'q_class_300'},inplace=True)\n",
    "\n",
    "data_answer = pd.merge(data_answer, qc_df, on='qId', how='left')\n",
    "data = pd.merge(data, qc_df, on='qId', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对聚类得到的问题单标签类别特征进行目标编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for fea in ['q_class_300']: #'q_class_100','q_class_300'\n",
    "    df = targetencoder(data[['writerId',fea,'inviteday','label']],data_answer,df,['writerId',fea])\n",
    "    del data_answer['writerId_'+fea]\n",
    "    print(fea+' is ok')\n",
    "for fea1 in ['q_class_300']:#'q_class_100','q_class_300'\n",
    "    for fea2 in ['bool_A','bool_B','bool_C','bool_E','category_D',]: #'sex','activity','bool_D','yanzhi','category_A','category_B','category_C','category_E','mostliketheme'\n",
    "        df = targetencoder(data[[fea1,fea2,'inviteday','label']],data_answer,df,[fea1,fea2])\n",
    "        del data_answer[fea1+'_'+fea2]\n",
    "        print(fea1+'_'+fea2+' is ok')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_2.3.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4、相似性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1)用户行为历史相似度，分别计算了当前待判断问题与用户历史回答问题的话题余弦相似度、标题余弦相似度以及历史邀请负样本话题的余弦相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cos_sim(vector_a, vector_b):\n",
    "\n",
    "    vector_a = np.mat(vector_a)\n",
    "    vector_b = np.mat(vector_b)\n",
    "    num = float(vector_a * vector_b.T)\n",
    "    denom = np.linalg.norm(vector_a) * np.linalg.norm(vector_b)\n",
    "    cos = num / denom\n",
    "    sim = 0.5 + 0.5 * cos\n",
    "    return sim\n",
    "\n",
    "def writerlastndayperformance_itemCF(dataf,data_asw,df,n,q2themes,q2words,dic_themes,dic_words):  \n",
    "    \n",
    "    scorefea = 'simhis_base_theme_theme_itemcf'\n",
    "    df[scorefea] = np.nan\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,scorefea] = get_sim_itemCF(data_asw[(data_asw['inviteday']<day)&(data_asw['inviteday']>=day-n)],dataf[dataf['inviteday'] == day],day,n,'themeId','themeId',q2themes,{},dic_themes,{})           \n",
    "    df.loc[dataf['label'] == -1,scorefea] = get_sim_itemCF(data_asw[(data_asw['inviteday']<t0_eval)&(data_asw['inviteday']>=t0_eval-n)],dataf[dataf['label'] == -1],t0_eval,n,'themeId','themeId',q2themes,{},dic_themes,{})\n",
    "    return df\n",
    "\n",
    "def writerlastndayperformance_itemCF_pool(dataf,data_asw,data_train_noclick,df,q2themes,q2words,dic_themes,dic_words):  \n",
    "    \n",
    "    #######sim_theme######\n",
    "    \n",
    "    scorefea = 'simhis_base_theme_theme_itemcf'\n",
    "    df[scorefea] = np.nan\n",
    "    n = 30\n",
    "\n",
    "    pool = Pool(2)\n",
    "    result = {}\n",
    "    result[t0_eval] = pool.apply_async(\n",
    "        func = get_sim_itemCF,\n",
    "        args = (data_asw.loc[(data_asw['inviteday']<t0_eval)&(data_asw['inviteday']>=t0_eval-n),['writerId','inviteday','invitehour','qId']].copy(),dataf.loc[dataf['label'] == -1,['writerId','qId']],t0_eval,n,'themeId','themeId',q2themes.copy(),{},dic_themes.copy(),{},False,)\n",
    "    )\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        result[day] = pool.apply_async(\n",
    "            func = get_sim_itemCF,\n",
    "            args = (data_asw.loc[(data_asw['inviteday']<day)&(data_asw['inviteday']>=day-n),['writerId','inviteday','invitehour','qId']].copy(),dataf.loc[dataf['inviteday'] == day,['writerId','qId']],day,n,'themeId','themeId',q2themes.copy(),{},dic_themes.copy(),{},False,)\n",
    "        )\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,scorefea] = result[day].get()        \n",
    "    df.loc[dataf['label'] == -1,scorefea] = result[t0_eval].get()\n",
    "    with open(datapath+'newfea_copy/sim_theme.pkl','wb') as f:\n",
    "        pickle.dump(df,f)\n",
    "\n",
    "    #######sim_title######\n",
    "    \n",
    "    scorefea = 'simhis_base_title_title_itemcf'\n",
    "    df[scorefea] = np.nan\n",
    "    n = 30\n",
    "\n",
    "    pool = Pool(2)\n",
    "    result = {}\n",
    "    result[t0_eval] = pool.apply_async(\n",
    "        func = get_sim_itemCF,\n",
    "        args = (data_asw.loc[(data_asw['inviteday']<t0_eval)&(data_asw['inviteday']>=t0_eval-n),['writerId','inviteday','invitehour','qId']].copy(),dataf.loc[dataf['label'] == -1,['writerId','qId']],t0_eval,n,'title_words_tfidf1','title_words_tfidf1',q2words.copy(),{},dic_words.copy(),{},False,)\n",
    "    )\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        result[day] = pool.apply_async(\n",
    "            func = get_sim_itemCF,\n",
    "            args = (data_asw.loc[(data_asw['inviteday']<day)&(data_asw['inviteday']>=day-n),['writerId','inviteday','invitehour','qId']].copy(),dataf.loc[dataf['inviteday'] == day,['writerId','qId']],day,n,'title_words_tfidf1','title_words_tfidf1',q2words.copy(),{},dic_words.copy(),{},False,)\n",
    "        )\n",
    "    \n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,scorefea] = result[day].get()        \n",
    "    df.loc[dataf['label'] == -1,scorefea] = result[t0_eval].get()\n",
    "    with open(datapath+'newfea_copy/sim_title.pkl','wb') as f:\n",
    "        pickle.dump(df,f)\n",
    "        \n",
    "    ######sim_noclick########\n",
    "    \n",
    "    scorefea = 'simhis_base_theme_theme_noclick'\n",
    "    df[scorefea] = np.nan\n",
    "    n = 15\n",
    "\n",
    "    data_asw = data_train_noclick[data_train_noclick['label'] == 0].reset_index(drop = True)\n",
    "    pool = Pool(2)\n",
    "    result = {}\n",
    "    result[t0_eval] = pool.apply_async(\n",
    "        func = get_sim_itemCF,\n",
    "        args = (data_asw.loc[(data_asw['inviteday']<t0_eval)&(data_asw['inviteday']>=t0_eval-n),['writerId','inviteday','invitehour','qId']].copy(),dataf.loc[dataf['label'] == -1,['writerId','qId']].copy(),t0_eval,n,'themeId','themeId',q2themes.copy(),{},dic_themes.copy(),{},True,)\n",
    "    )\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        result[day] = pool.apply_async(\n",
    "            func = get_sim_itemCF,\n",
    "            args = (data_asw.loc[(data_asw['inviteday']<day)&(data_asw['inviteday']>=day-n),['writerId','inviteday','invitehour','qId']].copy(),dataf.loc[dataf['inviteday'] == day,['writerId','qId']].copy(),day,n,'themeId','themeId',q2themes.copy(),{},dic_themes.copy(),{},True,)\n",
    "        )\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    for day in iter(dataf.loc[dataf['label'] != -1 ,'inviteday'].unique()):\n",
    "        df.loc[dataf['inviteday'] == day,scorefea] = result[day].get()        \n",
    "    df.loc[dataf['label'] == -1,scorefea] = result[t0_eval].get()\n",
    "    \n",
    "    return df\n",
    "\n",
    "\n",
    "def get_sim_itemCF(data_train,data_test,day,n,hisfea,curfea,q2hisfea,q2curfea,dic_his,dic_cur,noclick):\n",
    "    #day:待计算数据邀请日\n",
    "    #n:统计时长\n",
    "    #hisfea:历史聚合特征（theme/word）\n",
    "    #curfea:当前比较特征(theme/word)\n",
    "    #q2hisfea:问题到hisfea的dic\n",
    "    #q2curfea：问题到curfea的dic\n",
    "    #dic_cur:当前训练接的theme/word的emb\n",
    "    #dic_his:历史回答的theme/word的emb\n",
    "    if curfea == hisfea:\n",
    "        dic_cur = dic_his\n",
    "        q2curfea = q2hisfea\n",
    "    if noclick:\n",
    "        pathfold = hisfea+'_noclick'\n",
    "    else:\n",
    "        pathfold = hisfea\n",
    "    \n",
    "    if os.path.exists(datapath+'dic_q2hisfea/%s/his_%s_%s.pkl' %(pathfold,day,n)) and os.path.exists(datapath+'dic_q2hisfea/%s/his_delts_%s_%s.pkl' %(pathfold,day,n)):\n",
    "        with open(datapath+'dic_q2hisfea/%s/his_%s_%s.pkl' %(pathfold,day,n),'rb') as f:\n",
    "            dic_t = pickle.load(f)\n",
    "        with open(datapath+'dic_q2hisfea/%s/his_delts_%s_%s.pkl' %(pathfold,day,n),'rb') as f:\n",
    "            dic_delt = pickle.load(f)\n",
    "    else:\n",
    "        dic_t = {}\n",
    "        dic_delt = {}\n",
    "        data_train = data_train[data_train['writerId'].isin(data_test['writerId'].unique())].groupby('writerId')\n",
    "        for writerid in iter(data_train.size().index):\n",
    "            dic_t[writerid] = []#可能为kong\n",
    "            dic_delt[writerid] = []\n",
    "            gp = data_train.get_group(writerid)\n",
    "            gp = gp.sort_values(by = ['inviteday','invitehour'],ascending = False)#.iloc[:10,:]\n",
    "            for row in iter(gp[['inviteday','qId']].values):\n",
    "                themes = q2hisfea[row[1]]\n",
    "                if themes[0] == '-1':\n",
    "                    continue\n",
    "                dic_t[writerid].append(themes)\n",
    "                dic_delt[writerid].append(day-row[0])\n",
    "        with open(datapath+'dic_q2hisfea/%s/his_%s_%s.pkl' %(pathfold,day,n),'wb') as f:\n",
    "            pickle.dump(dic_t,f)\n",
    "        with open(datapath+'dic_q2hisfea/%s/his_delts_%s_%s.pkl' %(pathfold,day,n),'wb') as f:\n",
    "            pickle.dump(dic_delt,f)\n",
    "    \n",
    "    result = []\n",
    "    for row in iter(data_test[['writerId','qId']].values):\n",
    "        try:\n",
    "            list_t = dic_t[row[0]]\n",
    "#             list_delt = dic_delt[row[0]]\n",
    "        except:\n",
    "            result.append(np.nan)#无历史记录\n",
    "            continue\n",
    "        curthemes = q2curfea[row[1]]\n",
    "        if curthemes[0] == '-1':#当前问题t缺失\n",
    "            result.append(np.nan)\n",
    "            continue\n",
    "        \n",
    "        if len(list_t) == 0:#有历史记录，但历史记录的问题的t都是-1\n",
    "            result.append(np.nan)\n",
    "            continue\n",
    "        \n",
    "        sim_between_his_cur = 0\n",
    "        for index,histhemes in iter(enumerate(list_t)):#循环历史问题\n",
    "#             deltday = list_delt[index]\n",
    "#             weight = \n",
    "            sims_between_q = []\n",
    "            for histheme in iter(histhemes):#循环问题的t\n",
    "                for curtheme in iter(curthemes):#循环当前问题的t\n",
    "                    sims_between_q.append(cos_sim(dic_his[histheme],dic_cur[curtheme]))\n",
    "            q_sim = np.sort(sims_between_q)[-3:].mean()\n",
    "            if q_sim > 0.7:#simhold,相当于取与该问题相似的问题的topk与历史问题取交集\n",
    "                sim_between_his_cur += q_sim#*weight,关于间隔时间的权\n",
    "        result.append(sim_between_his_cur)\n",
    "    \n",
    "    return result\n",
    "\n",
    "#话题64维emb索引字典\n",
    "theme_vec = pd.read_csv(datapath+'data/theme_vector.csv')\n",
    "cols = theme_vec.columns.tolist()\n",
    "cols.remove('themeId')\n",
    "cols = ['themeId']+cols\n",
    "theme_vec = theme_vec[cols]\n",
    "dic_themes = {}\n",
    "for row in iter(theme_vec.values):\n",
    "    dic_themes[row[0]] = row[1:]\n",
    "\n",
    "#问题绑定话题的索引字典\n",
    "data_question = pd.read_csv(datapath+'data/data_question.csv')[['qId','themeId']]\n",
    "q2themes = {}\n",
    "for row in iter(data_question[['qId','themeId']].values):\n",
    "    q2themes[row[0]] = row[1].split(',')\n",
    "del data_question\n",
    "\n",
    "#切词64维emb索引字典\n",
    "word_vec = pd.read_csv(datapath+'data/word_vector.csv')\n",
    "cols = word_vec.columns.tolist()\n",
    "cols.remove('wordId')\n",
    "cols = ['wordId']+cols\n",
    "word_vec = word_vec[cols]\n",
    "dic_words = {}\n",
    "for row in iter(word_vec.values):\n",
    "    dic_words[row[0]] = row[1:]\n",
    "\n",
    "#问题标题&描述切词过滤后的索引字典\n",
    "data_question = pd.read_csv(datapath+'data_q_title_tfidf.csv')\n",
    "data_question = data_question[['问题id','title_topk','text_topk']].rename(columns = {'问题id':'qId',\n",
    "                                                                    'title_topk':'title_words_tfidf0','text_topk':'title_words_tfidf1'})\n",
    "q2words = {}\n",
    "for row in iter(data_question[['qId','title_words_tfidf1']].values):\n",
    "    q2words[row[0]] = row[1].split(',')\n",
    "del data_question\n",
    "\n",
    "df = writerlastndayperformance_itemCF_pool(data,data_answer,data,df,q2themes,q2words,dic_themes,dic_words)\n",
    "\n",
    "del data_answer\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2)用户关注/感兴趣话题与当前问题绑定话题的相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#用户与当前问题的theme相似度\n",
    "def get_user_theme_emb(data_writer):\n",
    "    \n",
    "    user_topic_fea_64 = []\n",
    "    for index,row in data_writer.iterrows():\n",
    "        if row['attentionthemes'] == '-1' and row['likethemes'] == '-1':\n",
    "            user_topic_fea_64.append(np.zeros(64))\n",
    "        if row['attentionthemes'] != '-1':\n",
    "            ft = row['attentionthemes'].strip().split(',')\n",
    "            temp = np.zeros(64)\n",
    "            for t in iter(ft):\n",
    "                temp = temp + t2v[t]\n",
    "            temp = temp/len(ft)\n",
    "            user_topic_fea_64.append(temp)\n",
    "        if row['attentionthemes'] == '-1' and row['likethemes'] != '-1':\n",
    "            it = row['likethemes'].strip().split(',')\n",
    "            temp = np.zeros(64)\n",
    "    #         w = []\n",
    "    #         for t in it:\n",
    "    #             w.append(float(t.split(':')[1]))\n",
    "    #         for t in it:\n",
    "    #             wei = float(t.split(':')[1]) / sum(w)\n",
    "    #             temp += wei*t2v_norm[int(t.split(':')[0][1:])]\n",
    "            for t in iter(it):\n",
    "                temp = temp + t2v[t.split(':')[0]]\n",
    "            temp = temp/len(it)\n",
    "            user_topic_fea_64.append(temp)\n",
    "    user_topic_fea_64 = pd.DataFrame(user_topic_fea_64,columns = ['writer_theme_emb%i' %i for i in range(64)])\n",
    "    return user_topic_fea_64\n",
    "\n",
    "def get_question_theme_emb(data_q):\n",
    "    result = []\n",
    "    for themes in iter(data_q['themeId']):\n",
    "        if themes == '-1':\n",
    "            result.append(np.zeros(64))\n",
    "            continue\n",
    "        themes = themes.split(',')\n",
    "        emb = np.zeros(64)\n",
    "        for theme in iter(themes):\n",
    "            emb = emb + t2v[theme]\n",
    "        emb = emb/len(themes)\n",
    "        result.append(emb)\n",
    "    result = pd.DataFrame(result,columns = ['q_theme_emb%i' %i for i in range(64)])\n",
    "    return result\n",
    "\n",
    "def cos_sim(vector_a, vector_b):\n",
    "\n",
    "    vector_a = np.mat(vector_a)\n",
    "    vector_b = np.mat(vector_b)\n",
    "    num = float(vector_a * vector_b.T)\n",
    "    denom = np.linalg.norm(vector_a) * np.linalg.norm(vector_b)\n",
    "    cos = num / denom\n",
    "    sim = 0.5 + 0.5 * cos\n",
    "    return sim\n",
    "\n",
    "def get_sim_theme_between_q_writer(data):\n",
    "    result = []\n",
    "    for row in iter(data.values):\n",
    "        result.append(cos_sim(row[:64],row[64:]))\n",
    "    \n",
    "    return pd.Series(result)\n",
    "\n",
    "theme_vec = pd.read_csv(datapath+'data/theme_vector.csv')\n",
    "cols = theme_vec.columns.tolist()\n",
    "cols.remove('themeId')\n",
    "cols = ['themeId']+cols\n",
    "theme_vec = theme_vec[cols]\n",
    "t2v = {}\n",
    "for row in iter(theme_vec.values):\n",
    "    t2v[row[0]] = row[1:]\n",
    "\n",
    "data_writer = pd.read_csv(datapath+'data/data_writer.csv')\n",
    "user_topic_fea_64 = parallelize_dataframe(data_writer,get_user_theme_emb).reset_index(drop = True)\n",
    "user_topic_fea_64['writerId'] = data_writer['writerId']\n",
    "data = data.merge(user_topic_fea_64,on = 'writerId',how = 'left')\n",
    "\n",
    "data_question = pd.read_csv(datapath+'data/data_question.csv')\n",
    "q_topic_fea_64 = parallelize_dataframe(data_question,get_question_theme_emb).reset_index(drop = True)\n",
    "q_topic_fea_64['qId'] = data_question['qId']\n",
    "data = data.merge(q_topic_fea_64,on = 'qId',how = 'left')\n",
    "\n",
    "cols = ['writer_theme_emb%i' %i for i in range(64)]+['q_theme_emb%i' %i for i in range(64)]\n",
    "df['sim_theme_between_q_writer'] = parallelize_dataframe(data[cols],get_sim_theme_between_q_writer).values\n",
    "data = data.drop(cols,axis = 1)\n",
    "del data_question\n",
    "del data_writer\n",
    "del theme_vec\n",
    "del t2v\n",
    "del user_topic_fea_64\n",
    "del q_topic_fea_64\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3）用户关注/感兴趣话题与当前问题绑定话题的重合统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def samethemenum_atten(row):\n",
    "    row = row.values\n",
    "    if row[0] == '-1' or row[1] == '-1':\n",
    "        return np.nan\n",
    "    row0 = row[0].strip().split(',')\n",
    "    row1 = row[1].strip().split(',')\n",
    "    num = 0\n",
    "    for theme in iter(row0):\n",
    "        if theme in row1:\n",
    "            num = num+1\n",
    "    return num\n",
    "def samethemenum_like(row):\n",
    "    row = row.values\n",
    "    if row[0] == '-1' or row[1] == '-1':\n",
    "        return [np.nan,np.nan]\n",
    "    row0 = row[0].strip().split(',')\n",
    "    row1 = row[1].strip().split(',')\n",
    "    num = 0\n",
    "    quannum = 0\n",
    "    for theme in iter(row1):\n",
    "        theme = theme.strip().split(':')\n",
    "        if theme[0] in row0:\n",
    "            num = num+1\n",
    "            quannum = quannum+float(theme[1])\n",
    "\n",
    "    return [num,quannum]\n",
    "\n",
    "def p1(dataf):\n",
    "    return dataf.apply(samethemenum_like,axis = 1)\n",
    "df['samenum_like'] = np.nan\n",
    "df['sameqnum_like'] = np.nan\n",
    "df[['samenum_like','sameqnum_like']] = parallelize_dataframe(data[['themeId','likethemes']],p1).values\n",
    "\n",
    "def p2(dataf):\n",
    "    return dataf.apply(samethemenum_atten,axis = 1)\n",
    "df['samenum_atten'] = parallelize_dataframe(data[['themeId','attentionthemes']],p2).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_2.4.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 三、用户历史反馈统计特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ans_quality(dataf,data_asw,df,feas,id_fea):   \n",
    "    data_train = dataf[dataf['label'] != -1]\n",
    "    data_test = dataf[dataf['label'] == -1]  \n",
    "    if (len(id_fea) > 1):\n",
    "        for fea in feas:\n",
    "            df[id_fea[1]+'_'+fea+'_sum'] = np.nan\n",
    "            df[id_fea[1]+'_'+fea+'_mean'] = np.nan\n",
    "            df[id_fea[1]+'_'+fea+'_max'] = np.nan\n",
    "            daylen_asw = 21\n",
    "            for day in iter(data_train['inviteday'].unique()):\n",
    "                df.loc[dataf['inviteday'] == day,[id_fea[1]+'_'+fea+'_sum',id_fea[1]+'_'+fea+'_mean',id_fea[1]+'_'+fea+'_max']] = \\\n",
    "                get_values(data_asw[(data_asw['inviteday']<day)],dataf[dataf['inviteday'] == day],fea,id_fea)#&(data_asw['inviteday']>=day-daylen_asw)\n",
    "#             df[[id_fea[1]+'_'+fea+'_sum',id_fea[1]+'_'+fea+'_mean',id_fea[1]+'_'+fea+'_max']] = \\\n",
    "            df.loc[data_test.index,[id_fea[1]+'_'+fea+'_sum',id_fea[1]+'_'+fea+'_mean',id_fea[1]+'_'+fea+'_max']] = \\\n",
    "            get_values(data_asw[(data_asw['inviteday']<t0_eval)],data_test,fea,id_fea)#&(data_asw['inviteday']>=t0_eval-daylen_asw)\n",
    "    #         df.loc[(~dataf[fea].isna())&(df[fea+'_label_count'].isna()),fea+'_label_count'] = 0\n",
    "    else:\n",
    "        for fea in feas:\n",
    "            df[fea+'_sum'] = np.nan\n",
    "            df[fea+'_mean'] = np.nan\n",
    "            df[fea+'_max'] = np.nan\n",
    "            daylen_asw = 21\n",
    "            for day in iter(data_train['inviteday'].unique()):\n",
    "                df.loc[dataf['inviteday'] == day,[fea+'_sum',fea+'_mean',fea+'_max']] = \\\n",
    "                get_values(data_asw[(data_asw['inviteday']<day)],dataf[dataf['inviteday'] == day],fea,id_fea)#&(data_asw['inviteday']>=day-daylen_asw)\n",
    "#             df[[fea+'_sum',fea+'_mean',fea+'_max']] = \\\n",
    "            df.loc[data_test.index,[fea+'_sum',fea+'_mean',fea+'_max']] = \\\n",
    "            get_values(data_asw[(data_asw['inviteday']<t0_eval)],data_test,fea,id_fea)#&(data_asw['inviteday']>=t0_eval-daylen_asw)\n",
    "    #         df.loc[(~dataf[fea].isna())&(df[fea+'_label_count'].isna()),fea+'_label_count'] = 0\n",
    "    del data_train\n",
    "    del data_test\n",
    "    gc.collect()\n",
    "    \n",
    "    return df\n",
    "def get_values(df_train,df_test,fea,id_fea):\n",
    "    df_train = df_train[[fea]+id_fea].groupby(id_fea)[fea].agg(['sum','mean','max']).reset_index()\n",
    "    new_fea_name = id_fea+[fea+'_sum',fea+'_mean',fea+'_max']\n",
    "    df_train.columns=new_fea_name\n",
    "    df_test = df_test.merge(df_train,on = id_fea, how = 'left')\n",
    "    return df_test[[fea+'_sum',fea+'_mean',fea+'_max']].values\n",
    "\n",
    "# 用户回答质量统计\n",
    "ans_quality_cols = ['collectnum', 'commentnum', 'good_bool','picture_bool', 'recommend_bool','yuanzhuo_bool', 'video_bool', 'unhelpnum','wordnum','unlikenum', 'cancellikenum','jubaonum', \\\n",
    " '3qnum', 'likenum']\n",
    "id_fea = ['writerId']#writerId\n",
    "df = ans_quality(data[id_fea+['inviteday','label']], data_answer,df, ans_quality_cols, id_fea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_3.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# .NN特征构建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = data[[]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1、加载问题关键字\n",
    "\n",
    "问题关键字构建方式：利用tf-idf过滤，对于只有标题的取最重要的1/3，对于有描述的取总的2/3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_question = pd.read_csv('data/data_q_title_tfidf.csv')\n",
    "data_question.columns = ['index', 'qId', 'title_words', 'describe_words']\n",
    "df['all_words'] = data[['qId']].merge(data_question[['describe_words', 'qId']],on = 'qId',how = 'left')['describe_words'].values\n",
    "del data_question\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2、基于用户历史回答信息得到的用户感兴趣的关键字 / 主题信息\n",
    "利用提供的answer文件，利用当前用户的历史回答信息（注意时序），得到用户感兴趣的关键字、主题信息，并按照出现频率为不同的主题、关键字设置权重，出现频率越高代表用户对这个主题的关注越高，设置的权重越高"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getHistInfo(data, data_ans, info_type, dayn=30):\n",
    "\n",
    "    def getUserhistTopic_all(dataf, data_ans, info_type, dayn=30, testflag=False):\n",
    "        rs = dataf[[]]\n",
    "        \n",
    "        rs[info_type[0]] = np.nan  \n",
    "        result = {}\n",
    "        for day in tqdm.tqdm_notebook(dataf.loc[dataf['label']!=-1, 'inviteday'].unique()):\n",
    "            rs.loc[dataf['inviteday']==day, info_type[0]] = getlastndayHisttopic(data_ans.loc[(data_ans['answerday']<day)&(data_ans['answerday']>=day-dayn)],\n",
    "                                                                                 dataf.loc[dataf['inviteday']==day], info_type[1])  \n",
    "\n",
    "        rs.loc[dataf['label']==-1, info_type[0]] = getlastndayHisttopic(data_ans.loc[(data_ans['answerday']<3868)&(data_ans['answerday']>=3868-dayn)],\n",
    "                                                 dataf[dataf['label']==-1], info_type[1])\n",
    "        return rs\n",
    "\n",
    "    #得到过去历史信息  \n",
    "    def getlastndayHisttopic(data_ans, dataf, fea): \n",
    "        gps = data_ans[['writerId', fea]].groupby('writerId')\n",
    "        rs = []\n",
    "        temp_dic = {}\n",
    "        for uid in dataf['writerId'].values:\n",
    "            topics = {}\n",
    "            try:\n",
    "                if uid in temp_dic:\n",
    "                    rs.append(temp_dic[uid])    \n",
    "                else:           \n",
    "                    for i in gps.get_group(uid)[fea]:\n",
    "                        tps = i.split(',')\n",
    "                        for tp in tps:\n",
    "                            if tp not in topics:\n",
    "                                topics[tp] = 1\n",
    "                            else:\n",
    "                                topics[tp] += 1  \n",
    "\n",
    "                    topics = pd.Series(topics)\n",
    "                    topics =(topics/topics.mean()).round(3).to_dict()\n",
    "                    topics = sorted(topics.items(),key=lambda x: x[1], reverse=True)[:100]\n",
    "                    temp = []\n",
    "                    for t in topics:\n",
    "                        temp.append(t[0]+ ':' +str(t[1]))\n",
    "                    rs.append(','.join(temp)) \n",
    "                    temp_dic[uid] = ','.join(temp)\n",
    "            except:\n",
    "                rs.append('-1')\n",
    "                temp_dic[uid] = '-1'\n",
    "        return rs\n",
    "    \n",
    "    type_dic = {'topics':['hist_user_themes', 'themeId'], 'words':['hist_user_unlike_themes', 'words_list']}\n",
    "    user_topic = getUserhistTopic_all(data, data_ans, type_dic[info_type], dayn=dayn)\n",
    "    return user_topic\n",
    "\n",
    "data_ans = pd.read_csv(datapath+'data/data_answer.csv')\n",
    "data_question = pd.read_csv(datapath+'data/data_question.csv')\n",
    "data_ans = data_ans.merge(data_question[['qId', 'themeId']],on = 'qId',how = 'left')\n",
    "# 加载word\n",
    "data_question = pd.read_csv(datapath+'data/data_q_title_tfidf.csv')\n",
    "data_question.columns = ['index', 'qId', 'title_words', 'describe_words']\n",
    "data_ans = data_ans.merge(data_question[['describe_words', 'qId']],on = 'qId',how = 'left').rename(columns = {'describe_words':'words_list'})\n",
    "\n",
    "# data_test 表示要处理的数据\n",
    "df['user_topics'] = getHistInfo(data, data_ans, 'topics')\n",
    "df['user_words'] = getHistInfo(data, data_ans, 'words')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3、基于用户历史未回答信息得到的用户未点击问题的关键字 / 主题信息\n",
    "利用当前用户的历史未回答的信息（注意时序），得到用户讨厌的关键字、主题信息，并按照出现频率为不同的主题、关键字设置权重，出现频率越高代表用户对这个主题的讨厌度越高，设置的权重越高"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getUnlikeInfo(data, data_ans, info_type, dayn=7): # data[data['label']==0]\n",
    " \n",
    "    def getUserhistTopic_F_all(dataf, data_ans, info_type, dayn=7):\n",
    "        rs = dataf[[]]\n",
    "        rs[info_type[0]] = np.nan  \n",
    "        result = {}\n",
    "\n",
    "        for day in tqdm.tqdm_notebook(dataf.loc[dataf['label']!=-1, 'inviteday'].unique()):\n",
    "            if day>3838:\n",
    "                rs.loc[dataf['inviteday']==day, info_type[0]] = getlastndayHisttopic(data_ans.loc[(data_ans['inviteday']<day)&(data_ans['inviteday']>=day-dayn)],\n",
    "                                                                                     dataf.loc[dataf['inviteday']==day], info_type[0])\n",
    "\n",
    "        rs.loc[dataf['label']==-1, info_type[0]] = getlastndayHisttopic(data_ans.loc[(data_ans['inviteday']<3868)&(data_ans['inviteday']>=3868-dayn)],\n",
    "                                             dataf[dataf['label']==-1], info_type[1])\n",
    "        return rs\n",
    "\n",
    "    #得到过去历史信息       \n",
    "    def getlastndayHisttopic(data_ans, dataf, fea): \n",
    "        gps = data_ans[['writerId', fea]].groupby('writerId')\n",
    "        rs = []\n",
    "        temp_dic = {}\n",
    "        for uid in dataf['writerId'].values:\n",
    "            topics = {}\n",
    "            try:\n",
    "                if uid in temp_dic:\n",
    "                    rs.append(temp_dic[uid])    \n",
    "                else:           \n",
    "                    for i in gps.get_group(uid)[fea]:\n",
    "                        tps = i.split(',')\n",
    "                        for tp in tps:\n",
    "                            if tp not in topics:\n",
    "                                topics[tp] = 1\n",
    "                            else:\n",
    "                                topics[tp] += 1  \n",
    "\n",
    "                    topics = pd.Series(topics)\n",
    "                    topics =(topics/topics.mean()).round(3).to_dict()\n",
    "                    topics = sorted(topics.items(),key=lambda x: x[1], reverse=True)[:100]\n",
    "                    temp = []\n",
    "                    for t in topics:\n",
    "                        temp.append(t[0]+ ':' +str(t[1]))\n",
    "                    rs.append(','.join(temp)) \n",
    "                    temp_dic[uid] = ','.join(temp)\n",
    "\n",
    "            except:\n",
    "                #print('error')\n",
    "                rs.append('-1')\n",
    "        return rs\n",
    "    \n",
    "    type_dic = {'topics':['hist_user_unlike_themes', 'themeId'], 'words':['hist_user_unlike_themes', 'words_list']}\n",
    "    user_unlike_topic = getUserhistTopic_F_all(data, data_ans, type_dic[info_type], dayn=dayn)\n",
    "    return user_unlike_topic\n",
    "\n",
    "unlikeData = data[data['label']==0] ## 用户未回答（label=0）的数据\n",
    "df['user_unlike_topic'] = getUnlikeTopics(data, unlikeData, 'topics')\n",
    "df['user_unlike_word'] = getUnlikeWord(data, unlikeData, 'words')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = memoryOptimization(df,np.float32)\n",
    "with open(datapath+'df/df_nn.pkl','wb') as f:\n",
    "    pickle.dump(df,f)\n",
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4、deepWalk\n",
    "\n",
    "经过分析，邀请本身就蕴含有丰富的信息，这里利用deepwalk得到社交网络信息，提取问题跟用户邀请的信息，将用户和问题放在同一空间  \n",
    "（本部分代码参考19年腾讯赛冠军队伍开源）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_sentences(dataf,present,target):\n",
    "    sentences=[]\n",
    "    dic={}\n",
    "    day=0\n",
    "\n",
    "    for item in iter(dataf[['inviteday',present,target]].values):\n",
    "        if day!=item[0]:\n",
    "            for key in iter(dic):\n",
    "                sentences.append(dic[key])\n",
    "            dic={}\n",
    "            day=item[0]\n",
    "        try:\n",
    "            dic[item[1]].append(str(item[2]))\n",
    "        except:\n",
    "            dic[item[1]]=[str(item[2])]\n",
    "    for key in iter(dic):\n",
    "        sentences.append(dic[key]) \n",
    "    random.shuffle(sentences)\n",
    "    return sentences\n",
    "\n",
    "def get_w2v(data_w2v, model,fea,Type, flag):\n",
    "    dic = {'qId':'item','writerId':'user'}\n",
    "    values = data_w2v[fea].unique()\n",
    "    w2v=[]\n",
    "    for v in iter(values):  \n",
    "        try:\n",
    "            a=[str(v)]\n",
    "            if flag:\n",
    "                v = dic[fea]+'_'+str(v)\n",
    "            a.extend(model[str(v)])\n",
    "            w2v.append(a)\n",
    "        except:\n",
    "            pass\n",
    "    return pd.DataFrame(w2v,columns = [fea]+[fea+'_'+Type+str(i) for i in range(32)])\n",
    "\n",
    "def get_sentences_deepwalk(log,f1,f2):\n",
    "    #构建图\n",
    "    dic={}\n",
    "    for item in iter(log[[f1,f2]].values):\n",
    "#         try:\n",
    "#             str(item[1])\n",
    "#             str(item[0])\n",
    "#         except:\n",
    "#             continue\n",
    "        try:\n",
    "            dic['item_'+str((item[1]))].add('user_'+str((item[0])))\n",
    "        except:\n",
    "            dic['item_'+str((item[1]))]=set(['user_'+str((item[0]))])\n",
    "        try:\n",
    "            dic['user_'+str((item[0]))].add('item_'+str((item[1])))\n",
    "        except:\n",
    "            dic['user_'+str((item[0]))]=set(['item_'+str((item[1]))])\n",
    "    dic_cont={}\n",
    "    for key in iter(dic):\n",
    "        dic[key]=list(dic[key])\n",
    "        dic_cont[key]=len(dic[key])\n",
    "    print(\"creating\")     \n",
    "    #构建路径\n",
    "    path_length=10        \n",
    "    sentences=[]\n",
    "    length=[]\n",
    "    for key in iter(dic):\n",
    "        sentence=[key]\n",
    "        while len(sentence)!=path_length:\n",
    "            key=dic[sentence[-1]][random.randint(0,dic_cont[sentence[-1]]-1)]\n",
    "            if len(sentence)>=2 and key == sentence[-2]:\n",
    "                break\n",
    "            else:\n",
    "                sentence.append(key)\n",
    "        sentences.append(sentence)\n",
    "        length.append(len(sentence))\n",
    "        if len(sentences)%100000==0:\n",
    "            print(len(sentences))\n",
    "    print(np.mean(length))\n",
    "    print(len(sentences))\n",
    "    random.shuffle(sentences)\n",
    "    return sentences\n",
    "\n",
    "model_deepwalk_sentence = get_sentences_deepwalk(data.sort_values(by='inviteday'), 'writerId', 'qId',)\n",
    "model_deepwalk = word2vec.Word2Vec(model_deepwalk_sentence,size = 32,window = 4,min_count = 1,sg = 1,workers = 15,iter = 20)\n",
    "print('model ok')\n",
    "deepw_mId = get_w2v(data, model_deepwalk,'writerId','dw',flag = True)\n",
    "deepw_qId = get_w2v(data, model_deepwalk,'qId','dw',flag = True)\n",
    "deepw_mId.head()\n",
    "\n",
    "with open(datapath+'data/m_dwdf.pkl', 'wb') as f:\n",
    "    pickle.dump(deepw_mId, f)\n",
    "with open(datapath+'data/q_dwdf.pkl', 'wb') as f:\n",
    "    pickle.dump(deepw_qId, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5、类别映射字典\n",
    "将字符串类型的特征转化为数字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "single_features = ['invitehour', 'createhour', 'inviteweekday', 'createweekday', 'sex', 'activity', 'bool_A',\n",
    "       'bool_B', 'bool_C', 'bool_D', 'bool_E', 'category_E', 'category_A', 'category_B', 'category_C', 'category_D']\n",
    "dic = {}\n",
    "temp = data[data['label']!=-1]\n",
    "for s in tqdm.tqdm_notebook(single_features):\n",
    "    dic[s] = {}\n",
    "    dic[s]['unk'] = 0\n",
    "    for i in temp[s].unique():\n",
    "        dic[s][str(i)] = len(dic[s])\n",
    "            \n",
    "print('singlefeatures ok')\n",
    "with open(datapath+'/data/dic_all.pkl', 'wb') as f:\n",
    "    pickle.dump(dic, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6、相关文件\n",
    "- word_weight_64.pkl 利用提供的word embedding 得到的嵌入矩阵\n",
    "- word2index：word映射index文件\n",
    "- topic_weight_64：利用提供的topic embedding 得到的嵌入矩阵\n",
    "- topic2index：topic映射index的文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#### word_weight_64， word2index\n",
    "text = codecs.open(datapath+'data/word_vectors_64d.txt').readlines()\n",
    "text = [i.replace('\\t',' ') for i in text]\n",
    "wf = codecs.open(datapath+'data/word_vectors_suntp.txt', 'w')\n",
    "for i in text:\n",
    "    wf.write(i)\n",
    "\n",
    "glove_file = dpath(datapath+'data/word_vectors_suntp.txt', )\n",
    "tmp_file = get_tmpfile(datapath+\"data/word_vectors_suntp_w2v.txt\")\n",
    "glove2word2vec(glove_file, tmp_file)\n",
    "vectors = Vectors(name='word_vectors_suntp_w2v.txt', cache=datapath+'/data')\n",
    "\n",
    "topic2index = dict()\n",
    "topic2index['unk'] = 0\n",
    "topic2index['pad'] = 1\n",
    "t = vectors.stoi\n",
    "for i in t:\n",
    "    topic2index[i] = t[i]+2\n",
    "    \n",
    "a = torch.Tensor(2, 64).uniform_(-1,1)\n",
    "weight = torch.cat([a,vectors.vectors], dim=0)\n",
    "print(weight.size())\n",
    "with open(datapath+'data/word_weight_64.pkl', 'wb') as f:\n",
    "    pickle.dump(weight, f)\n",
    "    \n",
    "with open(datapath+'data/word2index.pkl', 'wb') as f:\n",
    "    pickle.dump(topic2index, f)\n",
    "    \n",
    "###########topic_weight_64.pkl， topic2index\n",
    "\n",
    "text = codecs.open(datapath+'data/topic_vectors_64d.txt').readlines()\n",
    "text = [i.replace('\\t',' ') for i in text]\n",
    "wf = codecs.open(datapath+'data/topic_vectors_64d_sun.txt', 'w')\n",
    "for i in text:\n",
    "    wf.write(i)\n",
    "\n",
    "glove_file = dpath(datapath+'data/topic_vectors_64d_sun.txt')\n",
    "tmp_file = get_tmpfile(datapath+\"data/topic_vectors_64d_sun_w2v.txt\")\n",
    "glove2word2vec(glove_file, tmp_file)\n",
    "vectors = Vectors(name='topic_vectors_64d_sun_w2v.txt', cache=datapath+'/data')\n",
    "\n",
    "topic2index = dict()\n",
    "topic2index['unk'] = 0\n",
    "topic2index['pad'] = 1\n",
    "t = vectors.stoi\n",
    "for i in t:\n",
    "    topic2index[i] = t[i]+2\n",
    "    \n",
    "a = torch.Tensor(2, 64).uniform_(-1,1)\n",
    "weight = torch.cat([a,vectors.vectors], dim=0)\n",
    "with open(datapath+'data/topic_weight_64.pkl', 'wb') as f:\n",
    "    pickle.dump(weight, f)\n",
    "    \n",
    "with open(datapath+'data/topic2index.pkl', 'wb') as f:\n",
    "    pickle.dump(topic2index, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# .模型训练（树模型&NN）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1、稠密特征拼接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#labelEncoder\n",
    "labelfeatures = ['sex','activity','bool_A','bool_B','bool_C','bool_D','bool_E','category_E']\n",
    "data[labelfeatures] = data[labelfeatures].fillna('-1')\n",
    "for feature in labelfeatures:\n",
    "    le = LabelEncoder()\n",
    "    try:\n",
    "        data[feature] = le.fit_transform(data[feature].apply(int))\n",
    "    except:\n",
    "        data[feature] = le.fit_transform(data[feature])\n",
    "\n",
    "#拼接df\n",
    "for feagp in ['1.1','1.2','1.3','2.1','2.2','2.3','2.4','3']:\n",
    "    with open(datapath+'data/df_%s.pkl' %feagp,'rb') as f:\n",
    "        df = pickle.load(f)\n",
    "    data = pd.concat([data.df],axis = 1)\n",
    "    print(feagp+' is ok')\n",
    "    del df\n",
    "\n",
    "del data['themeId']\n",
    "del data['attentionthemes']\n",
    "del data['likethemes']\n",
    "data = memoryOptimization(data,np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2、树模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = data[data['label'] != -1]\n",
    "# train = down_sample(train,train,rate = 3)\n",
    "test = data[data['label'] == -1].reset_index(drop = True)\n",
    "train_x = train.drop(['label'],axis = 1)\n",
    "train_y = train['label']\n",
    "test_x = test.drop(['label'],axis = 1)\n",
    "del train\n",
    "del test\n",
    "gc.collect()\n",
    "\n",
    "mfeas = ['category_A','category_B','category_D','category_C','mostliketheme']\n",
    "catefeas = ['sex','activity','bool_A','bool_B','bool_C','bool_D','bool_E','category_E']\n",
    "\n",
    "#lgb\n",
    "pre_test,pre_train,score,fea_imp,iternum = lgb_train_pre1(train_x.drop(['qId','writerId'],axis = 1),train_y,\n",
    "                    test_x.drop(['qId','writerId'],axis = 1),catefeas,dropfeas = list(set(['inviteweekday','createweekday']+mfeas)&set(list(train_x.columns))),one = False,save_model=True)\n",
    "\n",
    "pre_train.to_csv(datapath+'data/lgb_0.1.csv', index=False)\n",
    "sub_sample = pd.read_csv(datapath+'invite_info_evaluate_2_0926.txt',sep = '\\t',header = None,names = ['qId','writerId','invitetime'])\n",
    "sub_sample['label'] = pre_test\n",
    "sub_sample.to_csv(datapath+'data/lgb_0.1.txt',sep = '\\t',header = False,index = False)\n",
    "\n",
    "#cat\n",
    "pre_test,pre_train,score,fea_imp,iternum = cat_train_pre1(train_x.drop(['qId','writerId'],axis = 1),train_y,\n",
    "                    test_x.drop(['qId','writerId'],axis = 1),catefeas,dropfeas = list(set(['inviteweekday','createweekday']+mfeas)&set(list(train_x.columns))),one = False,save_model=True)\n",
    "pre_train.to_csv(datapath+'data/cat_0.1.csv', index=False)\n",
    "sub_sample = pd.read_csv(datapath+'invite_info_evaluate_2_0926.txt',sep = '\\t',header = None,names = ['qId','writerId','invitetime'])\n",
    "sub_sample['label'] = pre_test\n",
    "sub_sample.to_csv(datapath+'data/cat_0.1.txt',sep = '\\t',header = False,index = False)\n",
    "\n",
    "#xgb\n",
    "pre_test,pre_train,score,fea_imp,iternum = xgb_train_pre1(train_x.drop(['qId','writerId'],axis = 1),train_y,\n",
    "                    test_x.drop(['qId','writerId'],axis = 1),dropfeas = list(set(['inviteweekday','createweekday']+mfeas+catefeas)&set(list(train_x.columns))),one = False,save_model=True)\n",
    "pre_train.to_csv(datapath+'data/xgb_0.1.csv', index=False)\n",
    "sub_sample = pd.read_csv(datapath+'invite_info_evaluate_2_0926.txt',sep = '\\t',header = None,names = ['qId','writerId','invitetime'])\n",
    "sub_sample['label'] = pre_test\n",
    "sub_sample.to_csv(datapath+'data/xgb_0.1.txt',sep = '\\t',header = False,index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3、nn部分特征拼接"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "除了上面用到的稠密特征外，nn部分还用到了其它特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#nn\n",
    "with open(datapath+'data/df_nn.pkl','rb') as f:\n",
    "    df = pickle.load(f)\n",
    "data = pd.concat([data.df],axis = 1)\n",
    "del df\n",
    "\n",
    "with open(datapath+'data/m_dwdf.pkl', 'rb') as f:\n",
    "    m_dwdf = pickle.load(f)\n",
    "with open(datapath+'data/q_dwdf.pkl', 'rb') as f:\n",
    "    q_dwdf = pickle.load(f)   \n",
    "data = pd.merge(data, m_dwdf, on='writerId', how='left')\n",
    "data = pd.merge(data, q_dwdf, on='qId', how='left')\n",
    "\n",
    "del m_dwdf\n",
    "del q_dwdf\n",
    "gc.collect()\n",
    "data = memoryOptimization(data,np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 特征处理\n",
    "- 1、类别特征处理， 转化成数值，方便进行embedding\n",
    "- 2、数值特征归一化  \n",
    "- 3、deepwalk 信息拼接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#类别特征处理， 转化成数值，方便进行embedding\n",
    "with open(datapath+'data/dic_all.pkl', 'rb') as f:\n",
    "    dic = pickle.load(f)\n",
    "\n",
    "single_features = ['invitehour', 'createhour', 'inviteweekday', 'createweekday', 'sex', 'activity', 'bool_A',\n",
    "       'bool_B', 'bool_C', 'bool_D', 'bool_E', 'category_E', 'category_A', 'category_B', 'category_C', 'category_D']\n",
    "for i in single_features:\n",
    "    data[i] = data[i].apply(lambda x: dic[i][str(x)] if str(x) in dic[i] else 0)\n",
    "\n",
    "#数值特征归一化  \n",
    "new_dense = ['qId_bool_B_last7day_count', 'writerId_title_words_deltop150_lastnday_labelcount1', 'hourlenfromlastinvite_qId', 'writerId_createhour_last7day_count_gp_qId', 'collect_sum', 'thanks_max', 'qId_mostliketheme_last7day_count', 'qId_bool_C_label_count', 'thumbs_up_max', 'writerId_createweekday_label_ctr', 'qId_count', 'title_words_tfidf1_label_count1', 'writerId_themeId_lastnday_labelcount3', 'mostliketheme_curdayinv_count', 'themeId_label_ctr1', 'writerId_title_words_deltop150_lastnday_labelctr0', 'writerId_themeId_lastnday_labelctr_mean', 'isgood_sum', 'qId_label_count_gp_writerId', 'qId_invitehour_last7day_count_gp_writerId', 'nohelp_mean', 'activity_curdayinv_count', 'writerId_last3invnum2', 'qId_count_gp_writerId', 'writerId_title_words_deltop150_lastnday_labelctr2', 'qId_sex_last7day_count', 'createhour_curdayinv_count', 'qId_label_ctr_gp_writerId', 'q_class_300_bool_B_label_count', 'category_E_curdayinv_count', 'writerId_count', 'qId_inviteweekday_label_ctr', 'qId_last3+1invnum_mean', 'qId_category_B_last7day_count_gp_writerId', 'qId_bool_D_label_ctr', 'q_class_300_category_D_label_count', 'themeId_label_count0', 'q_class_300_bool_C_label_count', 'themeId_label_ctr2', 'writerId_themeId_lastnday_labelcount4', 'qId_invitehour_label_ctr', 'category_A_createdaylastweek2label_count', 'writerId_inviteweekday_label_count', 'qId_invitedaylastweek2label_count', 'category_A_invitedaylastweek2label_rate', 'title_words_tfidf1_label_count2', 'attentionthemes_label_ctr1', 'writerId_themeId_lastnday_labelctr0', 'writerId_title_words_deltop150_lastnday_labelctr4', 'comment_sum', 'quzan_max', 'qId_invitehour_last7day_count', 'writerId_createday_last7day_count', 'qId_activeday_inv', 'qId_bool_B_last7day_count_gp_writerId', 'attentionthemes_label_count1', 'qId_last7day_count', 'writerId_createdaylastweek2label_count', 'qId_mostliketheme_last7day_count_gp_writerId', 'category_D_label_ctr', 'themeId_label_count2', 'writerId_title_words_deltop150_lastnday_labelcount2', 'qId_category_B_last7day_count', 'writerId_invitehour_label_count', 'qId_bool_E_last7day_count_gp_writerId', 'opposition_max', 'writerId_title_words_deltop150_lastnday_labelctr3', 'istabel_sum', 'q_class_300_bool_E_label_ctr', 'istabel_mean', 'mostliketheme_count', 'writerId_q_class_300_label_count', 'writerId_themeId_lastnday_labelctr3', 'createday_curdayinv_count', 'qId_last3+1invnum_std', 'invitehour_count', 'qId_last7count_gp_writerId', 'category_D_createdaylastweek2label_rate', 'quzan_sum', 'writerId_themeId_lastnday_labelcount1', 'writerId_createhour_last7day_count', 'writerId_createhour_label_count', 'writerId_last3invnum1', 'category_C_last7day_count', 'writerId_curcount_gp_qId', 'qId_bool_D_label_count', 'qId_bool_D_last7day_count', 'sex_last7day_count', 'activity_createdaylastweek2label_rate', 'writerId_createdaylastweek2label_rate', 'yanzhi_count', 'activity_createdaylastweek2label_count', 'qId_activeday_asw', 'category_C_label_ctr', 'writerId_last3+1invnum_std', 'writerId_themeId_lastnday_labelctr1', 'category_A_invitedaylastweek2label_count', 'thumbs_up_mean', 'qId_label_count', 'writerId_label_count', 'writerId_createday_last7day_count_gp_qId', 'thumbs_up_sum', 'report_mean', 'qId_bool_A_last7day_count_gp_writerId', 'writerId_createweekday_last7day_count_gp_qId', 'qId_bool_B_label_ctr', 'samenum_like', 'isvideo_max', 'simhis_base_theme_theme_itemcf', 'writerId_invitehour_label_ctr', 'title_words_tfidf1_label_ctr2', 'writerId_q_class_300_label_ctr', 'qId_last3invnum2', 'writerId_curdayinv_count', 'writerId_themeId_lastnday_labelcount0', 'qId_activity_last7day_count_gp_writerId', 'qId_bool_C_last7day_count', 'title_words_tfidf1_label_ctr0', 'writerId_invitedaylastweek2label_count', 'writerId_title_words_deltop150_lastnday_labelcount4', 'writerId_themeId_lastnday_labelcount_sum', 'opposition_sum', 'simhis_base_theme_theme_noclick', 'writerId_themeId_lastnday_labelcount2', 'qId_category_E_last7day_count_gp_writerId', 'writerId_last7count_gp_qId', 'writerId_createweekday_last7day_count', 'qId_bool_C_label_ctr', 'q_class_300_bool_B_label_ctr', 'title_words_tfidf1_label_count0', 'qId_category_C_last7day_count_gp_writerId', 'qId_category_E_last7day_count', 'topic_sim', 'mostliketheme_label_ctr', 'isvideo_sum', 'q_class_300_category_D_label_ctr', 'writerId_themeId_lastnday_labelctr2', 'comment_max', 'quzan_mean', 'qId_bool_E_label_ctr', 'createweekday_curdayinv_count', 'hourlenfromlastinvite_writerId', 'qId_bool_A_last7day_count', 'createhour_label_ctr', 'attentionthemes_label_count2', 'isrec_mean', 'writerId_last7day_count', 'length_max', 'qId_category_D_last7day_count_gp_writerId', 'writerId_label_ctr_gp_qId', 'writerId_count_gp_qId', 'qId_curdayinv_count', 'writerId_inviteweekday_label_ctr', 'writerId_activeday_asw', 'writerId_themeId_lastnday_labelcount_mean', 'writerId_activeday_inv', 'writerId_invitehour_last7day_count_gp_qId', 'category_D_invitedaylastweek2label_count', 'collect_max', 'attentionthemes_label_ctr2', 'q_class_300_bool_A_label_ctr', 'report_sum', 'createhour_count', 'title_words_tfidf1_label_ctr_mean', 'writerId_invitedaylastweek2label_rate', 'istabel_max', 'bool_D_curdayinv_count', 'yanzhi_label_ctr', 'isimage_sum', 'qId_invitedaylastweek2label_rate', 'category_A_createdaylastweek2label_rate', 'comment_mean', 'q_life', 'length_mean', 'activity_label_ctr', 'qId_invitehour_label_count', 'createhour_label_count', 'category_D_createdaylastweek2label_count', 'qId_yanzhi_last7day_count', 'themeId_label_count1', 'isimage_max', 'writerId_label_count_gp_qId', 'yanzhi_curdayinv_count', 'writerId_createweekday_label_count', 'title_words_tfidf1_label_count_mean', 'writerId_title_words_deltop150_lastnday_labelcount3', 'invitehour_label_count', 'mostliketheme_label_count', 'activity_last7day_count', 'qId_last3invnum1', 'nohelp_sum', 'createday_count', 'qId_yanzhi_last7day_count_gp_writerId', 'qId_inviteweekday_label_count', 'writerId_label_ctr', 'writerId_createhour_label_ctr', 'qId_bool_B_label_count', 'length_sum', 'qId_category_D_last7day_count', 'q_class_300_bool_E_label_count', 'q_class_300_bool_A_label_count', 'qId_category_C_last7day_count', 'writerId_title_words_deltop150_lastnday_labelcount0', 'writerId_last3+1invnum_mean', 'isrec_sum', 'thanks_sum', 'invitehour_curdayinv_count', 'writerId_last3invnum0', 'yanzhi_label_count', 'qId_sex_last7day_count_gp_writerId', 'qId_last3invnum0', 'samenum_atten', 'themeId_label_ctr0', 'simhis_base_title_title_itemcf', 'qId_bool_E_last7day_count', 'category_D_invitedaylastweek2label_rate', 'writerId_title_words_deltop150_lastnday_labelctr1', 'collect_mean', 'attentionthemes_label_ctr0', 'qId_label_ctr', 'attentionthemes_label_count0', 'q_class_300_bool_C_label_ctr', 'isimage_mean', 'qId_category_A_last7day_count', 'yanzhi_d_last7day_count', 'nohelp_max', 'qId_bool_E_label_count', 'invitehour_label_ctr', 'writerId_invitehour_last7day_count', 'qId_bool_D_last7day_count_gp_writerId', 'qId_activity_last7day_count', 'isgood_max', 'qId_category_A_last7day_count_gp_writerId', 'thanks_mean', 'report_max', 'isgood_mean', 'category_C_curdayinv_count', 'isvideo_mean', 'opposition_mean', 'qId_curcount_gp_writerId', 'isrec_max', 'title_words_tfidf1_label_ctr1', 'qId_bool_C_last7day_count_gp_writerId', 'writerId_themeId_lastnday_labelctr4']\n",
    "num_dic = {}\n",
    "for fea in tqdm.tqdm_notebook(new_dense):\n",
    "    try:\n",
    "        scaler_val = data[fea][~data[fea].isnull()].values\n",
    "        scaler = StandardScaler().fit(scaler_val.reshape((len(scaler_val), 1)))\n",
    "        num_dic[fea] = scaler\n",
    "        data[fea].fillna(scaler.mean_[0], inplace=True)\n",
    "        data[fea] = scaler.transform(data[fea].values.reshape((len(data), 1))).reshape((len(data),)).tolist()\n",
    "    except:\n",
    "        print(fea)\n",
    "del scaler_val, scaler\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4、nn 训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import pandas as pd\n",
    "import random\n",
    "import pickle\n",
    "import torch.nn.functional as F\n",
    "import tqdm\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from optimizer import Lookahead\n",
    "from optimizer import RAdam\n",
    "import numpy as np\n",
    "import os\n",
    "import torch.utils.data as Data\n",
    "import gc\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "import codecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#mask generate\n",
    "def sequence_mask(embed, feature):\n",
    "    \n",
    "    mask = (feature!=1).unsqueeze(-1).expand_as(embed).float()\n",
    "    return embed*mask\n",
    "\n",
    "class TextCNN(nn.Module):\n",
    "    def __init__(self, args):\n",
    "        super(TextCNN, self).__init__()\n",
    "        self.args = args\n",
    "\n",
    "        chanel_num = 1\n",
    "        filter_num = args['filter_num']\n",
    "        filter_sizes = args['filter_sizes']\n",
    "\n",
    "        vocabulary_size = args['vocabulary_size']\n",
    "        embedding_dimension = args['embedding_dim']\n",
    "        self.embedding = nn.Embedding(vocabulary_size, embedding_dimension)\n",
    "        #self.embedding = self.embedding.from_pretrained(args.vectors, freeze=False)\n",
    "        self.embedding.weight.data.copy_(args['pretrained_weight'])\n",
    "            \n",
    "        self.convs = nn.ModuleList([nn.Conv2d(1, filter_num, (size, embedding_dimension)) for size in filter_sizes])\n",
    "        self.dropout = nn.Dropout(args['dropout'])\n",
    "        #self.fc = nn.Linear(len(filter_sizes)*filter_num, class_num)\n",
    "        self.feature_num = len(filter_sizes)*filter_num*4\n",
    "    def forward(self, x, y):\n",
    "\n",
    "        x = self.embedding(x)\n",
    "        x = x.unsqueeze(1)\n",
    "        y = self.embedding(y)\n",
    "        y = y.unsqueeze(1)\n",
    "        \n",
    "        x = [F.relu(conv(x)).squeeze(3) for conv in self.convs]\n",
    "        x = [F.max_pool1d(item, item.size(2)).squeeze(2) for item in x]\n",
    "        x = torch.cat(x, 1)\n",
    "        y = [F.relu(conv(y)).squeeze(3) for conv in self.convs]\n",
    "        y = [F.max_pool1d(item, item.size(2)).squeeze(2) for item in y]\n",
    "        y = torch.cat(y, 1)\n",
    "        rs = torch.cat([x, y, torch.abs(x-y), x*y], 1)\n",
    "        return rs\n",
    "\n",
    "class xDeepFM(nn.Module):\n",
    "    \n",
    "    def __init__(self, params):\n",
    "        super(xDeepFM, self).__init__()\n",
    "        self.device = params['device']\n",
    "        #self.mlp_input_dim = params['field_size'] * params['embedding_size']\n",
    "        self.k = params['k'] ##\n",
    "        self.dic = params['dic']  # 字典\n",
    "        self.single_features = params['single_features'] #\n",
    "        self.muti_features = params['muti_features']#\n",
    "        self.num_features = params['num_features']\n",
    "        self.cross_features = params['cross_features']\n",
    "        self.topic_features = params['topic_features']\n",
    "        self.other_features = params['other_features']\n",
    "        self.l2 = params['l2']\n",
    "        self.norm = params['normNum']\n",
    "        self.usetext = params['usetext']\n",
    "        self.usecin = params['usecin']\n",
    "        self.usetopic = params['usetopic']\n",
    "        self.useword = params['useword']\n",
    "        self.word_features = params['word_features']\n",
    "        #textcnn\n",
    "        if params['usetext']:\n",
    "            self.textcnn = TextCNN(params['textargs'])\n",
    "            self.textlen = self.textcnn.feature_num\n",
    "  \n",
    "        #mem\n",
    "        self.usemem = params['usemem']\n",
    "        if self.usemem:\n",
    "            mem_dim = 0 #64*2\n",
    "            \n",
    "            self.linear1 = nn.Linear(128, 64)\n",
    "            self.sig = nn.Linear(128+64,64)\n",
    "            self.mem1 = nn.Embedding(len(self.dic['writerId']), 64)\n",
    "            self.mem0 = nn.Embedding(len(self.dic['writerId']), 64)\n",
    "        else:\n",
    "            mem_dim = 0\n",
    "    \n",
    "        first_orders = nn.ModuleDict()\n",
    "        second_orders = nn.ModuleDict()\n",
    "        ## feature -index\n",
    "        feature_index = {}\n",
    "        for s in self.single_features+self.num_features + self.other_features:\n",
    "            feature_index[s] = [len(feature_index), len(feature_index)+1]\n",
    "\n",
    "        if 'writerId' in self.single_features:\n",
    "            self.single_features.remove('writerId') \n",
    "            \n",
    "        temp_index = 0\n",
    "        if self.usetopic:\n",
    "            temp_index = len(feature_index)+100\n",
    "            feature_index['attentionthemes'] = [len(feature_index), temp_index]             \n",
    "            feature_index['themeId'] = [temp_index, temp_index+13] \n",
    "            feature_index['likethemes'] = [temp_index+13, temp_index+13+10]\n",
    "            feature_index['likethemes_att'] = [temp_index+13+10, temp_index+13+10+10]\n",
    "            #hist_user_themes\n",
    "            feature_index['hist_user_themes'] = [temp_index+13+10+10, temp_index+13+10+10+10]\n",
    "            feature_index['hist_user_themes_att'] = [temp_index+13+10+10+10, temp_index+13+10+10+10+10]      \n",
    "        if self.usetext:\n",
    "            feature_index['m_interest_topic'] = feature_index['likethemes']\n",
    "            feature_index['q_topic'] = feature_index['themeId']    \n",
    "        if self.useword:\n",
    "            if temp_index>0:\n",
    "                feature_index['hist_user_words'] = [temp_index+13+10+10+10+10, temp_index+13+10+10+10+10 + 20 ] # 20\n",
    "                feature_index['hist_user_words_att'] = [temp_index+13+10+10+10+10 + 20, temp_index+13+10+10+10+10 + 20 + 20 ]\n",
    "                feature_index['all_words'] = [temp_index+13+10+10+10+10 + 20 + 20, temp_index+13+10+10+10+10 + 20 + 20 + 10 ] # 20              \n",
    "        feature_index['hist_user_unlike_themes'] = [temp_index+13+10+10+10+10 + 20 + 20 + 10, temp_index+13+10+10+10+10 + 20 + 20 + 10 + 10 ] # 20\n",
    "        feature_index['hist_user_unlike_themes_att'] = [temp_index+13+10+10+10+10 + 20 + 20 + 10 + 10, temp_index+13+10+10+10+10 + 20 + 20 + 10 + 10 + 10 ] # 20\n",
    "        feature_index['hist_user_unlike_words'] = [temp_index+13+10+10+10+10 + 20 + 20 + 10 + 10 + 10, temp_index+13+10+10+10+10 + 20 + 20 + 10 + 10 + 10 + 20 ] # 20\n",
    "        feature_index['hist_user_unlike_words_att'] = [temp_index+13+10+10+10+10 + 20 + 20 + 10 + 10 + 10 + 20, temp_index+13+10+10+10+10 + 20 + 20 + 10 + 10 + 10 + 20 +20 ] # 20\n",
    "        \n",
    "        self.feature_index = feature_index  ## index\n",
    "        \n",
    "        for s in self.single_features:\n",
    "            first_orders[s] = nn.Embedding(len(self.dic[s]), 1)\n",
    "            nn.init.normal_(first_orders[s].weight, mean=0, std=0.0001)\n",
    "            second_orders[s] = nn.Embedding(len(self.dic[s]), self.k)\n",
    "            nn.init.normal_(second_orders[s].weight, mean=0, std=0.0001)\n",
    "        \n",
    "        for s in self.muti_features:\n",
    "            first_orders[s] = nn.Embedding(len(self.dic[s])+2, 1)\n",
    "            nn.init.normal_(first_orders[s].weight, mean=0, std=0.0001)\n",
    "            second_orders[s] = nn.Embedding(len(self.dic[s])+2, self.k)\n",
    "            nn.init.normal_(second_orders[s].weight, mean=0, std=0.0001)\n",
    "            \n",
    "        self.first_orders = first_orders.to(self.device)\n",
    "        self.second_orders = second_orders.to(self.device)\n",
    "       \n",
    "        self.norm_num = nn.ModuleDict()\n",
    "        for s in self.num_features:\n",
    "            self.norm_num[s] = nn.BatchNorm1d(1)\n",
    "            \n",
    "        ######################################################dnn\n",
    "        self.p = params['p'] # drop_out\n",
    "        self.layers = params['layers']\n",
    "        self.input_dim = (len(self.single_features)+ len(self.muti_features) + len(self.topic_features) + len(self.word_features)) * self.k + len(self.num_features) + mem_dim #*self.k #* self.k #+ 2* self.k\n",
    "        self.deep_layers = nn.Sequential()\n",
    "        net_dims = [self.input_dim]+self.layers\n",
    "        for i in range(len(self.layers)):\n",
    "            self.deep_layers.add_module('fc%d' % (i+1), nn.Linear(net_dims[i], net_dims[i+1]))\n",
    "            self.deep_layers.add_module('bn%d' % (i+1), nn.BatchNorm1d(net_dims[i+1]))\n",
    "            self.deep_layers.add_module('relu%d' % (i+1), nn.ReLU()) \n",
    "            self.deep_layers.add_module('dropout%d' % (i+1), nn.Dropout(self.p))       \n",
    "        for name, tensor in self.deep_layers.named_parameters():\n",
    "            if 'weight' in name:\n",
    "                nn.init.normal_(tensor, mean=0, std=0.0001)\n",
    "        self.deep_layers = self.deep_layers.to(self.device)\n",
    "          \n",
    "        ## topic \n",
    "        if params['usetopic']:\n",
    "            self.topic_weight = nn.Embedding(params['textargs']['vocabulary_size'], 64)\n",
    "            self.topic_weight.weight.data.copy_(params['textargs']['pretrained_weight'])\n",
    "            self.topic_linear = nn.Sequential(nn.Linear(64, self.k), nn.ReLU())\n",
    "        \n",
    "        #word\n",
    "        if self.useword:\n",
    "            temp_weight = params['word2index']#word_weight\n",
    "            self.word_weight = nn.Embedding(len(params['word2index']), 64)\n",
    "            self.word_weight.weight.data.copy_(params['word_weight'])\n",
    "            self.word_linear = nn.Sequential(nn.Linear(64, self.k), nn.ReLU())\n",
    "        \n",
    "        #############################################################cin      \n",
    "        self.num_field = len(self.cross_features)+ len(self.muti_features) + len(self.topic_features) + len(self.word_features) #+ len(self.num_features)#+ 2\n",
    "        self.conv1ds = nn.ModuleList()\n",
    "        self.cin_layers = params['cin_layers']\n",
    "        cin_layers_dims = [self.num_field]+self.cin_layers\n",
    "        self.split_half = params['split_half']\n",
    "        self.hidden_dims_split_half = [self.num_field]\n",
    "        prev_dim = 0\n",
    "        for i in range(len(self.cin_layers)):\n",
    "            self.conv1ds.append(nn.Conv1d(cin_layers_dims[0]*self.hidden_dims_split_half[-1], cin_layers_dims[i+1], 1))\n",
    "            if self.split_half and i != len(self.cin_layers)-1:\n",
    "                self.hidden_dims_split_half.append(cin_layers_dims[i+1] // 2)\n",
    "                prev_dim += cin_layers_dims[i+1] // 2\n",
    "            else:\n",
    "                self.hidden_dims_split_half.append(cin_layers_dims[i+1])\n",
    "                prev_dim += cin_layers_dims[i+1]\n",
    "        self.conv1ds = self.conv1ds.to(self.device)\n",
    "              \n",
    "        if self.usetext:\n",
    "            textlen = self.textlen\n",
    "        else:\n",
    "            textlen = 0\n",
    "\n",
    "        self.output = nn.Sequential(nn.Linear(prev_dim+self.layers[-1]+ textlen + len(self.other_features), 512), # no-linear\n",
    "                                     nn.BatchNorm1d(512),\n",
    "                                     nn.ReLU(),\n",
    "                                     nn.Dropout(self.p),\n",
    "                                     nn.Linear(512, 256),\n",
    "                                     nn.BatchNorm1d(256),\n",
    "                                     nn.ReLU(),\n",
    "                                     nn.Dropout(self.p),\n",
    "                                     nn.Linear(256,64),\n",
    "                                     nn.BatchNorm1d(64),\n",
    "                                     nn.ReLU(),\n",
    "                                     nn.Dropout(self.p))\n",
    "        self.end = nn.Sequential(nn.Linear(64, 1),)       \n",
    "        \n",
    "    def forward(self, input_x):\n",
    "        embed1 = {}\n",
    "        embed2 = {}\n",
    "        norm_num = {}\n",
    "        for s in self.single_features:             \n",
    "            embed1[s] = self.first_orders[s](input_x[:,self.feature_index[s][0]:self.feature_index[s][1]].long()).squeeze(-1) #B * 1 \n",
    "            embed2[s] = self.second_orders[s](input_x[:,self.feature_index[s][0]:self.feature_index[s][1]].long()) #b*1*k\n",
    "            \n",
    "        if self.usetopic:\n",
    "            topics = {}\n",
    "            for s in self.topic_features:\n",
    "                if s not in ['hist_user_themes', 'likethemes', 'hist_user_unlike_themes']:#'likethemes', \n",
    "                    temp = (torch.sum(sequence_mask(self.topic_weight(input_x[:,self.feature_index[s][0]:self.feature_index[s][1]].long()),\n",
    "                                                    input_x[:,self.feature_index[s][0]:self.feature_index[s][1]].long()), dim=1)/(torch.sum((input_x[:,self.feature_index[s][0]:self.feature_index[s][1]]!=1).float()+1e-10, dim=-1).unsqueeze(-1)))#.unsqueeze(1)# b * 1 * k\n",
    "                    topics[s] = self.topic_linear(temp).unsqueeze(1)\n",
    "                else:             \n",
    "                    s_att = s+'_att'\n",
    "                    s_value = self.topic_weight(input_x[:,self.feature_index[s][0]:self.feature_index[s][1]].long())\n",
    "                    s_att_val = input_x[:,self.feature_index[s_att][0]:self.feature_index[s_att][1]].unsqueeze(-1).expand_as(s_value).float()\n",
    "                    temp = torch.sum(s_value * s_att_val, dim=1)/(torch.sum((input_x[:,self.feature_index[s_att][0]:self.feature_index[s_att][1]]).float()+1e-10, dim=-1).unsqueeze(-1))                          \n",
    "                    temp = temp +1e-12\n",
    "                    topics[s] = self.topic_linear(temp).unsqueeze(1)\n",
    "        \n",
    "        if self.useword:\n",
    "            words = {}\n",
    "            for s in self.word_features:\n",
    "                if s in ['all_words']:     \n",
    "                    temp = (torch.sum(sequence_mask(self.word_weight(input_x[:,self.feature_index[s][0]:self.feature_index[s][1]].long()),\n",
    "                                                            input_x[:,self.feature_index[s][0]:self.feature_index[s][1]].long()), dim=1)/(torch.sum((input_x[:,self.feature_index[s][0]:self.feature_index[s][1]]!=1).float()+1e-10, dim=-1).unsqueeze(-1)))#.unsqueeze(1)# b * 1 * k\n",
    "                    words[s] = self.word_linear(temp).unsqueeze(1)\n",
    "                else:\n",
    "                    s_att = s+'_att'\n",
    "                    s_value = self.word_weight(input_x[:,self.feature_index[s][0]:self.feature_index[s][1]].long())\n",
    "                    s_att_val = input_x[:,self.feature_index[s_att][0]:self.feature_index[s_att][1]].unsqueeze(-1).expand_as(s_value).float()\n",
    "                    temp = torch.sum(s_value * s_att_val, dim=1)/(torch.sum((input_x[:,self.feature_index[s_att][0]:self.feature_index[s_att][1]]).float()+1e-10, dim=-1).unsqueeze(-1))                          \n",
    "                    temp = temp +1e-12\n",
    "                    words[s] = self.word_linear(temp).unsqueeze(1)\n",
    "        ##other\n",
    "        others_num = []\n",
    "        for s in self.other_features:\n",
    "            others_num.append(input_x[:,self.feature_index[s][0]:self.feature_index[s][1]])\n",
    "        others_num = torch.cat(others_num, dim=1)\n",
    "                    \n",
    "        if self.norm:   \n",
    "            for s in self.num_features:\n",
    "                norm_num[s] = self.norm_num[s](input_x[:,self.feature_index[s][0]:self.feature_index[s][1]])\n",
    "        else:\n",
    "            for s in self.num_features:\n",
    "                norm_num[s] = input_x[:,self.feature_index[s][0]:self.feature_index[s][1]]\n",
    "                \n",
    "        \n",
    "        ## mem\n",
    "        if self.usemem:\n",
    "            mem1 = self.mem1(input_x[:,self.feature_index['writerId'][0]:self.feature_index['writerId'][1]].long()).squeeze(1)\n",
    "            mem0 = self.mem0(input_x[:,self.feature_index['writerId'][0]:self.feature_index['writerId'][1]].long()).squeeze(1)\n",
    "        \n",
    "        # ###########################以下是MLP\n",
    "        y = []\n",
    "        input_size = 0\n",
    "        for s in embed2:\n",
    "            y.append(embed2[s])\n",
    "        #topic\n",
    "        if self.usetopic:\n",
    "            for s in topics:\n",
    "                y.append(topics[s])          \n",
    "        #words\n",
    "        if self.useword:\n",
    "            for s in words:\n",
    "                y.append(words[s])\n",
    "        \n",
    "        y = torch.cat(y,1)\n",
    "        input_size += len(embed2)* self.k\n",
    "        if self.usetopic:\n",
    "            if self.useword:\n",
    "                y = torch.reshape(y, [-1, (len(embed2)+len(topics)+len(words)) * self.k])\n",
    "            else:\n",
    "                y = torch.reshape(y, [-1, (len(embed2)+len(topics)) * self.k])\n",
    "        else:\n",
    "            y = torch.reshape(y, [-1, len(embed2)* self.k])\n",
    "          \n",
    "        temp = []\n",
    "        temp.append(y)        \n",
    "            \n",
    "        for s in self.num_features:\n",
    "            temp.append(norm_num[s])             \n",
    "        x = torch.cat(temp, -1)\n",
    "        \n",
    "        ## dnn_logits\n",
    "        dnn_logits = self.deep_layers(x) \n",
    "        # ##########################################################CIN \n",
    "        x = []\n",
    "        for s in self.muti_features+self.cross_features:\n",
    "            x.append(embed2[s])\n",
    "            \n",
    "        if self.usetopic:\n",
    "            for s in self.topic_features:\n",
    "                x.append(topics[s])\n",
    "                \n",
    "        #word\n",
    "        if self.useword:\n",
    "            for s in self.word_features:\n",
    "                x.append(words[s])\n",
    "        \n",
    "        x0 = torch.cat(x, 1)\n",
    "        res = []\n",
    "        x_list = [x0]\n",
    "        for k in range(1, len(self.cin_layers)+1):\n",
    "            z_k = torch.einsum('bhd,bmd->bhmd', x_list[-1], x_list[0])\n",
    "            z_k = z_k.reshape(x0.shape[0], x_list[-1].shape[1] * x0.shape[1], x0.shape[2])\n",
    "            x_k = self.conv1ds[k-1](z_k)\n",
    "            x_k = torch.relu(x_k)\n",
    "            if self.split_half and k != len(self.cin_layers):\n",
    "                next_hidden, hi = torch.split(x_k, x_k.shape[1] // 2, 1)\n",
    "            else:\n",
    "                next_hidden, hi = x_k, x_k\n",
    "\n",
    "            x_list.append(next_hidden)\n",
    "            res.append(hi)\n",
    "\n",
    "        res = torch.cat(res, dim=1)\n",
    "        res = torch.sum(res, dim=2)\n",
    "\n",
    "        if self.usetext:\n",
    "            textinfo = self.textcnn(input_x[:,self.feature_index['q_topic'][0]:self.feature_index['q_topic'][1]].long(),\n",
    "                            input_x[:,self.feature_index['m_interest_topic'][0]:self.feature_index['m_interest_topic'][1]].long())      \n",
    "            allput = torch.cat([dnn_logits, res, textinfo], dim=1)\n",
    "        else:  \n",
    "            allput = torch.cat([dnn_logits, res, others_num], dim=1)   \n",
    "        score = self.output(allput)\n",
    "        \n",
    "        if self.usemem:\n",
    "            with torch.no_grad():\n",
    "                mem_n = score.detach()        \n",
    "            m_info = F.tanh(self.linear1(torch.cat([mem0, mem1], dim=-1)))          \n",
    "            return self.end(score), mem_n, mem0, mem1   \n",
    "        else:\n",
    "            return self.end(score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### util\n",
    "主要对word，topic进行处理，将word，topic映射到对应的数值，方便进行embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## pad 1  加入topics信息\n",
    "def deal_text(textdata, params, maxlen=10):\n",
    "    temp = []\n",
    "    topic2ix = params['textargs']['topic2index']\n",
    "    \n",
    "    for text in textdata:\n",
    "        tps = str(text).split(',')\n",
    "        if '-1' in text:\n",
    "            rs = [1]* maxlen\n",
    "        else:\n",
    "            rs = list(map(lambda x: topic2ix[x],tps))\n",
    "            if len(rs)<=maxlen:\n",
    "                rs += [1]*(maxlen-len(rs))\n",
    "            else:\n",
    "                rs = rs[:maxlen]\n",
    "        temp.append(rs)\n",
    "        \n",
    "    return np.array(temp)\n",
    "\n",
    "def deal_text2(textdata, params):\n",
    "    temp = []\n",
    "    topic2ix = params['textargs']['topic2index']\n",
    "    weight = params['textargs']['pretrained_weight']\n",
    "    for text in textdata:\n",
    "        tps = str(text).split(',')\n",
    "        if '-1' in text:\n",
    "            rs = np.array([0]*64)\n",
    "        else:\n",
    "            rs = list(map(lambda x: np.array(weight[topic2ix[x]]), tps))\n",
    "            rs = np.mean(np.array(rs),axis=0)\n",
    "        temp.append(rs)\n",
    "        \n",
    "        \n",
    "    return np.array(temp)\n",
    "\n",
    "# 处理带权重的topics\n",
    "def deal_text3(textdata, params, maxlen=10):\n",
    "    temp = []\n",
    "    topic2ix = params['textargs']['topic2index']\n",
    "    temp_att = []\n",
    "    ix = 0\n",
    "\n",
    "    for text in textdata:\n",
    "        tps = str(text).split(',')\n",
    "        ix += 1\n",
    "        if '-1' in text:\n",
    "            rs = [1] * maxlen\n",
    "            rs_att = [0.0] * maxlen\n",
    "        else:\n",
    "            tps = list(filter(lambda x:'Infinity' not in x , tps))\n",
    "            rs = list(map(lambda x: topic2ix[x.split(':')[0]],tps))  \n",
    "            rs_att = list(map(lambda x: float(x.split(':')[1]),tps))\n",
    "            \n",
    "            if len(rs) <= maxlen:\n",
    "                rs += [1]*(maxlen-len(rs))\n",
    "                rs_att += [0.0] * (maxlen-len(rs_att))\n",
    "            else:\n",
    "                rs = rs[:maxlen]\n",
    "                rs_att = rs_att[:maxlen]\n",
    "                \n",
    "        temp.append(rs)\n",
    "        temp_att.append(rs_att)\n",
    "        \n",
    "    temp = np.array(temp)\n",
    "    temp_att = np.array(temp_att)\n",
    "    \n",
    "    return np.concatenate([temp,temp_att], axis=-1)\n",
    "\n",
    "# 权重word\n",
    "def deal_word(textdata, params, maxlen=10):\n",
    "    temp = []\n",
    "    topic2ix = params['word2index']\n",
    "    temp_att = []\n",
    "    ix = 0\n",
    "\n",
    "    for text in textdata:\n",
    "        tps = str(text).split(',')\n",
    "        ix += 1\n",
    "        if '-1' in text:\n",
    "            rs = [1] * maxlen\n",
    "            rs_att = [0.0] * maxlen\n",
    "        else:\n",
    "            tps = list(filter(lambda x:'Infinity' not in x , tps))\n",
    "            rs = list(map(lambda x: topic2ix[x.split(':')[0]],tps))  \n",
    "            rs_att = list(map(lambda x: float(x.split(':')[1]),tps))\n",
    "            \n",
    "            if len(rs) <= maxlen:\n",
    "                rs += [1]*(maxlen-len(rs))\n",
    "                rs_att += [0.0] * (maxlen-len(rs_att))\n",
    "            else:\n",
    "                rs = rs[:maxlen]\n",
    "                rs_att = rs_att[:maxlen]\n",
    "                \n",
    "        temp.append(rs)\n",
    "        temp_att.append(rs_att)\n",
    "        \n",
    "    temp = np.array(temp)\n",
    "    temp_att = np.array(temp_att)\n",
    "    \n",
    "    return np.concatenate([temp,temp_att], axis=-1)\n",
    "\n",
    "# 非权重\n",
    "def deal_word2(textdata, params, maxlen=10):\n",
    "    temp = []\n",
    "    topic2ix = params['word2index']\n",
    "    \n",
    "    for text in textdata:\n",
    "        tps = str(text).split(',')\n",
    "        if '-1' in text:\n",
    "            rs = [1]* maxlen\n",
    "        else:\n",
    "            rs = list(map(lambda x: topic2ix[x],tps))\n",
    "            if len(rs)<=maxlen:\n",
    "                rs += [1]*(maxlen-len(rs))\n",
    "            else:\n",
    "                rs = rs[:maxlen]\n",
    "        temp.append(rs)\n",
    "        \n",
    "    return np.array(temp) \n",
    "\n",
    "import os\n",
    "def setup_seed(seed):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train test\n",
    "本部分主要是模型的训练（使用Lookahead+Adam 优化器），验证，预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def eval(model, devloader, params):\n",
    "    preds = []    \n",
    "    print('eval')\n",
    "    model.eval()\n",
    "    trues = []\n",
    "    for x,y in devloader:\n",
    "        with torch.no_grad():\n",
    "            x = x.to(params['device']).float()\n",
    "            if params['usemem']:\n",
    "                score, _, _, _ = model(x)\n",
    "            else:\n",
    "                score = model(x)\n",
    "            preds+=score.cpu().reshape(-1).tolist()\n",
    "            trues+=y.cpu().reshape(-1).tolist()\n",
    "    auc = roc_auc_score(trues, preds)\n",
    "    print('auc: ', auc)\n",
    "    return auc\n",
    "\n",
    "temp_x = 0\n",
    "def train(params, trainset, devset, foldix=0):\n",
    "    x = []\n",
    "    for i in params['single_features']+ params['num_features']+ params['other_features']:\n",
    "        x.append(np.expand_dims(trainset[i], axis=1))\n",
    "    if params['usetext']:\n",
    "        x.append(deal_text(trainset['attentionthemes'], params, maxlen=10))\n",
    "        x.append(deal_text(trainset['themeId'], params, maxlen=10))\n",
    "    if params['usetopic']:\n",
    "        x.append(deal_text(trainset['attentionthemes'], params, maxlen=100))  \n",
    "        x.append(deal_text(trainset['themeId'], params, maxlen=13))\n",
    "        x.append(deal_text3(trainset['likethemes'], params, maxlen=10))#w_topics1\n",
    "        #hist_user_themes_att\n",
    "        x.append(deal_text3(trainset['hist_user_themes'], params, maxlen=10))\n",
    "    if params['useword']:\n",
    "        x.append(deal_word(trainset['hist_user_words'], params, maxlen=20))\n",
    "        x.append(deal_word2(trainset['all_words'], params, maxlen=10))\n",
    "\n",
    "    #hist_user_unlike_themes\n",
    "    x.append(deal_text3(trainset['hist_user_unlike_themes'], params, maxlen=10))\n",
    "    #hist_user_unlike_words\n",
    "    x.append(deal_word(trainset['hist_user_unlike_words'], params, maxlen=20))\n",
    "    \n",
    "    train_tensor_data = Data.TensorDataset(torch.from_numpy(np.concatenate(x, axis=-1)),\n",
    "                torch.from_numpy(np.expand_dims(trainset['label'], axis=1)))   \n",
    "\n",
    "    trainloader = Data.DataLoader(\n",
    "        dataset=train_tensor_data,\n",
    "        batch_size=params['batch_size'],\n",
    "        shuffle=not params['usemem'],\n",
    "        num_workers=0,\n",
    "    )\n",
    "    del trainset\n",
    "    gc.collect()\n",
    "    print('train load ok')\n",
    "    x_val = []    \n",
    "    for i in params['single_features']+ params['num_features']+ params['other_features']:\n",
    "        x_val.append(np.expand_dims(devset[i], axis=1))\n",
    "    if params['usetext']:\n",
    "        x_val.append(deal_text(devset['attentionthemes'], params, maxlen=10))\n",
    "        x_val.append(deal_text(devset['themeId'], params, maxlen=10))  \n",
    "    \n",
    "    if params['usetopic']:\n",
    "        x_val.append(deal_text(devset['attentionthemes'], params, maxlen=100))  \n",
    "        x_val.append(deal_text(devset['themeId'], params, maxlen=13))\n",
    "        x_val.append(deal_text3(devset['likethemes'], params, maxlen=10))\n",
    "        x_val.append(deal_text3(devset['hist_user_themes'], params, maxlen=10))\n",
    "    if params['useword']:#\n",
    "        x_val.append(deal_word(devset['hist_user_words'], params, maxlen=20))\n",
    "        x_val.append(deal_word2(devset['all_words'], params, maxlen=10))\n",
    "\n",
    "    x_val.append(deal_text3(devset['hist_user_unlike_themes'], params, maxlen=10))\n",
    "    #hist_user_unlike_words\n",
    "    x_val.append(deal_word(devset['hist_user_unlike_words'], params, maxlen=20))\n",
    "    \n",
    "    dev_tensor_data = Data.TensorDataset(torch.from_numpy(np.concatenate(x_val, axis=-1)),\n",
    "            torch.from_numpy(np.expand_dims(devset['label'], axis=1)))   \n",
    "\n",
    "    devloader = Data.DataLoader(\n",
    "        dataset=dev_tensor_data,\n",
    "        batch_size=params['batch_size'],\n",
    "        shuffle=False,\n",
    "        num_workers=0,\n",
    "    )\n",
    "    print('dev loader ok')\n",
    "    model = xDeepFM(params)\n",
    "    model.to(params['device'])\n",
    "    base_optimizer = torch.optim.Adam(model.parameters(), lr=params['lr'], weight_decay=params['l2'])\n",
    "    optimizer = Lookahead(base_optimizer=base_optimizer,k=5,alpha=0.5)\n",
    "    best_auc = 0\n",
    "    criterion = nn.MSELoss() ## mem\n",
    "    maxauc = 0\n",
    "    for epoch in tqdm.tqdm_notebook(range(params['epochs'])):\n",
    "        all_loss = 0\n",
    "        i=0     \n",
    "        all_loss_mem = 0\n",
    "        for x, y in tqdm.tqdm_notebook(trainloader):\n",
    "            x = x.to(params['device']).float()\n",
    "            y = y.to(params['device']).float() \n",
    "            if params['usemem']:\n",
    "                score, mem_n, mem0, mem1 = model(x)\n",
    "                loss = F.binary_cross_entropy_with_logits(score, y)\n",
    "                mem = (1-y.expand_as(mem_n)) * mem0 + y.expand_as(mem_n) * mem1\n",
    "                memloss = criterion(mem, mem_n)\n",
    "                loss = loss + memloss\n",
    "                all_loss_mem += memloss.detach().cpu().item()\n",
    "            else:\n",
    "                score = model(x) \n",
    "                loss = F.binary_cross_entropy_with_logits(score, y)\n",
    "                \n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            i+=1\n",
    "            optimizer.step()\n",
    "            all_loss += loss.detach().cpu().item()\n",
    "            if i % params['num_display_steps']==0:\n",
    "                if params['usemem']:\n",
    "                    #print('mem_loss:', all_loss_mem/params['num_display_steps'])\n",
    "                    all_loss_mem = 0\n",
    "                all_loss = 0\n",
    "        auc = eval(model, devloader, params)\n",
    "        model.train()\n",
    "        if auc > maxauc:\n",
    "            maxauc = auc\n",
    "        torch.save(model.state_dict(), datapath+'data/'+'_auc_'+ str(auc)+'_'+str(foldix)+'.pth')\n",
    "        print(datapath+'data/'+'_auc_'+ str(auc)+'_'+str(foldix)+'.pth saved!')\n",
    "\n",
    "    return datapath+'data/'+'_auc_'+ str(maxauc)+'_'+str(foldix)+'.pth'\n",
    "\n",
    "def test(params, testset, testpath, onefold=False):\n",
    "    model = xDeepFM(params)\n",
    "    model.load_state_dict(torch.load(testpath))\n",
    "    model.to(params['device'])\n",
    "    \n",
    "    x = []\n",
    "    for i in params['single_features']+ params['num_features'] + params['other_features']:\n",
    "        x.append(np.expand_dims(testset[i], axis=1))\n",
    "    if params['usetext']:\n",
    "        x.append(deal_text(testset['attentionthemes'], params, maxlen=10))\n",
    "        x.append(deal_text(testset['themeId'], params, maxlen=10)) \n",
    "\n",
    "    if params['usetopic']:\n",
    "        x.append(deal_text(testset['attentionthemes'], params, maxlen=100))  \n",
    "        x.append(deal_text(testset['themeId'], params, maxlen=13))\n",
    "        x.append(deal_text3(testset['likethemes'], params, maxlen=10))#w_topics1\n",
    "        x.append(deal_text3(testset['hist_user_themes'], params, maxlen=10))\n",
    "\n",
    "    if params['useword']:\n",
    "        x.append(deal_word(testset['hist_user_words'], params, maxlen=20))\n",
    "        x.append(deal_word2(testset['all_words'], params, maxlen=10))\n",
    "    #hist_user_unlike_themes\n",
    "    x.append(deal_text3(testset['hist_user_unlike_themes'], params, maxlen=10))\n",
    "    #hist_user_unlike_words\n",
    "    x.append(deal_word(testset['hist_user_unlike_words'], params, maxlen=20))\n",
    "    \n",
    "    test_tensor_data = Data.TensorDataset(torch.from_numpy(np.concatenate(x, axis=-1)),\n",
    "                torch.from_numpy(np.expand_dims(testset['label'], axis=1)))   \n",
    "\n",
    "    testloader = Data.DataLoader(\n",
    "        dataset=test_tensor_data,\n",
    "        batch_size=params['batch_size'],\n",
    "        shuffle=False,\n",
    "        num_workers=0,\n",
    "    )\n",
    "    preds = []\n",
    "    model.eval()\n",
    "    for x, y in testloader:\n",
    "        x = x.to(params['device']).float()\n",
    "        score = model(x)\n",
    "        score = torch.sigmoid(score)\n",
    "        preds+=score.cpu().reshape(-1).tolist()\n",
    "    if  onefold:    \n",
    "        testdata = pd.read_csv(datapath+'data/m_q_invite_test.csv') \n",
    "        testdata['label'] = pd.Series(preds)\n",
    "        testdata[['q_id', 'm_id', 'invite_time', 'label']].to_csv(datapath+'submit.txt', index = None, sep = '\\t', header=None)\n",
    "        print('test finished!')\n",
    "    return preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fold\n",
    "本部分包括随机5折、按时间顺序5折、5折的测试，对比随机5折和按时间顺序5折效果，发现随机5折的效果更好"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_fold(trainset, params, fold=5):\n",
    "    train_y = trainset['label']\n",
    "    train_x = trainset\n",
    "    rs_list = []\n",
    "    pre_train = pd.Series(np.zeros(len(train_y)))\n",
    "    \n",
    "    kf = StratifiedKFold(n_splits = fold,shuffle = True,random_state = 2019)\n",
    "    for ix,(train_index,eval_index) in enumerate(kf.split(train_x,train_y)):\n",
    "        dtrain_x = train_x.loc[train_index,:]\n",
    "        deval_x = train_x.loc[eval_index,:]\n",
    "        rs = train(params, dtrain_x, deval_x, ix)\n",
    "        pre_train[eval_index] = test(params, deval_x, rs) \n",
    "        rs_list.append(rs)\n",
    "    \n",
    "    train_pre = trainset[['qId', 'writerId', 'inviteday', 'label', 'invitehour']]\n",
    "    train_pre['pre_label'] = pre_train\n",
    "    with open(datapath + 'train-fold-3zhou.pkl', 'wb') as f:\n",
    "        pickle.dump(train_pre, f, protocol=4)\n",
    "    \n",
    "    file = codecs.open(datapath + 'fold_list_3zhou.txt', 'w')\n",
    "    file.write(','.join(rs_list))\n",
    "    print('train -fold end!')\n",
    "\n",
    "def train_fold_time(trainset, params, fold=5):\n",
    "    train_x = trainset\n",
    "    rs_list = []\n",
    "    pre_train = pd.Series(np.zeros(len(train_x)))\n",
    "    \n",
    "    start = train_x['inviteday'].min()\n",
    "    block_len = np.ceil(float(3868-start)/fold)\n",
    "    for i in range(5): \n",
    "        bool_eval = (train_x['inviteday']>=start)&(train_x['inviteday']<start+block_len)\n",
    "        bool_train = ~bool_eval\n",
    "        train_index = train_x[bool_train].index\n",
    "        eval_index = train_x[bool_eval].index\n",
    "        start = start+block_len\n",
    "        ###\n",
    "        dtrain_x = train_x.loc[train_index,:]\n",
    "        deval_x = train_x.loc[eval_index,:]\n",
    "        print(str(deval_x['inviteday'].min())+':'+str(deval_x['inviteday'].max()))\n",
    "        rs = train(params, dtrain_x, deval_x, i)\n",
    "        pre_train[eval_index] = test(params, deval_x, rs) \n",
    "        rs_list.append(rs)\n",
    "    \n",
    "    train_pre = trainset[['qId', 'writerId', 'inviteday', 'label', 'invitehour']]\n",
    "    train_pre['pre_label'] = pre_train\n",
    "    with open(datapath + 'train-fold-time.pkl', 'wb') as f:\n",
    "        pickle.dump(train_pre, f, protocol=4)\n",
    "    \n",
    "    file = codecs.open(datapath + 'fold_list-time.txt', 'w')\n",
    "    file.write(','.join(rs_list))\n",
    "    print('train -fold end!')    \n",
    "\n",
    "\n",
    "def test_fold(testset, params, modelList=None): # modelist= './fold_list.txt'\n",
    "    if modelList==None and os.path.exists(datapath + 'fold_list.txt'):\n",
    "        file = codecs.open(datapath+'fold_list.txt', 'r')\n",
    "        modelList = file.read().split(',')\n",
    "        print('加载 fold-list end!')\n",
    "    else:\n",
    "        file = codecs.open(modelList, 'r')\n",
    "        modelList = file.read().split(',')\n",
    "        print('加载 fold-list end!')\n",
    "        \n",
    "    pre_test = []\n",
    "    for i in modelList:\n",
    "        pre = test(params, testset, i) \n",
    "        pre_test.append(pre)\n",
    "        print(pre[:30])\n",
    "    pre_test = np.array(pre_test)\n",
    "    pre_test = np.mean(pre_test,axis = 0)\n",
    "    sub_sample = pd.read_csv(datapath + 'data/data_invite_test.csv').drop(['inviteday','invitehour'],axis = 1)\n",
    "    sub_sample['label'] = pre_test\n",
    "    sub_sample.to_csv(datapath+'submit-test-3zhou.txt',sep = '\\t',header = False,index = False)\n",
    "    print('test-fold finished!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型参数设置\n",
    "- 1、加载相关文件：类别特征的字典， topic的权重， topic转化字典， word的权重， word转化字典\n",
    "- 2、特征类别：单一特征，多值特征，交叉特征， topic特征， word特征， 其他特征（deepWalk）\n",
    "- 3、模型结构：比赛中尝试对模型结构进行改动，包括添加memory结构，想要得到用户历史信息；利用textcnn结构获取topic的特征；默认参数是达到最好结果的参数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 禁用topic时，注意feature=[] , usetopic=False\n",
    "setup_seed(1024)\n",
    "with open(datapath+'data/dic_all.pkl', 'rb') as f:\n",
    "    dic = pickle.load(f)\n",
    "with open(datapath+'data/topic_weight_64.pkl', 'rb') as f:\n",
    "    weight = pickle.load(f)\n",
    "with open(datapath+'data/topic2index.pkl', 'rb') as f:\n",
    "    topic2index = pickle.load(f)   \n",
    "    \n",
    "with open(datapath+'data/word_weight_64.pkl', 'rb') as f:\n",
    "    word_weight = pickle.load(f)\n",
    "with open(datapath+'data/word2index.pkl', 'rb') as f:\n",
    "    word2index = pickle.load(f) \n",
    "    \n",
    "    \n",
    "textargs = {\n",
    "    'filter_num': 64,\n",
    "    'filter_sizes':[2,3,4],\n",
    "    'vocabulary_size':weight.size()[0],\n",
    "    'embedding_dim':64,\n",
    "    'dropout':0,\n",
    "    'topic2index':topic2index,\n",
    "    'pretrained_weight':weight,\n",
    "}\n",
    "\n",
    "params = {\n",
    "    'k':8, #embedding dims 8\n",
    "    'batch_size':8000,\n",
    "    'lr':0.0005,\n",
    "    'l2':0.00001,\n",
    "    'device': torch.device('cuda:1' if torch.cuda.is_available() else \"cpu\"),\n",
    "    'p' : 0.5,\n",
    "    'single_features':['invitehour', 'createhour', 'inviteweekday', 'createweekday', 'sex', 'activity', 'bool_A',\n",
    "       'bool_B', 'bool_C', 'bool_D', 'bool_E', 'category_E', 'category_A', 'category_B', 'category_C', 'category_D'],\n",
    "    'cross_features':['invitehour', 'createhour', 'inviteweekday', 'createweekday', 'sex', 'activity', 'bool_A',\n",
    "       'bool_B', 'bool_C', 'bool_D', 'bool_E', 'category_E', 'category_A', 'category_B','category_C', 'category_D'],\n",
    "    'muti_features':[],\n",
    "    'topic_features':['attentionthemes', 'themeId', 'likethemes', 'hist_user_themes'], #hist_user_unlike_themes 加上这个特征效果稍稍变差，但是在模型融合的时候有帮助 \n",
    "    'other_features': ['qId_dw'+str(i) for i in range(32)] + ['writerId_dw'+str(i) for i in range(32)] ,\n",
    "    'word_features':['hist_user_words', 'all_words'],#hist_user_unlike_words 加上这个特征效果稍稍变差，但是在模型融合的时候有帮助 \n",
    "    'layers':[1024, 512], # dnn\n",
    "    'cin_layers':[128, 128], ### cin\n",
    "    'num_features': new_dense,#list(set(numPart) - set(['sameqnum_like'])) ,#+ ['qId_dw'+str(i) for i in range(32)], writerId_dw28\n",
    "    'num_display_steps':100,\n",
    "    'num_eval_steps':1000,\n",
    "    'epochs':50,\n",
    "    'split_half':True,\n",
    "    'normNum':False,\n",
    "    'textargs':textargs,\n",
    "    'usetext':False,\n",
    "    'usecin':True,\n",
    "    'usemem':False,\n",
    "    'usetopic':True,\n",
    "    'useword':True,\n",
    "    'word2index':word2index,\n",
    "    'word_weight':word_weight,\n",
    "}\n",
    "\n",
    "\n",
    "params['dic'] = dic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_fold(trainset, params)  \n",
    "test_fold(testset, params, datapath+'fold_list_3zhou.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# .Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load test\n",
    "test = pd.read_csv(datapath+'data/pre1_lgb_0.1.csv') # lgb_0.1_nonew.txt\n",
    "test = test[['lgb_pre1']]\n",
    "\n",
    "temp = pd.read_csv(datapath+'data/pre1_cat_0.1.csv') # cat3.txt\n",
    "temp = temp[['cat_pre1']]\n",
    "test = pd.concat([test,temp], axis=1)\n",
    "\n",
    "temp = pd.read_csv(datapath+'data/xgb_0.1.csv') # cat3.txt\n",
    "temp = temp[['xgb_pre1']]\n",
    "test = pd.concat([test,temp], axis=1)\n",
    "\n",
    "temp = pd.read_csv(datapath+'data/submit-test-3zhou.txt', sep = '\\t', header = None) \n",
    "temp.columns = ['qid','uid','inv_time','nn_pre1']\n",
    "temp = temp[['nn_pre1']]\n",
    "test = pd.concat([test,temp], axis=1)\n",
    "\n",
    "train_y = pd.read_csv(datapath+'train_y.csv', header = None)\n",
    "train_y.columns = ['label']\n",
    "train_y = train_y[['label']]\n",
    "\n",
    "# stacking\n",
    "usecols = list(train.columns)\n",
    "pre_train = pd.Series(np.zeros(len(train_y)))\n",
    "pre_test = []\n",
    "kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2019)\n",
    "for train_index, eval_index in kf.split(train, train_y['label']):\n",
    "    params_lgbc = {\n",
    "        'boosting_type': 'gbdt',\n",
    "        'objective': 'binary',  \n",
    "        'num_leaves': 10,\n",
    "        'learning_rate': 0.3,\n",
    "        'feature_fraction': 0.8,\n",
    "        'bagging_fraction': 0.8,\n",
    "        'bagging_freq': 1,\n",
    "        'min_sum_hessian_in_leaf': 20,\n",
    "        'num_threads': cpu_count() - 1,\n",
    "        'seed': 7,\n",
    "        'n_estimators': 10000,  \n",
    "        'max_depth': 5,\n",
    "        'subsample': 0.9,\n",
    "        'subsample_freq': 2,\n",
    "        'reg_alpha': 0,\n",
    "        'reg_lambda': 2\n",
    "    }\n",
    "    dtrain_x = train.loc[train_index, usecols]\n",
    "    deval_x = train.loc[eval_index, usecols]\n",
    "    dtrain_y = train_y.loc[train_index, 'label']\n",
    "    deval_y = train_y.loc[eval_index, 'label']\n",
    "    lgbc = lgb.LGBMClassifier(random_state=2019, **params_lgbc)\n",
    "    lgbc.fit(dtrain_x, dtrain_y, eval_set=[(deval_x, deval_y)], eval_names=['eval'], eval_metric='auc',\n",
    "             early_stopping_rounds=10, verbose=10)\n",
    "    pre_train[eval_index] = lgbc.predict_proba(deval_x)[:, 1]\n",
    "    pre_test.append(list(lgbc.predict_proba(test.loc[:, usecols].values)[:, 1]))\n",
    "\n",
    "pre_test = np.array(pre_test)\n",
    "pre_test = np.mean(pre_test, axis=0)\n",
    "score = roc_auc_score(train_y, pre_train)\n",
    "\n",
    "sub_sample = pd.read_csv(datapath+'invite_info_evaluate_2_0926.txt',sep = '\\t',header = None,names = ['qId','writerId','invitetime'])\n",
    "sub_sample['label'] = pre_test\n",
    "\n",
    "sub_sample.to_csv(datapath+'data/sub.txt',sep = '\\t',header = False,index = False)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}