method1-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.sql.types import StringType\n",
    "import time\n",
    "import numpy as np\n",
    "\n",
    "sc =SparkContext.getOrCreate()\n",
    "sqlContext = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readCSV(path):\n",
    "    '''\n",
    "    read csv file\n",
    "    return spark sql data frame\n",
    "    '''\n",
    "    return sqlContext.read.format(\"csv\").options(header=\"true\")\\\n",
    "    .load(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "team_df = readCSV(\"/Users/peggy/Desktop/footballManager/team_feat.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "player_df = readCSV(\"/Users/peggy/Desktop/footballManager/data_clean.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def playerSimilarity(p1, p2):\n",
    "    '''\n",
    "    length of p2 times cosine of p1 & p2\n",
    "    '''\n",
    "    cosine = np.dot(p1,p2)/(np.linalg.norm(p1)*(np.linalg.norm(p2)))\n",
    "    r =  np.sqrt(sum([i ** 2 for i in p1]))\n",
    "    return r * cosine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findTopK(playerList, K, player, sort_type):\n",
    "    playerList.append(player)\n",
    "    playerList.sort(key=lambda p: sort_type * p[1])\n",
    "    if(len(playerList) > K):\n",
    "        return playerList[:K]\n",
    "    return playerList\n",
    "\n",
    "\n",
    "def mergeTopK(pList1, pList2, K, sort_type):\n",
    "    result = pList1 + pList2\n",
    "    result.sort(key=lambda p:sort_type*p[1])\n",
    "    if(len(result) > K):\n",
    "        return result[:K]\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run time: 1.3768701553344727\n"
     ]
    }
   ],
   "source": [
    "def findSimilarPlayer(df, name, topK):\n",
    "    '''\n",
    "    given dataset and target player name\n",
    "    return top K most similar players data frame of target player\n",
    "    '''\n",
    "    player_df = df.select([\"ID\"] + df.columns[44:73]).where(df.Name == name)\n",
    "    if player_df == None:\n",
    "        raise NameError(\"No Player Found!\")\n",
    "    playerInfo = player_df.rdd.map(list)\\\n",
    "        .map(lambda l:(l[0], [int(l[i]) for i in range(1, len(l))])).collect()[0]\n",
    "    (playerId, playerList) = playerInfo[0], playerInfo[1]\n",
    "    mat = df.select([\"ID\"] + df.columns[44:73]).rdd.map(list)\\\n",
    "        .map(lambda l:(l[0], [int(l[i]) for i in range(1, len(l))]))\\\n",
    "        .filter(lambda kv: kv[0] != playerId)\\\n",
    "        .mapValues(lambda l: playerSimilarity(l, playerList))\n",
    "\n",
    "    res = mat.aggregate([], lambda inp1, inp2: findTopK(inp1, topK, inp2, -1), lambda inp1, inp2: mergeTopK(inp1, inp2, topK, -1))\n",
    "    res = [id for id, score in res]\n",
    "    id_df = sqlContext.createDataFrame(res, StringType()).toDF(\"ID\")\n",
    "    res = df.join(id_df, \"ID\", \"inner\").select(\"Name\", \"Age\", \"Nationality\", \"Club\", \"Height(cm)\", \"Weight(lbs)\")\n",
    "    return res\n",
    "    \n",
    "time1 = time.time()\n",
    "findSimilarPlayer(player_df, \"L. Messi\", 10)\n",
    "run_time = time.time() - time1\n",
    "print(\"run time: \" + str(run_time))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findBestReplicate(teamName, playerId, df, topK, weightVector):\n",
    "    '''\n",
    "    return list of [(player_id, replace_id, improve score)]\n",
    "    '''\n",
    "    player_info = df.select(df.columns[44:73]).where(df.ID == playerId).rdd.map(list)\\\n",
    "            .map(lambda l: [float(i) for i in l]).collect()[0] # list\n",
    "    candidatePlayers = df.select([\"ID\"] + df.columns[44:73]).where(df.Club != teamName).rdd.map(list)\\\n",
    "        .map(lambda l:(l[0], [float(l[i]) for i in range(1, len(l))]))\\\n",
    "        .mapValues(lambda vals: improve(vals, player_info, weightVector)) # rdd\n",
    "    res = candidatePlayers.aggregate([], lambda inp1, inp2: findTopK(inp1, topK, inp2, -1), lambda inp1, inp2: mergeTopK(inp1, inp2, topK, -1))\n",
    "    res = [(playerId, id, score) for id, score in res]\n",
    "    return res\n",
    "\n",
    "def improve(l1, l2, weight):\n",
    "    improve = 0\n",
    "    for i in range(len(l1)):\n",
    "        improve += (l1[i] - l2[i]) * weight[i]\n",
    "    return improve\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run time: 7.744986057281494\n"
     ]
    }
   ],
   "source": [
    "def featureThreshold(l):\n",
    "    temp = sorted(l)\n",
    "    return temp[int(len(l) / 4)]\n",
    "\n",
    "\n",
    "def findWorstFeatures(teamName, team_df):\n",
    "    '''\n",
    "    take the team name and team dataframe and return list of index of weak features start from 0 = Crossing\n",
    "    '''\n",
    "    targ_df = team_df.select('*').where(team_df.Club == teamName).rdd.map(list)\\\n",
    "            .map(lambda l: (l[0], [float(l[i]) for i in range(1, len(l))]))\\\n",
    "            .mapValues(lambda l: (featureThreshold(l), l))\\\n",
    "            .mapValues(lambda tup: [index for index, val in enumerate(tup[1]) if val < tup[0]])\n",
    "    feature_indexes = targ_df.collect()[0][1]\n",
    "    return feature_indexes\n",
    "    \n",
    "    \n",
    "def createWeightVector(feature_indexes):\n",
    "    '''\n",
    "    take list of weak features and return weight list of size 29\n",
    "    '''\n",
    "    norm = float(10 / (29 + len(feature_indexes)))\n",
    "    weightVector = [2.0 * norm if index in feature_indexes else norm for index in range(29)]\n",
    "    return weightVector\n",
    "     \n",
    "    \n",
    "def findWorstPlayers(teamName, player_df, feature_indexes):\n",
    "    '''\n",
    "    take team name, player dataframe, weak features index list\n",
    "    return list of worst players id\n",
    "    '''\n",
    "    worst_players = player_df.select([\"ID\"] + player_df.columns[44:73]).where(player_df.Club == teamName).rdd.map(list)\\\n",
    "            .map(lambda l: (l[0], [float(i) for i in l[1:]]))\\\n",
    "            .mapValues(lambda l: [l[i] for i in range(len(l)) if i in feature_indexes])\\\n",
    "            .mapValues(lambda l: sum(l)).collect()\n",
    "    worst_players.sort(key = lambda t: t[1], reverse=True)\n",
    "    return [id for id, index in worst_players][:10]\n",
    "\n",
    "\n",
    "    \n",
    "def replaceModeRecommendation(player_df, team_df, teamName, topK):\n",
    "    feature_indexes = findWorstFeatures(teamName, team_df)\n",
    "#     print([team_df.columns[i + 1] for i in feature_indexes])\n",
    "    weight_vector = createWeightVector(feature_indexes)\n",
    "#     print(weight_vector)\n",
    "    worst_players = findWorstPlayers(teamName, player_df, feature_indexes)\n",
    "    res = []\n",
    "    for player_id in worst_players:\n",
    "        res += findBestReplicate(teamName, player_id, player_df, topK, weight_vector)\n",
    "    res.sort(key = lambda l: l[2], reverse=True)\n",
    "    return res[:topK]\n",
    "    \n",
    "\n",
    "\n",
    "def printPlayerInfo(player_df, playerId):\n",
    "    player_info = player_df.select(\"ID\", 'Name', \"Age\", \"Nationality\", \"Overall\", \"Club\", \"Position\")\\\n",
    "            .where(player_df.ID == playerId).show()\n",
    "\n",
    "\n",
    "\n",
    "# team_name = 'FC Barcelona'\n",
    "time1 = time.time()\n",
    "team_name = 'LA Galaxy'\n",
    "res = replaceModeRecommendation(player_df, team_df, team_name, 3)\n",
    "print(\"run time: \" + str(time.time() - time1))\n",
    "# for i in res:\n",
    "#     print(\"player:\" + i[0] +\" replacement:\" + i[1] + \" improvement:\" + str(i[2]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}