Untitled1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = json.load(open('real-player.json','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data['ratings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(['fuzz','abbrev_if_new_row'],1)#.set_index(['slug','season'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index(['slug','season']).reset_index()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = list(df.columns[2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings  = {}\n",
    "for row in df.itertuples():\n",
    "    ratings[(row[1],row[2])] = list(row[3:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['bios']['jordami01']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings[('jordami01',1985)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "youth = []\n",
    "for k,p in data['bios'].items():\n",
    "    if 'bornYear' not in p or p['bornYear'] is None:\n",
    "        continue\n",
    "    yr = p['draftYear']\n",
    "    age = yr-p['bornYear']\n",
    "    if yr<2021 and yr >= 2000 and (k,yr+1) in ratings and age < 25 and p['draftPick'] > 0:# and p['draftPick'] < 45:\n",
    "        youth.append([age] + ratings[(k,yr+1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "years = defaultdict(list)\n",
    "corr = []\n",
    "for k,yr in ratings:\n",
    "    if (k,yr+1) in ratings and data['bios'][k]['bornYear'] is not None:\n",
    "        d1 = np.array(ratings[(k,yr+1)])-np.array(ratings[(k,yr)])\n",
    "        years[yr-data['bios'][k]['bornYear']].append(d1)\n",
    "        if (k,yr+2) in ratings:\n",
    "            d2 = np.array(ratings[(k,yr+2)])-np.array(ratings[(k,yr+1)])\n",
    "            assert(len(d1) == 15)\n",
    "            assert(len(d2) == 15)\n",
    "            corr.append(list(d1) + list(d2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = np.array(corr)\n",
    "for i in range(15):\n",
    "    plt.figure()\n",
    "    plt.scatter(corr[:,i],corr[:,15+i],alpha=0.1)\n",
    "    plt.title(cols[i])\n",
    "    std = 4*corr[:,1].std()\n",
    "    plt.xlim(-std,std)\n",
    "    plt.ylim(-std,std)\n",
    "    plt.grid(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('seaborn-white')\n",
    "xy = np.array([[k,np.mean(years[k])] for k in sorted(years.keys())])\n",
    "xyS = np.array([[k,np.std(years[k])] for k in sorted(years.keys())])\n",
    "\n",
    "plt.plot(xy[:,0],xy[:,1],label='nba')\n",
    "plt.fill_between(xy[:,0],xy[:,1]+xyS[:,1],xy[:,1]-xyS[:,1],alpha=0.1)\n",
    "\n",
    "plt.plot(np.arange(19,40),-0.3*(np.arange(19,40)-19)+2.2,label='formula')\n",
    "#plt.plot(np.arange(20,40),-0.12*(np.arange(20,40)-20))\n",
    "plt.legend()\n",
    "plt.plot([0,100],[0,0],c='k')\n",
    "plt.xlim(19,30)\n",
    "plt.ylim(-10,10)\n",
    "plt.xlabel('age')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xyS[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(8,8))\n",
    "plz = np.array([np.mean(years[i],0) for i in range(20,35)]).T - (-0.3*(np.arange(20,35)-19)+2.2)\n",
    "plt.imshow(plz,vmin=-5,vmax=5,cmap='RdBu')\n",
    "plt.colorbar(orientation='horizontal',fraction=0.046, pad=0.04)\n",
    "plt.xticks(np.arange(20,35)-20,np.arange(20,35))\n",
    "plt.yticks(np.arange(len(cols)),cols)\n",
    "plt.xlabel('age')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plz.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,c in enumerate(cols):\n",
    "\n",
    "    plt.figure()\n",
    "    plt.subplot(1,2,1)\n",
    "    plt.title(c)\n",
    "    plt.plot(np.arange(20,35),plz[i,:])#- xy[1:16,1])\n",
    "    #plt.plot(np.arange(20,35),-0.33*(np.arange(20,35)-20)+2)\n",
    "    #plt.plot(np.arange(20,35),-0.5*(np.arange(20,35)-20)+4)\n",
    "    mdl2 = sm.OLS(plz[i,:],sm.add_constant(np.arange(20,35)-20)).fit()\n",
    "    plt.plot(np.arange(20,35),mdl2.predict())\n",
    "    plt.title(c + ' ' + str(list(np.round(mdl2.params,1))))\n",
    "\n",
    "    plt.subplot(1,2,2)\n",
    "    plt.plot(np.arange(20,35),mdl2.predict()-0.3*(np.arange(20,35)-19)+2.2)\n",
    "    plt.plot([20,35],[0,0],c='k')\n",
    "    print(c,mdl2.params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mdl2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.array([[k,np.quantile(years[k],0.5)] for k in sorted(years.keys())])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(xyS[:,0],xyS[:,1])\n",
    "plt.xlim(right=35)\n",
    "plt.ylim(top=8)\n",
    "plt.xlabel('age')\n",
    "plt.ylabel('rating std dev')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.array(years[23]).mean(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "youth = np.array(youth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pts = youth[:,1:]\n",
    "print(youth.mean(0))\n",
    "for i,c in enumerate(cols):\n",
    "    print(c,np.round(youth.mean(0)-3)[i+1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import umap\n",
    "clf = umap.UMAP(a=15,b=0.4)\n",
    "emb = clf.fit_transform(pts.astype(np.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(emb[:,0],emb[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.mixture import GaussianMixture\n",
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clfk = KMeans(3,n_init=100)\n",
    "clfk.fit(pts.astype(np.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(clfk.cluster_centers_-clfk.cluster_centers_.mean(0),columns=cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(clfk.labels_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.round(clfk.cluster_centers_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(3):\n",
    "    print(np.round(np.diag(np.cov(pts[clfk.labels_==i].T))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clfg = GaussianMixture(3,means_init=clfk.cluster_centers_,covariance_type='diag')\n",
    "clfg.fit(pts.astype(np.float32))\n",
    "clfg.means_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clfg.weights_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(clfg.means_/pts.mean(0),columns=cols,index=['big','point','wing']).round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.round(pts.mean(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "clfp = PCA()\n",
    "clfp.fit(np.array(years[23])-np.array(years[23]).mean(0)) # \n",
    "clfp.explained_variance_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(years[23],columns=cols).corr().round(1).style.background_gradient(cmap='Greens',low=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(c,r) for c,r in zip(cols,np.round(clfp.components_[0],1)) if abs(r) > 0.1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(c,r) for c,r in zip(cols,np.round(clfp.components_[1],1)) if abs(r) > 0.1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(c,r) for c,r in zip(cols,np.round(clfp.components_[2],1)) if abs(r) > 0.1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(c,r) for c,r in zip(cols,np.round(clfp.components_[3],1)) if abs(r) > 0.1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cov = np.round(np.cov(np.array(years[23]).T))\n",
    "#cov = np.round(np.cov(pts.T))\n",
    "\n",
    "for i,c in enumerate(cols):\n",
    "    print(c,cov[i,i],sorted([(round(cov[i,j]/cov[i,i],1),_) for j,_ in enumerate(cols)],reverse=True)[:5])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(c,r) for c,r in zip(cols,np.round(0.9*pts.mean(0)))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import FeatureAgglomeration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 3\n",
    "clff = FeatureAgglomeration(N)\n",
    "clff.fit(years[23])\n",
    "for i in range(N):\n",
    "    print(np.array(cols)[clff.labels_ == i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "years2 = {k:np.array(v) for k,v in years.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,c in enumerate(cols):\n",
    "    for j in range(20,35):\n",
    "        print(c, j, np.quantile(years2[j][:,i],0.05),np.quantile(years2[j][:,i],0.95))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "years2[23]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "years2[25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "years[25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}