skill2vec_ver2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import cPickle \n",
    "from nltk.corpus import stopwords\n",
    "from gensim.models import word2vec\n",
    "import nltk.data\n",
    "import re\n",
    "import logging\n",
    "from nltk.stem.snowball import *\n",
    "import itertools\n",
    "\n",
    "# Python 2.x:\n",
    "import HTMLParser\n",
    "html_parser = HTMLParser.HTMLParser()\n",
    "\n",
    "import multiprocessing\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)   \n",
    "\n",
    "from nltk.stem import WordNetLemmatizer, PorterStemmer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()\n",
    "porter = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "TEST_CASE = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'angular'"
      ]
     },
     "execution_count": 394,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import HTMLParser\n",
    "html_parser = HTMLParser.HTMLParser()\n",
    "from nltk.stem import WordNetLemmatizer, PorterStemmer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()\n",
    "porter = PorterStemmer()\n",
    "\n",
    "def skill_transform(skill, remove_stopwords = True):\n",
    "    skill = unicode(skill)\n",
    "    skill = html_parser.unescape(skill)\n",
    "    \n",
    "    skill = skill.replace(\"_\", \" \").split()\n",
    "    skill = \" \".join([sk for sk in skill if sk])\n",
    "    \n",
    "    skill = re.sub(r\"\\(.*\\)\", \"\", skill)\n",
    "    skill = skill.replace(\"-\", \"\") \\\n",
    "        .replace(\".\", \"\") \\\n",
    "        .replace(\",\", \"\") \\\n",
    "        .replace(\"-\", \"\") \\\n",
    "        .replace(\":\", \"\") \\\n",
    "        .replace(\"(\", \"\") \\\n",
    "        .replace(\")\", \"\") \\\n",
    "        .replace(u\"åá\", \"\") \\\n",
    "        .replace(u\"&\", \"and\") \\\n",
    "        .replace(\" js\", \"js\") \\\n",
    "        .replace(\"-js\", \"js\") \\\n",
    "        .replace(\"_js\", \"js\") \\\n",
    "        .replace(\"java script\", \"js\") \n",
    "    \n",
    "    skill = skill.lower()\n",
    "    \n",
    "    # Special cases replace\n",
    "    special_case = {}\n",
    "    special_case[\"javascript\"] = [ \"js\", \"java script\", \"javascripts\", \"java scrip\" ]\n",
    "    special_case[\"wireframe\"] = [ \"wireframes\", \"wire frame\", \"wire frames\", \"wire-frame\", \"wirefram\", \"wire fram\", \"wireframing\" ]\n",
    "    special_case[\"OOP\"] = [  \"object oriented\", \"object oriented programming\", ]\n",
    "    special_case[\"OOD\"] = [ \"object oriented design\", ]\n",
    "    special_case[\"OLAP\"] = [ \"online analytical processing\",  ]\n",
    "    special_case[\"Ecommerce\"] = [ \"e commerce\",  ]\n",
    "    special_case[\"consultant\"] = [ \"consulting\",  ]\n",
    "    special_case[\"ux\"] = [ \"user experience\", \"web user experience design\", \"user experience design\", \"ux designer\", \"user experience/ux\" ]\n",
    "    special_case[\"html5\"] = [ \"html 5\",  ]\n",
    "    special_case[\"j2ee\"] = [ \"jee\",  ]\n",
    "    special_case[\"osx\"] = [ \"mac os x\", \"os x\" ]\n",
    "    special_case[\"senior\"] = [ \"sr\" ]\n",
    "    special_case[\"qa\"] = [ \"quality\",  ]\n",
    "    special_case[\"bigdata\"] = [ \"big data\",  ]\n",
    "    special_case[\"webservice\"] = [ \"webservices\", \"website\", \"webapps\" ]\n",
    "    special_case[\"xml\"] = [ \"xml file\", \"xml schemas\", \"xml/json\", \"xml web service\" ]\n",
    "    special_case[\"bigdata\"] = [ \"big data\",  ]\n",
    "    special_case[\"nlp\"] = [ \"natural language process\", \"natural language\", \"nltk\" ]\n",
    "    for root_skill in special_case:\n",
    "        if skill in special_case[root_skill]:\n",
    "            skill = root_skill\n",
    "    \n",
    "    # Special case regex\n",
    "    special_case_regex = {\n",
    "        r'^angular.*$': 'angularjs',\n",
    "        r'^node.*$': 'nodejs',\n",
    "        r'^(.*)[_\\s]js$': '\\\\1js',\n",
    "        r'^(.*) js$': '\\\\1js',\n",
    "    }\n",
    "    for regex_rule in special_case_regex:\n",
    "        after_skill = re.sub(regex_rule, special_case_regex[regex_rule], skill)\n",
    "        if after_skill != skill:\n",
    "            skill = after_skill\n",
    "            break\n",
    "    \n",
    "    # Stem\n",
    "    if len(skill) > 2:\n",
    "        skill_after = skill.split(\" \")\n",
    "        skill_after = [wordnet_lemmatizer.lemmatize(sk, pos=\"v\") for sk in skill_after]\n",
    "        skill_after = \" \".join(skill_after)\n",
    "        skill = skill_after\n",
    "    \n",
    "    # skill stopwords \n",
    "    if remove_stopwords:\n",
    "        skill_stopwords = [ \"app\", \"touch\", \"the\", \"application\" ]\n",
    "        skill_after = skill.split(\" \")\n",
    "        skill = \" \".join([ sk for sk in skill_after if sk not in skill_stopwords ])\n",
    "    \n",
    "    skill = skill.lower().strip().replace(\" \", \"_\")\n",
    "    skill = re.sub(' +',' ', skill)\n",
    "    \n",
    "    # NOTE: replace js tail\n",
    "    skill = re.sub('js$','', skill)\n",
    "    \n",
    "    return skill\n",
    "\n",
    "skill_transform(\"js\")\n",
    "skill_transform(\"angularjs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_case(model, model_name):\n",
    "    cases = [ \"machine learning\", \"js\", \"javascript\", \"python\", \"html\", \"html5\", \"css\", \"angular\", \"nodejs\" ]\n",
    "    TEST_CASE[model_name] = []\n",
    "    for case in cases:\n",
    "        try:\n",
    "            TEST_CASE[model_name].append(model.similar_by_word(skill_transform(case)))\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before: 8589\n",
      "After: 6341\n",
      "Resize: 73.8269880079%\n"
     ]
    }
   ],
   "source": [
    "data_must_have = pd.read_csv('mustHaveSkills-2.csv', header = 0, encoding='ISO-8859-1')\n",
    "data_must_have.drop_duplicates(inplace=True, keep='last')\n",
    "data_must_have = data_must_have[data_must_have[\"job_title\"] != 0]\n",
    "del data_must_have[\"job_brief_id\"]\n",
    "# data_must_have['Count'] = data_must_have.groupby('job_title')['keyword_name'].transform(pd.Series.value_counts)\n",
    "data_must_have.drop_duplicates(inplace=True)\n",
    "\n",
    "unique_skill_before = len(data_must_have['keyword_name'].unique())\n",
    "print \"Before: %s\" % unique_skill_before\n",
    "\n",
    "data_must_have['skill'] = data_must_have['keyword_name'].apply(skill_transform)\n",
    "\n",
    "unique_skill_after = len(data_must_have['skill'].unique())\n",
    "print \"After: %s\" % unique_skill_after \n",
    "\n",
    "print \"Resize: %s%%\" % (unique_skill_after * 100.0 / unique_skill_before)\n",
    "\n",
    "# data_must_have['keyword_name'] = data_must_have['keyword_name'].str.replace(' ' ,'_')\n",
    "# data_must_have"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:02:44,320 : INFO : collecting all words and their counts\n",
      "2017-04-17 13:02:44,322 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-04-17 13:02:44,366 : INFO : collected 6341 word types from a corpus of 80810 raw words and 5649 sentences\n",
      "2017-04-17 13:02:44,367 : INFO : Loading a fresh vocabulary\n",
      "2017-04-17 13:02:44,396 : INFO : min_count=1 retains 6341 unique words (100% of original 6341, drops 0)\n",
      "2017-04-17 13:02:44,397 : INFO : min_count=1 leaves 80810 word corpus (100% of original 80810, drops 0)\n",
      "2017-04-17 13:02:44,419 : INFO : deleting the raw counts dictionary of 6341 items\n",
      "2017-04-17 13:02:44,421 : INFO : sample=0.001 downsamples 34 most-common words\n",
      "2017-04-17 13:02:44,422 : INFO : downsampling leaves estimated 77635 word corpus (96.1% of prior 80810)\n",
      "2017-04-17 13:02:44,423 : INFO : estimated required memory for 6341 words and 300 dimensions: 18388900 bytes\n",
      "2017-04-17 13:02:44,444 : INFO : resetting layer weights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:02:44,579 : INFO : training model with 32 workers on 6341 vocabulary and 300 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-04-17 13:02:44,845 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-04-17 13:02:44,894 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-04-17 13:02:44,928 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-04-17 13:02:44,979 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-04-17 13:02:45,053 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-04-17 13:02:45,423 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-04-17 13:02:45,448 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-04-17 13:02:45,462 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-04-17 13:02:45,545 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-04-17 13:02:45,746 : INFO : PROGRESS: at 34.41% examples, 91385 words/s, in_qsize 22, out_qsize 1\n",
      "2017-04-17 13:02:45,748 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-04-17 13:02:45,777 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-04-17 13:02:45,780 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-04-17 13:02:45,809 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-04-17 13:02:45,819 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-04-17 13:02:45,852 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-04-17 13:02:45,865 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-04-17 13:02:45,880 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-04-17 13:02:45,903 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-04-17 13:02:45,939 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-04-17 13:02:45,945 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-04-17 13:02:45,948 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-04-17 13:02:45,957 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-04-17 13:02:45,975 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-04-17 13:02:45,984 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-04-17 13:02:45,991 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-04-17 13:02:45,998 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-17 13:02:45,999 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-17 13:02:46,025 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-17 13:02:46,031 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-17 13:02:46,033 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-17 13:02:46,034 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-17 13:02:46,037 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-17 13:02:46,038 : INFO : training on 323240 raw words (310571 effective words) took 1.4s, 215075 effective words/s\n",
      "2017-04-17 13:02:46,038 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-17 13:02:46,043 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    }
   ],
   "source": [
    "# pd.set_option('display.max_rows', 1500)\n",
    "# data_must_have.drop_duplicates(subset=[\"keyword_name\", \"skill\"])[[\"keyword_name\", \"skill\"]].sort_values(\"skill\")[3000:]\n",
    "musthave_data = data_must_have.groupby('job_title')['skill'].apply(list).tolist()\n",
    "\n",
    "print (\"Training model...\")\n",
    "musthave_model = word2vec.Word2Vec(musthave_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=300, \n",
    "          min_count=1, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )\n",
    "\n",
    "musthave_model.init_sims(replace=True)\n",
    "test_case(musthave_model, \"musthave_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:02:55,676 : INFO : collecting all words and their counts\n",
      "2017-04-17 13:02:55,678 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-04-17 13:02:55,759 : INFO : PROGRESS: at sentence #10000, processed 217169 words, keeping 11788 word types\n",
      "2017-04-17 13:02:55,774 : INFO : collected 12749 word types from a corpus of 255598 raw words and 12795 sentences\n",
      "2017-04-17 13:02:55,775 : INFO : Loading a fresh vocabulary\n",
      "2017-04-17 13:02:55,812 : INFO : min_count=1 retains 12749 unique words (100% of original 12749, drops 0)\n",
      "2017-04-17 13:02:55,814 : INFO : min_count=1 leaves 255598 word corpus (100% of original 255598, drops 0)\n",
      "2017-04-17 13:02:55,843 : INFO : deleting the raw counts dictionary of 12749 items\n",
      "2017-04-17 13:02:55,844 : INFO : sample=0.001 downsamples 1 most-common words\n",
      "2017-04-17 13:02:55,845 : INFO : downsampling leaves estimated 255585 word corpus (100.0% of prior 255598)\n",
      "2017-04-17 13:02:55,846 : INFO : estimated required memory for 12749 words and 300 dimensions: 36972100 bytes\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12795\n",
      "Training model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:02:55,881 : INFO : resetting layer weights\n",
      "2017-04-17 13:02:56,071 : INFO : training model with 32 workers on 12749 vocabulary and 300 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-04-17 13:02:57,157 : INFO : PROGRESS: at 11.99% examples, 139878 words/s, in_qsize 64, out_qsize 0\n",
      "2017-04-17 13:02:58,172 : INFO : PROGRESS: at 46.24% examples, 225021 words/s, in_qsize 56, out_qsize 0\n",
      "2017-04-17 13:02:59,197 : INFO : PROGRESS: at 68.49% examples, 232152 words/s, in_qsize 28, out_qsize 9\n",
      "2017-04-17 13:02:59,238 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-04-17 13:02:59,263 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-04-17 13:02:59,266 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-04-17 13:02:59,281 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-04-17 13:02:59,283 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-04-17 13:02:59,285 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-04-17 13:02:59,288 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-04-17 13:02:59,289 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-04-17 13:02:59,291 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-04-17 13:02:59,293 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-04-17 13:02:59,295 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-04-17 13:02:59,296 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-04-17 13:02:59,298 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-04-17 13:02:59,316 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-04-17 13:02:59,333 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-04-17 13:02:59,335 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-04-17 13:02:59,398 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-04-17 13:02:59,402 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-04-17 13:02:59,415 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-04-17 13:02:59,449 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-04-17 13:02:59,461 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-04-17 13:02:59,501 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-04-17 13:02:59,512 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-04-17 13:02:59,515 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-04-17 13:02:59,531 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-04-17 13:02:59,578 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-17 13:02:59,580 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-17 13:02:59,588 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-17 13:02:59,600 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-17 13:02:59,613 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-17 13:02:59,617 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-17 13:02:59,623 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-17 13:02:59,624 : INFO : training on 1022392 raw words (1022337 effective words) took 3.5s, 288968 effective words/s\n",
      "2017-04-17 13:02:59,625 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-04-17 13:02:59,636 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    }
   ],
   "source": [
    "with open(\"prep_data_tokens_underscore_1\", \"rb\") as g:\n",
    "    data_dice = cPickle.load(g)\n",
    "dice_data = [ [ skill_transform(d.replace(\"_\", \" \")) for d in dice if d is not None ] for dice in data_dice if dice is not None ]\n",
    "\n",
    "\n",
    "musthave_dice_data = dice_data + musthave_data\n",
    "musthave_dice_data = [ list(set([ d for d in data if d is not None and len(d) > 0 ])) for data in musthave_dice_data  ]\n",
    "\n",
    "print len(musthave_dice_data)\n",
    "musthave_dice_data\n",
    "\n",
    "print (\"Training model...\")\n",
    "musthave_dice_model = word2vec.Word2Vec(musthave_dice_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=300, \n",
    "          min_count=1, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )\n",
    "\n",
    "musthave_dice_model.init_sims(replace=True)\n",
    "test_case(musthave_dice_model, \"musthave_dice_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# model.similar_by_word(skill_transform('machine learning'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# model.similar_by_word('python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120107\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "119072"
      ]
     },
     "execution_count": 363,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_data = musthave_dice_data\n",
    "data_naruki = pd.read_csv('naukri_skill_full', header = 0, encoding='ISO-8859-1')\n",
    "data_naruki.drop_duplicates(subset=['id', 'skill'], keep='last')\n",
    "\n",
    "data_naruki['skill'] = data_naruki['skill'].apply(skill_transform)\n",
    "data_naruki_final = data_naruki.groupby('id')['skill'].apply(list)\n",
    "print len(data_naruki_final)\n",
    "\n",
    "for skills in data_naruki_final:\n",
    "    if len(skills) > 2 and skills not in data_train_w2v:\n",
    "        musthave_dice_naruki_data.append(skills)\n",
    "        \n",
    "len(musthave_dice_naruki_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:05:13,394 : INFO : collecting all words and their counts\n",
      "2017-04-17 13:05:13,395 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-04-17 13:05:13,514 : INFO : PROGRESS: at sentence #10000, processed 217169 words, keeping 11788 word types\n",
      "2017-04-17 13:05:13,572 : INFO : PROGRESS: at sentence #20000, processed 318186 words, keeping 21928 word types\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:05:13,629 : INFO : PROGRESS: at sentence #30000, processed 405501 words, keeping 30692 word types\n",
      "2017-04-17 13:05:13,693 : INFO : PROGRESS: at sentence #40000, processed 493703 words, keeping 38555 word types\n",
      "2017-04-17 13:05:13,750 : INFO : PROGRESS: at sentence #50000, processed 582980 words, keeping 45965 word types\n",
      "2017-04-17 13:05:13,800 : INFO : PROGRESS: at sentence #60000, processed 674010 words, keeping 52084 word types\n",
      "2017-04-17 13:05:13,851 : INFO : PROGRESS: at sentence #70000, processed 761402 words, keeping 58506 word types\n",
      "2017-04-17 13:05:13,910 : INFO : PROGRESS: at sentence #80000, processed 847322 words, keeping 64909 word types\n",
      "2017-04-17 13:05:13,961 : INFO : PROGRESS: at sentence #90000, processed 932263 words, keeping 70946 word types\n",
      "2017-04-17 13:05:14,009 : INFO : PROGRESS: at sentence #100000, processed 1022074 words, keeping 75246 word types\n",
      "2017-04-17 13:05:14,052 : INFO : PROGRESS: at sentence #110000, processed 1111899 words, keeping 80104 word types\n",
      "2017-04-17 13:05:14,094 : INFO : collected 84813 word types from a corpus of 1191204 raw words and 119072 sentences\n",
      "2017-04-17 13:05:14,096 : INFO : Loading a fresh vocabulary\n",
      "2017-04-17 13:05:14,245 : INFO : min_count=3 retains 25494 unique words (30% of original 84813, drops 59319)\n",
      "2017-04-17 13:05:14,246 : INFO : min_count=3 leaves 1121971 word corpus (94% of original 1191204, drops 69233)\n",
      "2017-04-17 13:05:14,347 : INFO : deleting the raw counts dictionary of 84813 items\n",
      "2017-04-17 13:05:14,352 : INFO : sample=0.001 downsamples 17 most-common words\n",
      "2017-04-17 13:05:14,352 : INFO : downsampling leaves estimated 1101505 word corpus (98.2% of prior 1121971)\n",
      "2017-04-17 13:05:14,353 : INFO : estimated required memory for 25494 words and 300 dimensions: 73932600 bytes\n",
      "2017-04-17 13:05:14,475 : INFO : resetting layer weights\n",
      "2017-04-17 13:05:14,984 : INFO : training model with 32 workers on 25494 vocabulary and 300 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-04-17 13:05:16,041 : INFO : PROGRESS: at 1.25% examples, 132998 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:05:17,075 : INFO : PROGRESS: at 8.81% examples, 235526 words/s, in_qsize 56, out_qsize 0\n",
      "2017-04-17 13:05:18,079 : INFO : PROGRESS: at 18.33% examples, 274770 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:05:19,417 : INFO : PROGRESS: at 25.31% examples, 257815 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:05:20,526 : INFO : PROGRESS: at 34.90% examples, 295375 words/s, in_qsize 63, out_qsize 1\n",
      "2017-04-17 13:05:21,564 : INFO : PROGRESS: at 48.25% examples, 326769 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:05:22,596 : INFO : PROGRESS: at 52.85% examples, 321914 words/s, in_qsize 48, out_qsize 8\n",
      "2017-04-17 13:05:23,596 : INFO : PROGRESS: at 65.28% examples, 341694 words/s, in_qsize 62, out_qsize 0\n",
      "2017-04-17 13:05:24,600 : INFO : PROGRESS: at 75.28% examples, 347691 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:05:25,647 : INFO : PROGRESS: at 82.45% examples, 351254 words/s, in_qsize 64, out_qsize 3\n",
      "2017-04-17 13:05:26,424 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-04-17 13:05:26,455 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-04-17 13:05:26,524 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-04-17 13:05:26,547 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-04-17 13:05:26,575 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-04-17 13:05:26,585 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-04-17 13:05:26,587 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-04-17 13:05:26,589 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-04-17 13:05:26,591 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-04-17 13:05:26,593 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-04-17 13:05:26,595 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-04-17 13:05:26,597 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-04-17 13:05:26,645 : INFO : PROGRESS: at 95.56% examples, 363639 words/s, in_qsize 19, out_qsize 1\n",
      "2017-04-17 13:05:26,648 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-04-17 13:05:26,666 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-04-17 13:05:26,671 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-04-17 13:05:26,727 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-04-17 13:05:26,759 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-04-17 13:05:26,762 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-04-17 13:05:26,771 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-04-17 13:05:26,802 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-04-17 13:05:26,826 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-04-17 13:05:26,828 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-04-17 13:05:26,833 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-04-17 13:05:26,842 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-04-17 13:05:26,848 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-04-17 13:05:26,852 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-17 13:05:26,861 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-17 13:05:26,866 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-17 13:05:26,874 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-17 13:05:26,884 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-17 13:05:26,885 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-17 13:05:26,888 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-17 13:05:26,888 : INFO : training on 4764816 raw words (4406024 effective words) took 11.9s, 370570 effective words/s\n",
      "2017-04-17 13:05:26,943 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    }
   ],
   "source": [
    "print (\"Training model...\")\n",
    "musthave_dice_naruki_model = word2vec.Word2Vec(musthave_dice_naruki_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=300, \n",
    "          min_count=3, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )\n",
    "\n",
    "musthave_dice_naruki_model.init_sims(replace=True)\n",
    "test_case(musthave_dice_naruki_model, \"musthave_dice_naruki_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'musthave_dice_model': [[(u'doctoral', 0.9114964604377747),\n",
       "   (u'postdoc', 0.911190390586853),\n",
       "   (u'hdfs', 0.9058669805526733),\n",
       "   ('bigdata', 0.9043334722518921),\n",
       "   (u'doctor', 0.8989962339401245),\n",
       "   (u'dsp', 0.893810510635376),\n",
       "   (u'hadoop', 0.8844509720802307),\n",
       "   (u'data_science', 0.8758455514907837),\n",
       "   (u'statistics', 0.8751969933509827),\n",
       "   (u'recommendation', 0.8750705122947693)],\n",
       "  [(u'css', 0.9601218700408936),\n",
       "   (u'css3', 0.9592987298965454),\n",
       "   (u'react', 0.9526255130767822),\n",
       "   (u'prototype', 0.9475393295288086),\n",
       "   (u'html5/css3/javascript', 0.9416552782058716),\n",
       "   (u'css5', 0.9397596120834351),\n",
       "   (u'framework', 0.9380757808685303),\n",
       "   (u'cascade_style_sheet', 0.9371317625045776),\n",
       "   (u'user_journey', 0.9369064569473267),\n",
       "   (u'or_ruby_on_rail', 0.9366532564163208)],\n",
       "  [(u'css', 0.9601218700408936),\n",
       "   (u'css3', 0.9592987298965454),\n",
       "   (u'react', 0.9526255130767822),\n",
       "   (u'prototype', 0.9475393295288086),\n",
       "   (u'html5/css3/javascript', 0.9416552782058716),\n",
       "   (u'css5', 0.9397596120834351),\n",
       "   (u'framework', 0.9380757808685303),\n",
       "   (u'cascade_style_sheet', 0.9371317625045776),\n",
       "   (u'user_journey', 0.9369064569473267),\n",
       "   (u'or_ruby_on_rail', 0.9366532564163208)],\n",
       "  [(u'perl', 0.9333171844482422),\n",
       "   (u'aws', 0.9323707222938538),\n",
       "   (u'tool', 0.9233933687210083),\n",
       "   (u'administrator', 0.9188926219940186),\n",
       "   (u'map_reduce', 0.9173422455787659),\n",
       "   (u'ee', 0.9118616580963135),\n",
       "   (u'kinesis', 0.9094211459159851),\n",
       "   (u'elastic_beanstalk', 0.9077350497245789),\n",
       "   (u'ruby', 0.9063856601715088),\n",
       "   (u'scalable', 0.9054598808288574)],\n",
       "  [(u'css', 0.9712449312210083),\n",
       "   (u'ued', 0.9592583179473877),\n",
       "   (u'uxpin', 0.9587060213088989),\n",
       "   (u'prototype', 0.9578349590301514),\n",
       "   (u'ui_designer', 0.9577643871307373),\n",
       "   (u'framer', 0.9574655294418335),\n",
       "   (u'invision', 0.9518224000930786),\n",
       "   (u'angular', 0.9517871737480164),\n",
       "   (u'xd', 0.9513174295425415),\n",
       "   (u'cq', 0.9506344795227051)],\n",
       "  [(u'interaction', 0.9787406921386719),\n",
       "   (u'ui_design', 0.9721903204917908),\n",
       "   (u'frontends', 0.9719209671020508),\n",
       "   (u'user_experience_designer', 0.971385657787323),\n",
       "   (u'quartz_composer', 0.970252275466919),\n",
       "   (u'html5/css3/javascript', 0.9699590802192688),\n",
       "   (u'web_ux_design', 0.9693603515625),\n",
       "   (u'visual', 0.969336986541748),\n",
       "   (u'front_end_developer', 0.9663808941841125),\n",
       "   (u'websites', 0.9661548733711243)],\n",
       "  [(u'prototype', 0.972412645816803),\n",
       "   (u'html', 0.9712449312210083),\n",
       "   (u'css3', 0.9687755107879639),\n",
       "   (u'ued', 0.9646890163421631),\n",
       "   (u'uxpin', 0.9643912315368652),\n",
       "   (u'ui_designer', 0.9634807109832764),\n",
       "   (u'invision', 0.960956335067749),\n",
       "   (u'xd', 0.9608256816864014),\n",
       "   (u'javascript', 0.9601218700408936),\n",
       "   (u'quartz_composer', 0.959280252456665)],\n",
       "  [(u'npm', 0.9788156151771545),\n",
       "   (u'less', 0.9781175851821899),\n",
       "   (u'bower', 0.9763824939727783),\n",
       "   (u'lamp', 0.9720073938369751),\n",
       "   (u'sharepoint', 0.9707627296447754),\n",
       "   (u'cq', 0.9697834849357605),\n",
       "   (u'three', 0.9690839052200317),\n",
       "   (u'responsive', 0.9684596061706543),\n",
       "   (u'bs/ms_in_computer_science', 0.9677680134773254),\n",
       "   (u'template', 0.9665625691413879)],\n",
       "  [(u'bower', 0.9793540239334106),\n",
       "   (u'require', 0.9765837788581848),\n",
       "   (u'gulp', 0.9758298397064209),\n",
       "   (u'npm', 0.9753208756446838),\n",
       "   (u'cq', 0.974446177482605),\n",
       "   (u'load', 0.9743047952651978),\n",
       "   (u'template', 0.9720919728279114),\n",
       "   (u'mustache', 0.9712168574333191),\n",
       "   (u'adobe_aem', 0.969833254814148),\n",
       "   (u'prototype', 0.9693892598152161)]],\n",
       " 'musthave_dice_naruki_model': [[(u'data_scientist', 0.9089775085449219),\n",
       "   (u'artificial_intelligence', 0.9047968983650208),\n",
       "   (u'algorithm_development', 0.8976550102233887),\n",
       "   ('nlp', 0.8902470469474792),\n",
       "   (u'data_science', 0.8867775797843933),\n",
       "   (u'neural_network', 0.8784679770469666),\n",
       "   (u'random_forest', 0.8734996914863586),\n",
       "   (u'mllib', 0.8693686723709106),\n",
       "   (u'anomaly_detection', 0.8691093921661377),\n",
       "   (u'hivesql', 0.8687126040458679)],\n",
       "  [(u'ajax', 0.885636568069458),\n",
       "   (u'css3', 0.8678185939788818),\n",
       "   (u'html5', 0.8671081066131592),\n",
       "   (u'web_development', 0.8534179925918579),\n",
       "   (u'jquery', 0.8437654972076416),\n",
       "   (u'ui_development', 0.8378852605819702),\n",
       "   (u'json', 0.827910304069519),\n",
       "   (u'xhtml', 0.8225245475769043),\n",
       "   (u'bootstrap', 0.8215859532356262),\n",
       "   (u'css', 0.820676326751709)],\n",
       "  [(u'ajax', 0.885636568069458),\n",
       "   (u'css3', 0.8678185939788818),\n",
       "   (u'html5', 0.8671081066131592),\n",
       "   (u'web_development', 0.8534179925918579),\n",
       "   (u'jquery', 0.8437654972076416),\n",
       "   (u'ui_development', 0.8378852605819702),\n",
       "   (u'json', 0.827910304069519),\n",
       "   (u'xhtml', 0.8225245475769043),\n",
       "   (u'bootstrap', 0.8215859532356262),\n",
       "   (u'css', 0.820676326751709)],\n",
       "  [(u'ruby', 0.8223974704742432),\n",
       "   (u'perl', 0.785459041595459),\n",
       "   (u'postgresql', 0.7768748998641968),\n",
       "   (u'bash', 0.7669945955276489),\n",
       "   (u'shell', 0.7502236366271973),\n",
       "   (u'open_source', 0.7485029697418213),\n",
       "   (u'site_reliability', 0.74522465467453),\n",
       "   (u'relational_databases', 0.7447708249092102),\n",
       "   (u'shell_script', 0.742505669593811),\n",
       "   (u'cloudcomputing', 0.7421342730522156)],\n",
       "  [(u'xhtml', 0.8314751386642456),\n",
       "   (u'ajax', 0.82417893409729),\n",
       "   (u'web_development', 0.8232278823852539),\n",
       "   (u'javascript', 0.8183778524398804),\n",
       "   (u'html5', 0.8168011903762817),\n",
       "   (u'css3', 0.8062242269515991),\n",
       "   (u'ui_development', 0.8038212060928345),\n",
       "   (u'web_design', 0.7939602136611938),\n",
       "   (u'css', 0.7904075384140015),\n",
       "   (u'dhtml', 0.7900073528289795)],\n",
       "  [(u'css3', 0.9485752582550049),\n",
       "   (u'ui_development', 0.9320353865623474),\n",
       "   (u'ui_developer', 0.9273726940155029),\n",
       "   (u'bootstrap', 0.9258832335472107),\n",
       "   (u'media_query', 0.90875244140625),\n",
       "   (u'object_orient_javascript', 0.9055501222610474),\n",
       "   (u'css2', 0.9051429629325867),\n",
       "   (u'css/javascript', 0.904003381729126),\n",
       "   (u'object__orient_javascript', 0.9037318229675293),\n",
       "   (u'jqueryui', 0.9033766388893127)],\n",
       "  [(u'html5', 0.8969886302947998),\n",
       "   (u'ui_development', 0.8606520891189575),\n",
       "   (u'bootstrap', 0.8538317680358887),\n",
       "   (u'css3', 0.8484974503517151),\n",
       "   (u'web_development', 0.836186408996582),\n",
       "   (u'ui_developer', 0.8358325958251953),\n",
       "   (u'web_tool', 0.8273044228553772),\n",
       "   (u'xhtml', 0.8251689672470093),\n",
       "   (u'web_program', 0.8210442662239075),\n",
       "   (u'javascript', 0.820676326751709)],\n",
       "  [(u'node', 0.9416387677192688),\n",
       "   (u'bootstrap', 0.9233359098434448),\n",
       "   (u'react', 0.9168567061424255),\n",
       "   (u'backbone', 0.8977123498916626),\n",
       "   (u'knockout', 0.8766510486602783),\n",
       "   (u'html5', 0.8708734512329102),\n",
       "   (u'ui_developer', 0.8624071478843689),\n",
       "   (u'ui_development', 0.8560938835144043),\n",
       "   (u'frontend', 0.8554548025131226),\n",
       "   (u'bower', 0.8548884391784668)],\n",
       "  [(u'react', 0.9584593772888184),\n",
       "   (u'angular', 0.941638708114624),\n",
       "   (u'backbone', 0.9394224286079407),\n",
       "   (u'mongodb', 0.9199068546295166),\n",
       "   (u'mean_stack', 0.9178934693336487),\n",
       "   (u'bower', 0.9154012799263),\n",
       "   (u'mean', 0.9141607284545898),\n",
       "   (u'full_stack_developer', 0.9121633768081665),\n",
       "   (u'bootstrap', 0.9094018936157227),\n",
       "   (u'ui_architect', 0.9093934297561646)]],\n",
       " 'musthave_model': [[(u'data_mine', 0.9690237045288086),\n",
       "   (u'anomaly_detection', 0.9341932535171509),\n",
       "   (u'log_mine', 0.932994544506073),\n",
       "   (u'parallel_program', 0.9219974279403687),\n",
       "   (u'algorithms', 0.9192023277282715),\n",
       "   (u'statistics', 0.9128892421722412),\n",
       "   (u'mapreduce', 0.9073060154914856),\n",
       "   (u'large_scale', 0.9025181531906128),\n",
       "   (u'odesign', 0.8966615200042725),\n",
       "   (u'math', 0.8830868601799011)],\n",
       "  [(u'jquery', 0.972502589225769),\n",
       "   (u'less', 0.9649890661239624),\n",
       "   (u'bootstrap', 0.964809775352478),\n",
       "   (u'sass', 0.9635562896728516),\n",
       "   (u'asp', 0.9605113863945007),\n",
       "   (u'ajax', 0.9589462280273438),\n",
       "   (u'jasmine', 0.9577000737190247),\n",
       "   (u'air_national_guard', 0.9559256434440613),\n",
       "   (u'dojo', 0.9549013376235962),\n",
       "   (u'drupal', 0.9543566703796387)],\n",
       "  [(u'jquery', 0.972502589225769),\n",
       "   (u'less', 0.9649890661239624),\n",
       "   (u'bootstrap', 0.964809775352478),\n",
       "   (u'sass', 0.9635562896728516),\n",
       "   (u'asp', 0.9605113863945007),\n",
       "   (u'ajax', 0.9589462280273438),\n",
       "   (u'jasmine', 0.9577000737190247),\n",
       "   (u'air_national_guard', 0.9559256434440613),\n",
       "   (u'dojo', 0.9549013376235962),\n",
       "   (u'drupal', 0.9543566703796387)],\n",
       "  [(u'engineer_in_test', 0.9416403770446777),\n",
       "   (u'ruby_on_rail', 0.9366534948348999),\n",
       "   (u'ror', 0.9358513355255127),\n",
       "   (u'debug', 0.9305291175842285),\n",
       "   (u'natural_language_process', 0.9279118776321411),\n",
       "   (u'perl', 0.9255269765853882),\n",
       "   (u'test_frameworks', 0.9253503084182739),\n",
       "   (u'jira', 0.9251596331596375),\n",
       "   (u'blackbox_test', 0.9249788522720337),\n",
       "   (u'test_automation', 0.9240061044692993)],\n",
       "  [(u'cq', 0.9762070178985596),\n",
       "   (u'css3', 0.9720344543457031),\n",
       "   (u'template', 0.9720041751861572),\n",
       "   (u'adobe_experience_management_suite', 0.9708493947982788),\n",
       "   (u'javacript', 0.9700725078582764),\n",
       "   (u'adobe_cq', 0.9672921299934387),\n",
       "   (u'css', 0.9669742584228516),\n",
       "   (u'bs/ms_in_computer_science', 0.9660462141036987),\n",
       "   (u'xhtml', 0.9649450778961182),\n",
       "   (u'wordpress', 0.96361243724823)],\n",
       "  [(u'css3', 0.9747869372367859),\n",
       "   (u'less', 0.9622827768325806),\n",
       "   (u'durandal', 0.9593340158462524),\n",
       "   (u'jquery', 0.9585875868797302),\n",
       "   (u'javascript', 0.9538443684577942),\n",
       "   (u'bootstrap', 0.9513095617294312),\n",
       "   (u'knockout', 0.9512669444084167),\n",
       "   (u'jquerry', 0.9466763734817505),\n",
       "   (u'adobe_experience_management_suite', 0.945675253868103),\n",
       "   (u'angular', 0.9453380107879639)],\n",
       "  [(u'html', 0.9669742584228516),\n",
       "   (u'cq', 0.9486520290374756),\n",
       "   (u'css3', 0.9481748342514038),\n",
       "   (u'template', 0.9443554878234863),\n",
       "   (u'd3', 0.9403502345085144),\n",
       "   (u'ccs3', 0.9401510953903198),\n",
       "   (u'adobe_experience_manager', 0.9399069547653198),\n",
       "   (u'xhtml', 0.9397686719894409),\n",
       "   (u'adobe_experience_management_suite', 0.9394006729125977),\n",
       "   (u'adobe_cq', 0.9390513300895691)],\n",
       "  [(u'html/css', 0.990841805934906),\n",
       "   (u'spa', 0.9899202585220337),\n",
       "   (u'react', 0.9898966550827026),\n",
       "   (u'knockout', 0.9884867668151855),\n",
       "   (u'adobe_experience_management_suite', 0.9880437254905701),\n",
       "   (u'foundation', 0.9875792860984802),\n",
       "   (u'npm', 0.9873133897781372),\n",
       "   (u'durandal', 0.9871742129325867),\n",
       "   (u'coffeescript', 0.9859660267829895),\n",
       "   (u'bower', 0.985776424407959)],\n",
       "  [(u'or_ruby_on_rail', 0.9861066341400146),\n",
       "   (u'pixi', 0.985155463218689),\n",
       "   (u'strut', 0.9812556505203247),\n",
       "   (u'lamp', 0.980836033821106),\n",
       "   (u'zend', 0.9802595376968384),\n",
       "   (u'ember', 0.9801542162895203),\n",
       "   (u'cors', 0.9788671731948853),\n",
       "   (u'webgl', 0.9776936173439026),\n",
       "   (u'css2', 0.9775791168212891),\n",
       "   (u'svg', 0.9772987365722656)]]}"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST_CASE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "TEST_CASE_TABLE = [ pd.DataFrame(TEST_CASE[t]) for t in TEST_CASE ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['musthave_dice_model', 'musthave_model', 'musthave_dice_naruki_model']"
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ t for t in TEST_CASE ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(doctoral, 0.911496460438)</td>\n",
       "      <td>(postdoc, 0.911190390587)</td>\n",
       "      <td>(hdfs, 0.905866980553)</td>\n",
       "      <td>(bigdata, 0.904333472252)</td>\n",
       "      <td>(doctor, 0.89899623394)</td>\n",
       "      <td>(dsp, 0.893810510635)</td>\n",
       "      <td>(hadoop, 0.88445097208)</td>\n",
       "      <td>(data_science, 0.875845551491)</td>\n",
       "      <td>(statistics, 0.875196993351)</td>\n",
       "      <td>(recommendation, 0.875070512295)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(css, 0.960121870041)</td>\n",
       "      <td>(css3, 0.959298729897)</td>\n",
       "      <td>(react, 0.952625513077)</td>\n",
       "      <td>(prototype, 0.947539329529)</td>\n",
       "      <td>(html5/css3/javascript, 0.941655278206)</td>\n",
       "      <td>(css5, 0.939759612083)</td>\n",
       "      <td>(framework, 0.938075780869)</td>\n",
       "      <td>(cascade_style_sheet, 0.937131762505)</td>\n",
       "      <td>(user_journey, 0.936906456947)</td>\n",
       "      <td>(or_ruby_on_rail, 0.936653256416)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(css, 0.960121870041)</td>\n",
       "      <td>(css3, 0.959298729897)</td>\n",
       "      <td>(react, 0.952625513077)</td>\n",
       "      <td>(prototype, 0.947539329529)</td>\n",
       "      <td>(html5/css3/javascript, 0.941655278206)</td>\n",
       "      <td>(css5, 0.939759612083)</td>\n",
       "      <td>(framework, 0.938075780869)</td>\n",
       "      <td>(cascade_style_sheet, 0.937131762505)</td>\n",
       "      <td>(user_journey, 0.936906456947)</td>\n",
       "      <td>(or_ruby_on_rail, 0.936653256416)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(perl, 0.933317184448)</td>\n",
       "      <td>(aws, 0.932370722294)</td>\n",
       "      <td>(tool, 0.923393368721)</td>\n",
       "      <td>(administrator, 0.918892621994)</td>\n",
       "      <td>(map_reduce, 0.917342245579)</td>\n",
       "      <td>(ee, 0.911861658096)</td>\n",
       "      <td>(kinesis, 0.909421145916)</td>\n",
       "      <td>(elastic_beanstalk, 0.907735049725)</td>\n",
       "      <td>(ruby, 0.906385660172)</td>\n",
       "      <td>(scalable, 0.905459880829)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(css, 0.971244931221)</td>\n",
       "      <td>(ued, 0.959258317947)</td>\n",
       "      <td>(uxpin, 0.958706021309)</td>\n",
       "      <td>(prototype, 0.95783495903)</td>\n",
       "      <td>(ui_designer, 0.957764387131)</td>\n",
       "      <td>(framer, 0.957465529442)</td>\n",
       "      <td>(invision, 0.951822400093)</td>\n",
       "      <td>(angular, 0.951787173748)</td>\n",
       "      <td>(xd, 0.951317429543)</td>\n",
       "      <td>(cq, 0.950634479523)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(interaction, 0.978740692139)</td>\n",
       "      <td>(ui_design, 0.972190320492)</td>\n",
       "      <td>(frontends, 0.971920967102)</td>\n",
       "      <td>(user_experience_designer, 0.971385657787)</td>\n",
       "      <td>(quartz_composer, 0.970252275467)</td>\n",
       "      <td>(html5/css3/javascript, 0.969959080219)</td>\n",
       "      <td>(web_ux_design, 0.969360351562)</td>\n",
       "      <td>(visual, 0.969336986542)</td>\n",
       "      <td>(front_end_developer, 0.966380894184)</td>\n",
       "      <td>(websites, 0.966154873371)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(prototype, 0.972412645817)</td>\n",
       "      <td>(html, 0.971244931221)</td>\n",
       "      <td>(css3, 0.968775510788)</td>\n",
       "      <td>(ued, 0.964689016342)</td>\n",
       "      <td>(uxpin, 0.964391231537)</td>\n",
       "      <td>(ui_designer, 0.963480710983)</td>\n",
       "      <td>(invision, 0.960956335068)</td>\n",
       "      <td>(xd, 0.960825681686)</td>\n",
       "      <td>(javascript, 0.960121870041)</td>\n",
       "      <td>(quartz_composer, 0.959280252457)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(npm, 0.978815615177)</td>\n",
       "      <td>(less, 0.978117585182)</td>\n",
       "      <td>(bower, 0.976382493973)</td>\n",
       "      <td>(lamp, 0.972007393837)</td>\n",
       "      <td>(sharepoint, 0.970762729645)</td>\n",
       "      <td>(cq, 0.969783484936)</td>\n",
       "      <td>(three, 0.96908390522)</td>\n",
       "      <td>(responsive, 0.968459606171)</td>\n",
       "      <td>(bs/ms_in_computer_science, 0.967768013477)</td>\n",
       "      <td>(template, 0.966562569141)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(bower, 0.979354023933)</td>\n",
       "      <td>(require, 0.976583778858)</td>\n",
       "      <td>(gulp, 0.975829839706)</td>\n",
       "      <td>(npm, 0.975320875645)</td>\n",
       "      <td>(cq, 0.974446177483)</td>\n",
       "      <td>(load, 0.974304795265)</td>\n",
       "      <td>(template, 0.972091972828)</td>\n",
       "      <td>(mustache, 0.971216857433)</td>\n",
       "      <td>(adobe_aem, 0.969833254814)</td>\n",
       "      <td>(prototype, 0.969389259815)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               0                            1  \\\n",
       "0     (doctoral, 0.911496460438)    (postdoc, 0.911190390587)   \n",
       "1          (css, 0.960121870041)       (css3, 0.959298729897)   \n",
       "2          (css, 0.960121870041)       (css3, 0.959298729897)   \n",
       "3         (perl, 0.933317184448)        (aws, 0.932370722294)   \n",
       "4          (css, 0.971244931221)        (ued, 0.959258317947)   \n",
       "5  (interaction, 0.978740692139)  (ui_design, 0.972190320492)   \n",
       "6    (prototype, 0.972412645817)       (html, 0.971244931221)   \n",
       "7          (npm, 0.978815615177)       (less, 0.978117585182)   \n",
       "8        (bower, 0.979354023933)    (require, 0.976583778858)   \n",
       "\n",
       "                             2                                           3  \\\n",
       "0       (hdfs, 0.905866980553)                   (bigdata, 0.904333472252)   \n",
       "1      (react, 0.952625513077)                 (prototype, 0.947539329529)   \n",
       "2      (react, 0.952625513077)                 (prototype, 0.947539329529)   \n",
       "3       (tool, 0.923393368721)             (administrator, 0.918892621994)   \n",
       "4      (uxpin, 0.958706021309)                  (prototype, 0.95783495903)   \n",
       "5  (frontends, 0.971920967102)  (user_experience_designer, 0.971385657787)   \n",
       "6       (css3, 0.968775510788)                       (ued, 0.964689016342)   \n",
       "7      (bower, 0.976382493973)                      (lamp, 0.972007393837)   \n",
       "8       (gulp, 0.975829839706)                       (npm, 0.975320875645)   \n",
       "\n",
       "                                         4  \\\n",
       "0                  (doctor, 0.89899623394)   \n",
       "1  (html5/css3/javascript, 0.941655278206)   \n",
       "2  (html5/css3/javascript, 0.941655278206)   \n",
       "3             (map_reduce, 0.917342245579)   \n",
       "4            (ui_designer, 0.957764387131)   \n",
       "5        (quartz_composer, 0.970252275467)   \n",
       "6                  (uxpin, 0.964391231537)   \n",
       "7             (sharepoint, 0.970762729645)   \n",
       "8                     (cq, 0.974446177483)   \n",
       "\n",
       "                                         5                                6  \\\n",
       "0                    (dsp, 0.893810510635)          (hadoop, 0.88445097208)   \n",
       "1                   (css5, 0.939759612083)      (framework, 0.938075780869)   \n",
       "2                   (css5, 0.939759612083)      (framework, 0.938075780869)   \n",
       "3                     (ee, 0.911861658096)        (kinesis, 0.909421145916)   \n",
       "4                 (framer, 0.957465529442)       (invision, 0.951822400093)   \n",
       "5  (html5/css3/javascript, 0.969959080219)  (web_ux_design, 0.969360351562)   \n",
       "6            (ui_designer, 0.963480710983)       (invision, 0.960956335068)   \n",
       "7                     (cq, 0.969783484936)           (three, 0.96908390522)   \n",
       "8                   (load, 0.974304795265)       (template, 0.972091972828)   \n",
       "\n",
       "                                       7  \\\n",
       "0         (data_science, 0.875845551491)   \n",
       "1  (cascade_style_sheet, 0.937131762505)   \n",
       "2  (cascade_style_sheet, 0.937131762505)   \n",
       "3    (elastic_beanstalk, 0.907735049725)   \n",
       "4              (angular, 0.951787173748)   \n",
       "5               (visual, 0.969336986542)   \n",
       "6                   (xd, 0.960825681686)   \n",
       "7           (responsive, 0.968459606171)   \n",
       "8             (mustache, 0.971216857433)   \n",
       "\n",
       "                                             8  \\\n",
       "0                 (statistics, 0.875196993351)   \n",
       "1               (user_journey, 0.936906456947)   \n",
       "2               (user_journey, 0.936906456947)   \n",
       "3                       (ruby, 0.906385660172)   \n",
       "4                         (xd, 0.951317429543)   \n",
       "5        (front_end_developer, 0.966380894184)   \n",
       "6                 (javascript, 0.960121870041)   \n",
       "7  (bs/ms_in_computer_science, 0.967768013477)   \n",
       "8                  (adobe_aem, 0.969833254814)   \n",
       "\n",
       "                                   9  \n",
       "0   (recommendation, 0.875070512295)  \n",
       "1  (or_ruby_on_rail, 0.936653256416)  \n",
       "2  (or_ruby_on_rail, 0.936653256416)  \n",
       "3         (scalable, 0.905459880829)  \n",
       "4               (cq, 0.950634479523)  \n",
       "5         (websites, 0.966154873371)  \n",
       "6  (quartz_composer, 0.959280252457)  \n",
       "7         (template, 0.966562569141)  \n",
       "8        (prototype, 0.969389259815)  "
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST_CASE_TABLE[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(data_mine, 0.969023704529)</td>\n",
       "      <td>(anomaly_detection, 0.934193253517)</td>\n",
       "      <td>(log_mine, 0.932994544506)</td>\n",
       "      <td>(parallel_program, 0.92199742794)</td>\n",
       "      <td>(algorithms, 0.919202327728)</td>\n",
       "      <td>(statistics, 0.912889242172)</td>\n",
       "      <td>(mapreduce, 0.907306015491)</td>\n",
       "      <td>(large_scale, 0.902518153191)</td>\n",
       "      <td>(odesign, 0.896661520004)</td>\n",
       "      <td>(math, 0.88308686018)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(jquery, 0.972502589226)</td>\n",
       "      <td>(less, 0.964989066124)</td>\n",
       "      <td>(bootstrap, 0.964809775352)</td>\n",
       "      <td>(sass, 0.963556289673)</td>\n",
       "      <td>(asp, 0.960511386395)</td>\n",
       "      <td>(ajax, 0.958946228027)</td>\n",
       "      <td>(jasmine, 0.957700073719)</td>\n",
       "      <td>(air_national_guard, 0.955925643444)</td>\n",
       "      <td>(dojo, 0.954901337624)</td>\n",
       "      <td>(drupal, 0.95435667038)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(jquery, 0.972502589226)</td>\n",
       "      <td>(less, 0.964989066124)</td>\n",
       "      <td>(bootstrap, 0.964809775352)</td>\n",
       "      <td>(sass, 0.963556289673)</td>\n",
       "      <td>(asp, 0.960511386395)</td>\n",
       "      <td>(ajax, 0.958946228027)</td>\n",
       "      <td>(jasmine, 0.957700073719)</td>\n",
       "      <td>(air_national_guard, 0.955925643444)</td>\n",
       "      <td>(dojo, 0.954901337624)</td>\n",
       "      <td>(drupal, 0.95435667038)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(engineer_in_test, 0.941640377045)</td>\n",
       "      <td>(ruby_on_rail, 0.936653494835)</td>\n",
       "      <td>(ror, 0.935851335526)</td>\n",
       "      <td>(debug, 0.930529117584)</td>\n",
       "      <td>(natural_language_process, 0.927911877632)</td>\n",
       "      <td>(perl, 0.925526976585)</td>\n",
       "      <td>(test_frameworks, 0.925350308418)</td>\n",
       "      <td>(jira, 0.92515963316)</td>\n",
       "      <td>(blackbox_test, 0.924978852272)</td>\n",
       "      <td>(test_automation, 0.924006104469)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(cq, 0.976207017899)</td>\n",
       "      <td>(css3, 0.972034454346)</td>\n",
       "      <td>(template, 0.972004175186)</td>\n",
       "      <td>(adobe_experience_management_suite, 0.97084939...</td>\n",
       "      <td>(javacript, 0.970072507858)</td>\n",
       "      <td>(adobe_cq, 0.967292129993)</td>\n",
       "      <td>(css, 0.966974258423)</td>\n",
       "      <td>(bs/ms_in_computer_science, 0.966046214104)</td>\n",
       "      <td>(xhtml, 0.964945077896)</td>\n",
       "      <td>(wordpress, 0.963612437248)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(css3, 0.974786937237)</td>\n",
       "      <td>(less, 0.962282776833)</td>\n",
       "      <td>(durandal, 0.959334015846)</td>\n",
       "      <td>(jquery, 0.95858758688)</td>\n",
       "      <td>(javascript, 0.953844368458)</td>\n",
       "      <td>(bootstrap, 0.951309561729)</td>\n",
       "      <td>(knockout, 0.951266944408)</td>\n",
       "      <td>(jquerry, 0.946676373482)</td>\n",
       "      <td>(adobe_experience_management_suite, 0.94567525...</td>\n",
       "      <td>(angular, 0.945338010788)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(html, 0.966974258423)</td>\n",
       "      <td>(cq, 0.948652029037)</td>\n",
       "      <td>(css3, 0.948174834251)</td>\n",
       "      <td>(template, 0.944355487823)</td>\n",
       "      <td>(d3, 0.940350234509)</td>\n",
       "      <td>(ccs3, 0.94015109539)</td>\n",
       "      <td>(adobe_experience_manager, 0.939906954765)</td>\n",
       "      <td>(xhtml, 0.939768671989)</td>\n",
       "      <td>(adobe_experience_management_suite, 0.93940067...</td>\n",
       "      <td>(adobe_cq, 0.93905133009)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(html/css, 0.990841805935)</td>\n",
       "      <td>(spa, 0.989920258522)</td>\n",
       "      <td>(react, 0.989896655083)</td>\n",
       "      <td>(knockout, 0.988486766815)</td>\n",
       "      <td>(adobe_experience_management_suite, 0.98804372...</td>\n",
       "      <td>(foundation, 0.987579286098)</td>\n",
       "      <td>(npm, 0.987313389778)</td>\n",
       "      <td>(durandal, 0.987174212933)</td>\n",
       "      <td>(coffeescript, 0.985966026783)</td>\n",
       "      <td>(bower, 0.985776424408)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(or_ruby_on_rail, 0.98610663414)</td>\n",
       "      <td>(pixi, 0.985155463219)</td>\n",
       "      <td>(strut, 0.98125565052)</td>\n",
       "      <td>(lamp, 0.980836033821)</td>\n",
       "      <td>(zend, 0.980259537697)</td>\n",
       "      <td>(ember, 0.98015421629)</td>\n",
       "      <td>(cors, 0.978867173195)</td>\n",
       "      <td>(webgl, 0.977693617344)</td>\n",
       "      <td>(css2, 0.977579116821)</td>\n",
       "      <td>(svg, 0.977298736572)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    0                                    1  \\\n",
       "0         (data_mine, 0.969023704529)  (anomaly_detection, 0.934193253517)   \n",
       "1            (jquery, 0.972502589226)               (less, 0.964989066124)   \n",
       "2            (jquery, 0.972502589226)               (less, 0.964989066124)   \n",
       "3  (engineer_in_test, 0.941640377045)       (ruby_on_rail, 0.936653494835)   \n",
       "4                (cq, 0.976207017899)               (css3, 0.972034454346)   \n",
       "5              (css3, 0.974786937237)               (less, 0.962282776833)   \n",
       "6              (html, 0.966974258423)                 (cq, 0.948652029037)   \n",
       "7          (html/css, 0.990841805935)                (spa, 0.989920258522)   \n",
       "8    (or_ruby_on_rail, 0.98610663414)               (pixi, 0.985155463219)   \n",
       "\n",
       "                             2  \\\n",
       "0   (log_mine, 0.932994544506)   \n",
       "1  (bootstrap, 0.964809775352)   \n",
       "2  (bootstrap, 0.964809775352)   \n",
       "3        (ror, 0.935851335526)   \n",
       "4   (template, 0.972004175186)   \n",
       "5   (durandal, 0.959334015846)   \n",
       "6       (css3, 0.948174834251)   \n",
       "7      (react, 0.989896655083)   \n",
       "8       (strut, 0.98125565052)   \n",
       "\n",
       "                                                   3  \\\n",
       "0                  (parallel_program, 0.92199742794)   \n",
       "1                             (sass, 0.963556289673)   \n",
       "2                             (sass, 0.963556289673)   \n",
       "3                            (debug, 0.930529117584)   \n",
       "4  (adobe_experience_management_suite, 0.97084939...   \n",
       "5                            (jquery, 0.95858758688)   \n",
       "6                         (template, 0.944355487823)   \n",
       "7                         (knockout, 0.988486766815)   \n",
       "8                             (lamp, 0.980836033821)   \n",
       "\n",
       "                                                   4  \\\n",
       "0                       (algorithms, 0.919202327728)   \n",
       "1                              (asp, 0.960511386395)   \n",
       "2                              (asp, 0.960511386395)   \n",
       "3         (natural_language_process, 0.927911877632)   \n",
       "4                        (javacript, 0.970072507858)   \n",
       "5                       (javascript, 0.953844368458)   \n",
       "6                               (d3, 0.940350234509)   \n",
       "7  (adobe_experience_management_suite, 0.98804372...   \n",
       "8                             (zend, 0.980259537697)   \n",
       "\n",
       "                              5                                           6  \\\n",
       "0  (statistics, 0.912889242172)                 (mapreduce, 0.907306015491)   \n",
       "1        (ajax, 0.958946228027)                   (jasmine, 0.957700073719)   \n",
       "2        (ajax, 0.958946228027)                   (jasmine, 0.957700073719)   \n",
       "3        (perl, 0.925526976585)           (test_frameworks, 0.925350308418)   \n",
       "4    (adobe_cq, 0.967292129993)                       (css, 0.966974258423)   \n",
       "5   (bootstrap, 0.951309561729)                  (knockout, 0.951266944408)   \n",
       "6         (ccs3, 0.94015109539)  (adobe_experience_manager, 0.939906954765)   \n",
       "7  (foundation, 0.987579286098)                       (npm, 0.987313389778)   \n",
       "8        (ember, 0.98015421629)                      (cors, 0.978867173195)   \n",
       "\n",
       "                                             7  \\\n",
       "0                (large_scale, 0.902518153191)   \n",
       "1         (air_national_guard, 0.955925643444)   \n",
       "2         (air_national_guard, 0.955925643444)   \n",
       "3                        (jira, 0.92515963316)   \n",
       "4  (bs/ms_in_computer_science, 0.966046214104)   \n",
       "5                    (jquerry, 0.946676373482)   \n",
       "6                      (xhtml, 0.939768671989)   \n",
       "7                   (durandal, 0.987174212933)   \n",
       "8                      (webgl, 0.977693617344)   \n",
       "\n",
       "                                                   8  \\\n",
       "0                          (odesign, 0.896661520004)   \n",
       "1                             (dojo, 0.954901337624)   \n",
       "2                             (dojo, 0.954901337624)   \n",
       "3                    (blackbox_test, 0.924978852272)   \n",
       "4                            (xhtml, 0.964945077896)   \n",
       "5  (adobe_experience_management_suite, 0.94567525...   \n",
       "6  (adobe_experience_management_suite, 0.93940067...   \n",
       "7                     (coffeescript, 0.985966026783)   \n",
       "8                             (css2, 0.977579116821)   \n",
       "\n",
       "                                   9  \n",
       "0              (math, 0.88308686018)  \n",
       "1            (drupal, 0.95435667038)  \n",
       "2            (drupal, 0.95435667038)  \n",
       "3  (test_automation, 0.924006104469)  \n",
       "4        (wordpress, 0.963612437248)  \n",
       "5          (angular, 0.945338010788)  \n",
       "6          (adobe_cq, 0.93905133009)  \n",
       "7            (bower, 0.985776424408)  \n",
       "8              (svg, 0.977298736572)  "
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST_CASE_TABLE[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(data_scientist, 0.908977508545)</td>\n",
       "      <td>(artificial_intelligence, 0.904796898365)</td>\n",
       "      <td>(algorithm_development, 0.897655010223)</td>\n",
       "      <td>(nlp, 0.890247046947)</td>\n",
       "      <td>(data_science, 0.886777579784)</td>\n",
       "      <td>(neural_network, 0.878467977047)</td>\n",
       "      <td>(random_forest, 0.873499691486)</td>\n",
       "      <td>(mllib, 0.869368672371)</td>\n",
       "      <td>(anomaly_detection, 0.869109392166)</td>\n",
       "      <td>(hivesql, 0.868712604046)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(ajax, 0.885636568069)</td>\n",
       "      <td>(css3, 0.867818593979)</td>\n",
       "      <td>(html5, 0.867108106613)</td>\n",
       "      <td>(web_development, 0.853417992592)</td>\n",
       "      <td>(jquery, 0.843765497208)</td>\n",
       "      <td>(ui_development, 0.837885260582)</td>\n",
       "      <td>(json, 0.82791030407)</td>\n",
       "      <td>(xhtml, 0.822524547577)</td>\n",
       "      <td>(bootstrap, 0.821585953236)</td>\n",
       "      <td>(css, 0.820676326752)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(ajax, 0.885636568069)</td>\n",
       "      <td>(css3, 0.867818593979)</td>\n",
       "      <td>(html5, 0.867108106613)</td>\n",
       "      <td>(web_development, 0.853417992592)</td>\n",
       "      <td>(jquery, 0.843765497208)</td>\n",
       "      <td>(ui_development, 0.837885260582)</td>\n",
       "      <td>(json, 0.82791030407)</td>\n",
       "      <td>(xhtml, 0.822524547577)</td>\n",
       "      <td>(bootstrap, 0.821585953236)</td>\n",
       "      <td>(css, 0.820676326752)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(ruby, 0.822397470474)</td>\n",
       "      <td>(perl, 0.785459041595)</td>\n",
       "      <td>(postgresql, 0.776874899864)</td>\n",
       "      <td>(bash, 0.766994595528)</td>\n",
       "      <td>(shell, 0.750223636627)</td>\n",
       "      <td>(open_source, 0.748502969742)</td>\n",
       "      <td>(site_reliability, 0.745224654675)</td>\n",
       "      <td>(relational_databases, 0.744770824909)</td>\n",
       "      <td>(shell_script, 0.742505669594)</td>\n",
       "      <td>(cloudcomputing, 0.742134273052)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(xhtml, 0.831475138664)</td>\n",
       "      <td>(ajax, 0.824178934097)</td>\n",
       "      <td>(web_development, 0.823227882385)</td>\n",
       "      <td>(javascript, 0.81837785244)</td>\n",
       "      <td>(html5, 0.816801190376)</td>\n",
       "      <td>(css3, 0.806224226952)</td>\n",
       "      <td>(ui_development, 0.803821206093)</td>\n",
       "      <td>(web_design, 0.793960213661)</td>\n",
       "      <td>(css, 0.790407538414)</td>\n",
       "      <td>(dhtml, 0.790007352829)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(css3, 0.948575258255)</td>\n",
       "      <td>(ui_development, 0.932035386562)</td>\n",
       "      <td>(ui_developer, 0.927372694016)</td>\n",
       "      <td>(bootstrap, 0.925883233547)</td>\n",
       "      <td>(media_query, 0.908752441406)</td>\n",
       "      <td>(object_orient_javascript, 0.905550122261)</td>\n",
       "      <td>(css2, 0.905142962933)</td>\n",
       "      <td>(css/javascript, 0.904003381729)</td>\n",
       "      <td>(object__orient_javascript, 0.903731822968)</td>\n",
       "      <td>(jqueryui, 0.903376638889)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(html5, 0.896988630295)</td>\n",
       "      <td>(ui_development, 0.860652089119)</td>\n",
       "      <td>(bootstrap, 0.853831768036)</td>\n",
       "      <td>(css3, 0.848497450352)</td>\n",
       "      <td>(web_development, 0.836186408997)</td>\n",
       "      <td>(ui_developer, 0.835832595825)</td>\n",
       "      <td>(web_tool, 0.827304422855)</td>\n",
       "      <td>(xhtml, 0.825168967247)</td>\n",
       "      <td>(web_program, 0.821044266224)</td>\n",
       "      <td>(javascript, 0.820676326752)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(node, 0.941638767719)</td>\n",
       "      <td>(bootstrap, 0.923335909843)</td>\n",
       "      <td>(react, 0.916856706142)</td>\n",
       "      <td>(backbone, 0.897712349892)</td>\n",
       "      <td>(knockout, 0.87665104866)</td>\n",
       "      <td>(html5, 0.870873451233)</td>\n",
       "      <td>(ui_developer, 0.862407147884)</td>\n",
       "      <td>(ui_development, 0.856093883514)</td>\n",
       "      <td>(frontend, 0.855454802513)</td>\n",
       "      <td>(bower, 0.854888439178)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(react, 0.958459377289)</td>\n",
       "      <td>(angular, 0.941638708115)</td>\n",
       "      <td>(backbone, 0.939422428608)</td>\n",
       "      <td>(mongodb, 0.91990685463)</td>\n",
       "      <td>(mean_stack, 0.917893469334)</td>\n",
       "      <td>(bower, 0.915401279926)</td>\n",
       "      <td>(mean, 0.914160728455)</td>\n",
       "      <td>(full_stack_developer, 0.912163376808)</td>\n",
       "      <td>(bootstrap, 0.909401893616)</td>\n",
       "      <td>(ui_architect, 0.909393429756)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  0  \\\n",
       "0  (data_scientist, 0.908977508545)   \n",
       "1            (ajax, 0.885636568069)   \n",
       "2            (ajax, 0.885636568069)   \n",
       "3            (ruby, 0.822397470474)   \n",
       "4           (xhtml, 0.831475138664)   \n",
       "5            (css3, 0.948575258255)   \n",
       "6           (html5, 0.896988630295)   \n",
       "7            (node, 0.941638767719)   \n",
       "8           (react, 0.958459377289)   \n",
       "\n",
       "                                           1  \\\n",
       "0  (artificial_intelligence, 0.904796898365)   \n",
       "1                     (css3, 0.867818593979)   \n",
       "2                     (css3, 0.867818593979)   \n",
       "3                     (perl, 0.785459041595)   \n",
       "4                     (ajax, 0.824178934097)   \n",
       "5           (ui_development, 0.932035386562)   \n",
       "6           (ui_development, 0.860652089119)   \n",
       "7                (bootstrap, 0.923335909843)   \n",
       "8                  (angular, 0.941638708115)   \n",
       "\n",
       "                                         2                                  3  \\\n",
       "0  (algorithm_development, 0.897655010223)              (nlp, 0.890247046947)   \n",
       "1                  (html5, 0.867108106613)  (web_development, 0.853417992592)   \n",
       "2                  (html5, 0.867108106613)  (web_development, 0.853417992592)   \n",
       "3             (postgresql, 0.776874899864)             (bash, 0.766994595528)   \n",
       "4        (web_development, 0.823227882385)        (javascript, 0.81837785244)   \n",
       "5           (ui_developer, 0.927372694016)        (bootstrap, 0.925883233547)   \n",
       "6              (bootstrap, 0.853831768036)             (css3, 0.848497450352)   \n",
       "7                  (react, 0.916856706142)         (backbone, 0.897712349892)   \n",
       "8               (backbone, 0.939422428608)           (mongodb, 0.91990685463)   \n",
       "\n",
       "                                   4  \\\n",
       "0     (data_science, 0.886777579784)   \n",
       "1           (jquery, 0.843765497208)   \n",
       "2           (jquery, 0.843765497208)   \n",
       "3            (shell, 0.750223636627)   \n",
       "4            (html5, 0.816801190376)   \n",
       "5      (media_query, 0.908752441406)   \n",
       "6  (web_development, 0.836186408997)   \n",
       "7          (knockout, 0.87665104866)   \n",
       "8       (mean_stack, 0.917893469334)   \n",
       "\n",
       "                                            5  \\\n",
       "0            (neural_network, 0.878467977047)   \n",
       "1            (ui_development, 0.837885260582)   \n",
       "2            (ui_development, 0.837885260582)   \n",
       "3               (open_source, 0.748502969742)   \n",
       "4                      (css3, 0.806224226952)   \n",
       "5  (object_orient_javascript, 0.905550122261)   \n",
       "6              (ui_developer, 0.835832595825)   \n",
       "7                     (html5, 0.870873451233)   \n",
       "8                     (bower, 0.915401279926)   \n",
       "\n",
       "                                    6                                       7  \\\n",
       "0     (random_forest, 0.873499691486)                 (mllib, 0.869368672371)   \n",
       "1               (json, 0.82791030407)                 (xhtml, 0.822524547577)   \n",
       "2               (json, 0.82791030407)                 (xhtml, 0.822524547577)   \n",
       "3  (site_reliability, 0.745224654675)  (relational_databases, 0.744770824909)   \n",
       "4    (ui_development, 0.803821206093)            (web_design, 0.793960213661)   \n",
       "5              (css2, 0.905142962933)        (css/javascript, 0.904003381729)   \n",
       "6          (web_tool, 0.827304422855)                 (xhtml, 0.825168967247)   \n",
       "7      (ui_developer, 0.862407147884)        (ui_development, 0.856093883514)   \n",
       "8              (mean, 0.914160728455)  (full_stack_developer, 0.912163376808)   \n",
       "\n",
       "                                             8  \\\n",
       "0          (anomaly_detection, 0.869109392166)   \n",
       "1                  (bootstrap, 0.821585953236)   \n",
       "2                  (bootstrap, 0.821585953236)   \n",
       "3               (shell_script, 0.742505669594)   \n",
       "4                        (css, 0.790407538414)   \n",
       "5  (object__orient_javascript, 0.903731822968)   \n",
       "6                (web_program, 0.821044266224)   \n",
       "7                   (frontend, 0.855454802513)   \n",
       "8                  (bootstrap, 0.909401893616)   \n",
       "\n",
       "                                  9  \n",
       "0         (hivesql, 0.868712604046)  \n",
       "1             (css, 0.820676326752)  \n",
       "2             (css, 0.820676326752)  \n",
       "3  (cloudcomputing, 0.742134273052)  \n",
       "4           (dhtml, 0.790007352829)  \n",
       "5        (jqueryui, 0.903376638889)  \n",
       "6      (javascript, 0.820676326752)  \n",
       "7           (bower, 0.854888439178)  \n",
       "8    (ui_architect, 0.909393429756)  "
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST_CASE_TABLE[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:04:28,660 : INFO : storing 84813x300 projection weights into musthave_dice_naruki.model\n"
     ]
    }
   ],
   "source": [
    "musthave_dice_naruki_model.wv.save_word2vec_format(\"musthave_dice_naruki.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 311200\r\n",
      "-rw-r--r-- 1 root root  17781151 12:57 17 Th04 musthave_dice_naruki_100d.model\r\n",
      "-rw-r--r-- 1 root root 242934950 13:04 17 Th04 musthave_dice_naruki.model\r\n",
      "-rw-r--r-- 1 root root   8885546 15:44 16 Th04 mustHaveSkills-2.csv\r\n",
      "-rw-r--r-- 1 root root  18240737 11:17 17 Th04 naukri_skill_full\r\n",
      "-rw-r--r-- 1 root root   1276232 15:57 16 Th04 prepare_data.ipynb\r\n",
      "-rw-r--r-- 1 root root   3972850 15:54 16 Th04 prep_data_tokens_underscore_1\r\n",
      "-rw-r--r-- 1 root root     34398 13:04 17 Th04 skill2vec_ver2.ipynb\r\n",
      "-rw-r--r-- 1 root root     77333 15:55 16 Th04 word2vec_skills.ipynb\r\n",
      "-rw-r--r-- 1 root root     12329 16:33 16 Th04 word2vec_skills_nltk.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "!ls -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:38:09,846 : INFO : collecting all words and their counts\n",
      "2017-04-17 13:38:09,847 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-04-17 13:38:10,020 : INFO : PROGRESS: at sentence #10000, processed 217169 words, keeping 11788 word types\n",
      "2017-04-17 13:38:10,084 : INFO : PROGRESS: at sentence #20000, processed 318186 words, keeping 21928 word types\n",
      "2017-04-17 13:38:10,148 : INFO : PROGRESS: at sentence #30000, processed 405501 words, keeping 30692 word types\n",
      "2017-04-17 13:38:10,201 : INFO : PROGRESS: at sentence #40000, processed 493703 words, keeping 38555 word types\n",
      "2017-04-17 13:38:10,252 : INFO : PROGRESS: at sentence #50000, processed 582980 words, keeping 45965 word types\n",
      "2017-04-17 13:38:10,307 : INFO : PROGRESS: at sentence #60000, processed 674010 words, keeping 52084 word types\n",
      "2017-04-17 13:38:10,360 : INFO : PROGRESS: at sentence #70000, processed 761402 words, keeping 58506 word types\n",
      "2017-04-17 13:38:10,414 : INFO : PROGRESS: at sentence #80000, processed 847322 words, keeping 64909 word types\n",
      "2017-04-17 13:38:10,455 : INFO : PROGRESS: at sentence #90000, processed 932263 words, keeping 70946 word types\n",
      "2017-04-17 13:38:10,497 : INFO : PROGRESS: at sentence #100000, processed 1022074 words, keeping 75246 word types\n",
      "2017-04-17 13:38:10,539 : INFO : PROGRESS: at sentence #110000, processed 1111899 words, keeping 80104 word types\n",
      "2017-04-17 13:38:10,576 : INFO : collected 84813 word types from a corpus of 1191204 raw words and 119072 sentences\n",
      "2017-04-17 13:38:10,578 : INFO : Loading a fresh vocabulary\n",
      "2017-04-17 13:38:10,664 : INFO : min_count=5 retains 18426 unique words (21% of original 84813, drops 66387)\n",
      "2017-04-17 13:38:10,667 : INFO : min_count=5 leaves 1098180 word corpus (92% of original 1191204, drops 93024)\n",
      "2017-04-17 13:38:10,726 : INFO : deleting the raw counts dictionary of 84813 items\n",
      "2017-04-17 13:38:10,730 : INFO : sample=0.001 downsamples 18 most-common words\n",
      "2017-04-17 13:38:10,732 : INFO : downsampling leaves estimated 1076877 word corpus (98.1% of prior 1098180)\n",
      "2017-04-17 13:38:10,734 : INFO : estimated required memory for 18426 words and 100 dimensions: 23953800 bytes\n",
      "2017-04-17 13:38:10,819 : INFO : resetting layer weights\n",
      "2017-04-17 13:38:11,110 : INFO : training model with 32 workers on 18426 vocabulary and 100 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-04-17 13:38:12,149 : INFO : PROGRESS: at 9.73% examples, 501283 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:38:13,179 : INFO : PROGRESS: at 26.11% examples, 596511 words/s, in_qsize 63, out_qsize 1\n",
      "2017-04-17 13:38:14,174 : INFO : PROGRESS: at 43.76% examples, 631433 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:38:15,222 : INFO : PROGRESS: at 58.43% examples, 639677 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:38:16,254 : INFO : PROGRESS: at 74.98% examples, 639571 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:38:17,123 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-04-17 13:38:17,136 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-04-17 13:38:17,188 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-04-17 13:38:17,257 : INFO : PROGRESS: at 93.43% examples, 663126 words/s, in_qsize 23, out_qsize 11\n",
      "2017-04-17 13:38:17,270 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-04-17 13:38:17,274 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-04-17 13:38:17,277 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-04-17 13:38:17,292 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-04-17 13:38:17,293 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-04-17 13:38:17,294 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-04-17 13:38:17,296 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-04-17 13:38:17,298 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-04-17 13:38:17,299 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-04-17 13:38:17,300 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-04-17 13:38:17,301 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-04-17 13:38:17,302 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-04-17 13:38:17,303 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-04-17 13:38:17,304 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-04-17 13:38:17,305 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-04-17 13:38:17,306 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-04-17 13:38:17,315 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-04-17 13:38:17,336 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-04-17 13:38:17,362 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-04-17 13:38:17,364 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-04-17 13:38:17,374 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-04-17 13:38:17,377 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-04-17 13:38:17,378 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-17 13:38:17,383 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-17 13:38:17,387 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-17 13:38:17,389 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-17 13:38:17,391 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-17 13:38:17,402 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-17 13:38:17,413 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-17 13:38:17,413 : INFO : training on 4764816 raw words (4307343 effective words) took 6.3s, 685913 effective words/s\n",
      "2017-04-17 13:38:17,432 : INFO : collecting all words and their counts\n",
      "2017-04-17 13:38:17,433 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-04-17 13:38:17,551 : INFO : PROGRESS: at sentence #10000, processed 217169 words, keeping 11788 word types\n",
      "2017-04-17 13:38:17,618 : INFO : PROGRESS: at sentence #20000, processed 318186 words, keeping 21928 word types\n",
      "2017-04-17 13:38:17,668 : INFO : PROGRESS: at sentence #30000, processed 405501 words, keeping 30692 word types\n",
      "2017-04-17 13:38:17,718 : INFO : PROGRESS: at sentence #40000, processed 493703 words, keeping 38555 word types\n",
      "2017-04-17 13:38:17,762 : INFO : PROGRESS: at sentence #50000, processed 582980 words, keeping 45965 word types\n",
      "2017-04-17 13:38:17,807 : INFO : PROGRESS: at sentence #60000, processed 674010 words, keeping 52084 word types\n",
      "2017-04-17 13:38:17,857 : INFO : PROGRESS: at sentence #70000, processed 761402 words, keeping 58506 word types\n",
      "2017-04-17 13:38:17,932 : INFO : PROGRESS: at sentence #80000, processed 847322 words, keeping 64909 word types\n",
      "2017-04-17 13:38:18,010 : INFO : PROGRESS: at sentence #90000, processed 932263 words, keeping 70946 word types\n",
      "2017-04-17 13:38:18,075 : INFO : PROGRESS: at sentence #100000, processed 1022074 words, keeping 75246 word types\n",
      "2017-04-17 13:38:18,131 : INFO : PROGRESS: at sentence #110000, processed 1111899 words, keeping 80104 word types\n",
      "2017-04-17 13:38:18,172 : INFO : collected 84813 word types from a corpus of 1191204 raw words and 119072 sentences\n",
      "2017-04-17 13:38:18,174 : INFO : Loading a fresh vocabulary\n",
      "2017-04-17 13:38:18,888 : INFO : min_count=5 retains 18426 unique words (21% of original 84813, drops 66387)\n",
      "2017-04-17 13:38:18,890 : INFO : min_count=5 leaves 1098180 word corpus (92% of original 1191204, drops 93024)\n",
      "2017-04-17 13:38:18,947 : INFO : deleting the raw counts dictionary of 84813 items\n",
      "2017-04-17 13:38:18,951 : INFO : sample=0.001 downsamples 18 most-common words\n",
      "2017-04-17 13:38:18,953 : INFO : downsampling leaves estimated 1076877 word corpus (98.1% of prior 1098180)\n",
      "2017-04-17 13:38:18,954 : INFO : estimated required memory for 18426 words and 200 dimensions: 38694600 bytes\n",
      "2017-04-17 13:38:19,008 : INFO : resetting layer weights\n",
      "2017-04-17 13:38:19,260 : INFO : training model with 32 workers on 18426 vocabulary and 200 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-04-17 13:38:20,283 : INFO : PROGRESS: at 4.71% examples, 319191 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:38:21,310 : INFO : PROGRESS: at 20.48% examples, 450357 words/s, in_qsize 63, out_qsize 0\n",
      "2017-04-17 13:38:22,301 : INFO : PROGRESS: at 31.33% examples, 482344 words/s, in_qsize 64, out_qsize 0\n",
      "2017-04-17 13:38:23,302 : INFO : PROGRESS: at 46.83% examples, 505242 words/s, in_qsize 60, out_qsize 0\n",
      "2017-04-17 13:38:24,328 : INFO : PROGRESS: at 51.73% examples, 457682 words/s, in_qsize 55, out_qsize 12\n",
      "2017-04-17 13:38:25,354 : INFO : PROGRESS: at 69.91% examples, 499899 words/s, in_qsize 57, out_qsize 0\n",
      "2017-04-17 13:38:26,600 : INFO : PROGRESS: at 76.60% examples, 465923 words/s, in_qsize 58, out_qsize 22\n",
      "2017-04-17 13:38:27,586 : INFO : PROGRESS: at 92.46% examples, 484255 words/s, in_qsize 32, out_qsize 0\n",
      "2017-04-17 13:38:27,709 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-04-17 13:38:27,715 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-04-17 13:38:27,720 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-04-17 13:38:27,723 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-04-17 13:38:27,725 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-04-17 13:38:27,727 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-04-17 13:38:27,728 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-04-17 13:38:27,729 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-04-17 13:38:27,730 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-04-17 13:38:27,731 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-04-17 13:38:27,732 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-04-17 13:38:27,733 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-04-17 13:38:27,736 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-04-17 13:38:27,759 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-04-17 13:38:27,799 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-04-17 13:38:27,801 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-04-17 13:38:27,814 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-04-17 13:38:27,817 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-04-17 13:38:27,825 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-04-17 13:38:27,856 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-04-17 13:38:27,861 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-04-17 13:38:27,862 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-04-17 13:38:27,864 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-04-17 13:38:27,872 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-04-17 13:38:27,878 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-04-17 13:38:27,883 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-04-17 13:38:27,893 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-04-17 13:38:27,901 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-04-17 13:38:27,905 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-04-17 13:38:27,920 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-04-17 13:38:27,921 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-04-17 13:38:27,923 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-04-17 13:38:27,923 : INFO : training on 4764816 raw words (4307470 effective words) took 8.7s, 497941 effective words/s\n"
     ]
    }
   ],
   "source": [
    "musthave_dice_naruki_model_100d = word2vec.Word2Vec(musthave_dice_naruki_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=100, \n",
    "          min_count=5, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )\n",
    "\n",
    "musthave_dice_naruki_model_200d = word2vec.Word2Vec(musthave_dice_naruki_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=200, \n",
    "          min_count=5, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-04-17 13:40:14,610 : INFO : storing 18426x100 projection weights into musthave_dice_naruki_100d.model\n",
      "2017-04-17 13:40:16,028 : INFO : storing 18426x200 projection weights into musthave_dice_naruki_200d.model\n"
     ]
    }
   ],
   "source": [
    "musthave_dice_naruki_model_100d.wv.save_word2vec_format(\"musthave_dice_naruki_100d.model\")\n",
    "musthave_dice_naruki_model_200d.wv.save_word2vec_format(\"musthave_dice_naruki_200d.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 83640\n",
      "-rw-r--r-- 1 root root     1069 13:19 17 Th04 LICENSE\n",
      "-rw-r--r-- 1 root root 17771251 13:40 17 Th04 musthave_dice_naruki_100d.model\n",
      "-rw-r--r-- 1 root root 35249875 13:40 17 Th04 musthave_dice_naruki_200d.model\n",
      "-rw-r--r-- 1 root root  8885546 15:44 16 Th04 mustHaveSkills-2.csv\n",
      "-rw-r--r-- 1 root root 18240737 11:17 17 Th04 naukri_skill_full\n",
      "-rw-r--r-- 1 root root  1276232 15:57 16 Th04 prepare_data.ipynb\n",
      "-rw-r--r-- 1 root root  3972850 15:54 16 Th04 prep_data_tokens_underscore_1\n",
      "-rw-r--r-- 1 root root       42 13:19 17 Th04 README.md\n",
      "-rw-r--r-- 1 root root   138294 13:38 17 Th04 skill2vec_ver2.ipynb\n",
      "drwxr-xr-x 2 root root       46 13:19 17 Th04 test\n",
      "drwxr-xr-x 2 root root       63 13:19 17 Th04 word2vec_skills\n",
      "-rw-r--r-- 1 root root    77333 15:55 16 Th04 word2vec_skills.ipynb\n",
      "-rw-r--r-- 1 root root    12329 16:33 16 Th04 word2vec_skills_nltk.ipynb\n",
      "              total        used        free      shared  buff/cache   available\n",
      "Mem:         128673       16449       42482         341       69741      111149\n",
      "Swap:             0           0           0\n"
     ]
    }
   ],
   "source": [
    "!ls -l\n",
    "!free -m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'data_scientist', 0.9083810448646545),\n",
       " ('nlp', 0.8880348205566406),\n",
       " (u'algorithm_development', 0.8754720091819763),\n",
       " (u'recommendation', 0.8676395416259766),\n",
       " (u'random_forest', 0.861264705657959),\n",
       " (u'anomaly_detection', 0.8607847690582275),\n",
       " (u'data_science', 0.8525664806365967),\n",
       " (u'caffe', 0.8518109321594238),\n",
       " (u'neural_network', 0.8495544791221619),\n",
       " (u'recommender', 0.8487647771835327)]"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model_100d.similar_by_word(skill_transform('machine learning'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'data_scientist', 0.8998317122459412),\n",
       " (u'algorithm_development', 0.8750029802322388),\n",
       " ('nlp', 0.8717856407165527),\n",
       " (u'artificial_intelligence', 0.8528630137443542),\n",
       " (u'data_science', 0.8504092693328857),\n",
       " (u'anomaly_detection', 0.8483552932739258),\n",
       " (u'hivesql', 0.8452751636505127),\n",
       " (u'tensorflow', 0.8447992205619812),\n",
       " (u'random_forest', 0.842191219329834),\n",
       " (u'recommendation', 0.8411462306976318)]"
      ]
     },
     "execution_count": 385,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model.similar_by_word(skill_transform('machine learning'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'user_experience_design', 0.8861395716667175),\n",
       " (u'visual_design', 0.8819597959518433),\n",
       " (u'ux_design', 0.8733750581741333),\n",
       " (u'user_interface_design', 0.8626808524131775),\n",
       " (u'interaction_design', 0.8534577488899231),\n",
       " (u'usability', 0.8527886867523193),\n",
       " (u'interaction_designer', 0.8406970500946045),\n",
       " (u'wire_frame', 0.8393874168395996),\n",
       " (u'ux_designer', 0.83745276927948),\n",
       " (u'user_experience/ux', 0.8368674516677856)]"
      ]
     },
     "execution_count": 391,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model.similar_by_word(skill_transform('ux'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'ajax', 0.885636568069458),\n",
       " (u'css3', 0.8678185939788818),\n",
       " (u'html5', 0.8671081066131592),\n",
       " (u'web_development', 0.8534179925918579),\n",
       " (u'jquery', 0.8437654972076416),\n",
       " (u'ui_development', 0.8378852605819702),\n",
       " (u'json', 0.827910304069519),\n",
       " (u'xhtml', 0.8225245475769043),\n",
       " (u'bootstrap', 0.8215859532356262),\n",
       " (u'css', 0.820676326751709)]"
      ]
     },
     "execution_count": 379,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model.similar_by_word(skill_transform('javascript'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'react', 0.9416168332099915),\n",
       " (u'backbone', 0.9306129813194275),\n",
       " (u'angular', 0.9224972128868103),\n",
       " (u'mean_stack', 0.9146184921264648),\n",
       " (u'bower', 0.9023303985595703),\n",
       " (u'mean', 0.8987296223640442),\n",
       " (u'mongodb', 0.8956993222236633),\n",
       " (u'grunt', 0.8945599794387817),\n",
       " (u'npm', 0.8914229869842529),\n",
       " (u'full_stack_developer', 0.8905653953552246),\n",
       " (u'mongo_db', 0.8900968432426453),\n",
       " (u'css/javascript', 0.8900823593139648),\n",
       " (u'bootstrap', 0.8898901343345642),\n",
       " (u'mocha', 0.8849277496337891),\n",
       " (u'javascript_developer', 0.8823011517524719),\n",
       " (u'ui_architect', 0.8822310566902161),\n",
       " (u'express', 0.8795454502105713),\n",
       " (u'gulp', 0.8791297674179077),\n",
       " (u'responsive_web', 0.8787859678268433),\n",
       " (u'electronic_city', 0.8785770535469055),\n",
       " (u'front_end_web_developer', 0.8777059316635132),\n",
       " (u'less', 0.8755059242248535),\n",
       " (u'frontend', 0.8755017518997192),\n",
       " (u'extension', 0.8748639225959778),\n",
       " (u'mean_stack_developer', 0.8732047080993652),\n",
       " (u'loopback', 0.8720988631248474),\n",
       " (u'ruby_rail', 0.8696930408477783),\n",
       " (u'javascript_mvc', 0.8695090413093567),\n",
       " (u'full_stack', 0.8693056106567383),\n",
       " (u'front__end_developer', 0.8678404092788696),\n",
       " (u'full_stack_development', 0.867804765701294),\n",
       " (u'ionic_framework', 0.8677880764007568),\n",
       " (u'oo_javascript', 0.8677499294281006),\n",
       " (u'require', 0.8676064014434814),\n",
       " (u'userinterface', 0.867368757724762),\n",
       " (u'yeoman', 0.8673057556152344),\n",
       " (u'meanstack', 0.866361141204834),\n",
       " (u'front_end_engineer', 0.8660608530044556),\n",
       " (u'advance_javascript', 0.8642422556877136),\n",
       " (u'mustache', 0.8640897274017334),\n",
       " (u'karma', 0.8637064099311829),\n",
       " (u'firebase', 0.8629325032234192),\n",
       " (u'html5_apis', 0.8618912696838379),\n",
       " (u'aurelia', 0.8617426753044128),\n",
       " (u'ui_developement', 0.861459493637085),\n",
       " (u'bs/ms_in_computer_science', 0.8611633777618408),\n",
       " (u'frontend_developer', 0.8608938455581665),\n",
       " (u'fullstack', 0.8608871698379517),\n",
       " (u'single_page', 0.8608673214912415),\n",
       " (u'angular_', 0.8607972860336304),\n",
       " (u'object_orient_javascript', 0.8607783317565918),\n",
       " (u'front_end_program', 0.8607054948806763),\n",
       " (u'ui_developer', 0.8601522445678711),\n",
       " (u'redux', 0.860116720199585),\n",
       " (u'front__end_development', 0.8600871562957764),\n",
       " (u'progressive_web_apps', 0.8600633144378662),\n",
       " (u'ui_developers', 0.8598580360412598),\n",
       " (u'api_design', 0.8591593503952026),\n",
       " (u'front_end_programmer', 0.858847439289093),\n",
       " (u'ionic', 0.858130693435669),\n",
       " (u'type_script', 0.8580596446990967),\n",
       " (u'scss', 0.8578962087631226),\n",
       " (u'durandal', 0.8578891754150391),\n",
       " (u'highcharts', 0.8578832149505615),\n",
       " (u'ractive', 0.8567509651184082),\n",
       " (u'vue', 0.8566321134567261),\n",
       " (u'ui_programmer', 0.8565716743469238),\n",
       " (u'mongo_database', 0.8562326431274414),\n",
       " (u'front_end_developer', 0.8559131622314453),\n",
       " (u'frontend_development', 0.8556913733482361),\n",
       " (u'oo', 0.855413019657135),\n",
       " (u'backend_developer', 0.8552072644233704),\n",
       " (u'web_engineer', 0.855121910572052),\n",
       " (u'rest_api', 0.8549337983131409),\n",
       " (u'html5_and_css3', 0.8547133207321167),\n",
       " (u'spring_data', 0.8545491099357605),\n",
       " (u'css/sass', 0.8544028997421265),\n",
       " (u'socketio', 0.8540019989013672),\n",
       " (u'javascript_frameworks', 0.8539374470710754),\n",
       " (u'service_workers', 0.8537371158599854),\n",
       " (u'ui_frameworks', 0.8532044887542725),\n",
       " (u'es6', 0.8531894087791443),\n",
       " (u'ui_framework', 0.852928638458252),\n",
       " (u'or', 0.8516718149185181),\n",
       " (u'three', 0.8514177799224854),\n",
       " (u'ui_development', 0.8513758778572083),\n",
       " (u'user_interface_developer', 0.8512065410614014),\n",
       " (u'angula', 0.8506923317909241),\n",
       " (u'ccs3', 0.8497893810272217),\n",
       " (u'jasmine', 0.8494973182678223),\n",
       " (u'full_stack_software_developer', 0.8491736054420471),\n",
       " (u'websockets', 0.8491555452346802),\n",
       " (u'handlebar', 0.848856508731842),\n",
       " (u'ember', 0.8478131294250488),\n",
       " (u'mcv', 0.8477417230606079),\n",
       " (u'fullstack_developer', 0.846505343914032),\n",
       " (u'transform', 0.846007764339447),\n",
       " (u'web_sockets', 0.8457252979278564),\n",
       " (u'ror', 0.845497727394104),\n",
       " (u'ruby_on_rail_developer', 0.8451708555221558),\n",
       " (u'user_interface_development', 0.8447953462600708),\n",
       " (u'object_orient_java_script', 0.844504714012146),\n",
       " (u'sass', 0.8441214561462402),\n",
       " (u'restful_api', 0.8440039157867432),\n",
       " (u'domain_drive_design', 0.8431628346443176),\n",
       " (u'ui_engineer', 0.8427528738975525),\n",
       " (u'backend_development', 0.8426660299301147),\n",
       " (u'anguler', 0.8425569534301758),\n",
       " (u'knockout', 0.8420947790145874),\n",
       " (u'odoo', 0.8416228294372559),\n",
       " (u'backend_engineer', 0.8414077758789062),\n",
       " (u'html5/css3/javascript', 0.841398298740387),\n",
       " (u'js_angular', 0.8410046100616455),\n",
       " (u'postcss', 0.8407747745513916),\n",
       " (u'compass', 0.8405935168266296),\n",
       " (u'lamp_stack', 0.840316891670227),\n",
       " (u'jqueryui', 0.8401439189910889),\n",
       " (u'cordova', 0.8399810791015625),\n",
       " (u'server_side_development', 0.8392708897590637),\n",
       " (u'front__end', 0.8391138315200806),\n",
       " (u'java_ui_developer', 0.8390355110168457),\n",
       " (u'pixi', 0.8389677405357361),\n",
       " (u'hybrid_mobile_development', 0.8388615250587463),\n",
       " (u'django', 0.8385516405105591),\n",
       " (u'behaviour_drive_development', 0.8381047248840332),\n",
       " (u'meteor', 0.8377951383590698),\n",
       " (u'rubyonrails', 0.8359246253967285),\n",
       " (u'rich_internet_applications', 0.8357016444206238),\n",
       " (u'hybrid_mobile_developer', 0.835646390914917),\n",
       " (u'javascipt', 0.8352106213569641),\n",
       " (u'ruby_rail_developer', 0.8348395228385925),\n",
       " (u'object_oriented', 0.8347841501235962),\n",
       " (u'css5', 0.8345158100128174),\n",
       " (u'hybrid_mobile', 0.8344641923904419),\n",
       " (u'anjular', 0.8342759609222412),\n",
       " (u'javscript', 0.8342579007148743),\n",
       " (u'responsive_web_design', 0.8341003060340881),\n",
       " (u'oracle_backend', 0.8340749740600586),\n",
       " (u'webdeveloper', 0.8339034914970398),\n",
       " (u'postgre_sql', 0.8336447477340698),\n",
       " (u'and_javascript', 0.8334376811981201),\n",
       " (u'javascript_framework', 0.8334314823150635),\n",
       " (u'hybrid_applications', 0.8332929611206055),\n",
       " (u'crawl', 0.8332432508468628),\n",
       " (u'ux_programmer', 0.8332258462905884),\n",
       " (u'or_ruby_on_rail', 0.833071231842041),\n",
       " (u'responsive', 0.8330236673355103),\n",
       " (u'twitter_bootstrap', 0.8326057195663452),\n",
       " (u'full_stack_web_developer', 0.8321430087089539),\n",
       " (u'ui_specialist', 0.8320354223251343),\n",
       " (u'full_stack_software_engineer', 0.8318779468536377),\n",
       " (u'air_force', 0.8310393691062927),\n",
       " (u'html5', 0.8309826850891113),\n",
       " (u'veteran', 0.8304873108863831),\n",
       " (u'bootstrap_3', 0.8304763436317444),\n",
       " (u'javascript_development', 0.8302295804023743),\n",
       " (u'frontends', 0.8299104571342468),\n",
       " (u'responsive_web_development', 0.8295255899429321),\n",
       " (u'protractor', 0.8294609785079956),\n",
       " (u'rest_apis', 0.8292799592018127),\n",
       " (u'ror_developer', 0.8292418718338013),\n",
       " (u'dynamodb', 0.8292378783226013),\n",
       " (u'ux_developer', 0.8287686705589294),\n",
       " (u'fontend', 0.8284523487091064),\n",
       " (u'test_drive_development', 0.8284451961517334),\n",
       " (u'js_libraries', 0.8281190395355225),\n",
       " (u'hybrid_mobile_apps', 0.8280735015869141),\n",
       " (u'api_development', 0.8279067277908325),\n",
       " (u'reactjs_developer', 0.8276447057723999),\n",
       " (u'yii2', 0.8274866342544556),\n",
       " (u'server_side', 0.8272095322608948),\n",
       " (u'air_national_guard', 0.8271716237068176),\n",
       " (u'mongo', 0.8270388245582581),\n",
       " (u'javascript_program', 0.8266960978507996),\n",
       " (u'yii_framework', 0.8266428709030151),\n",
       " (u'typescript', 0.8260884284973145),\n",
       " (u'vs_2010', 0.8260370492935181),\n",
       " (u'nosql_db', 0.825998842716217),\n",
       " (u'agile_software', 0.8257812261581421),\n",
       " (u'opensource', 0.8256227970123291),\n",
       " (u'aws_sdk', 0.8255846500396729),\n",
       " (u'mock', 0.8255838751792908),\n",
       " (u'marine_corps', 0.825485110282898),\n",
       " (u'css3', 0.8254145383834839),\n",
       " (u'management_experience', 0.8252961039543152),\n",
       " (u'us_military', 0.8252125978469849),\n",
       " (u'ruby_and_rail', 0.8251681327819824),\n",
       " (u'kendoui', 0.8246169090270996),\n",
       " (u'java_j2ee_developer', 0.8245559334754944),\n",
       " (u'curl', 0.824226975440979),\n",
       " (u'mvc_frameworks', 0.8233301043510437),\n",
       " (u'native_ios', 0.8232134580612183),\n",
       " (u'ui/ux', 0.8229209184646606),\n",
       " (u'senior_programmer', 0.8225547075271606),\n",
       " (u'js_framework', 0.8221807479858398),\n",
       " (u'bs_in_computer_science', 0.8220607042312622),\n",
       " (u'nosql', 0.8220254182815552),\n",
       " (u'ui_lead', 0.8218957185745239),\n",
       " (u'cascade_style_sheet', 0.821716845035553),\n",
       " (u'android_platform', 0.821548342704773),\n",
       " (u'react_native_developer', 0.8211182951927185),\n",
       " (u'front_end_development', 0.8211123943328857),\n",
       " (u'html4', 0.8210716843605042),\n",
       " (u'java_spring', 0.8206604719161987),\n",
       " (u'redis', 0.8204728960990906),\n",
       " (u'ms_azure', 0.8202428817749023),\n",
       " (u'akka_or_erlang', 0.8201108574867249),\n",
       " (u'phython', 0.8199530839920044),\n",
       " (u'mvw', 0.8198055028915405),\n",
       " (u'git_hub', 0.8198023438453674),\n",
       " (u'devtools', 0.8196097612380981),\n",
       " (u'markup_languages', 0.8193941116333008),\n",
       " (u'phonegap_developer', 0.8192694187164307),\n",
       " (u'net_angular', 0.8192223906517029),\n",
       " (u'html5_developer', 0.8190504312515259),\n",
       " (u'jquerymobile', 0.8188654184341431),\n",
       " (u'phone_gap', 0.8185100555419922),\n",
       " (u'restful_web_service', 0.8184419870376587),\n",
       " (u'aws_service', 0.8183995485305786),\n",
       " (u'php_architect', 0.818109929561615),\n",
       " (u'bdd', 0.8181009292602539),\n",
       " (u'ruby_on_rail', 0.8180829286575317),\n",
       " (u'full__stack', 0.8180351853370667),\n",
       " (u'play', 0.8179824352264404),\n",
       " (u'ext', 0.8179388642311096),\n",
       " (u'front_end', 0.8178890943527222),\n",
       " (u'restful_webservices', 0.8177791833877563),\n",
       " (u'js_frameworks', 0.8169779777526855),\n",
       " (u'consumer_brand', 0.8169114589691162),\n",
       " (u'spring_boot', 0.8167427778244019),\n",
       " (u'object_relational_mapper', 0.8166913390159607),\n",
       " (u'boostrap', 0.8164705634117126),\n",
       " (u'hybrid_development', 0.8162182569503784),\n",
       " (u'web_ui_developer', 0.8162060976028442),\n",
       " (u'laravel_developer', 0.8157458305358887),\n",
       " (u'data_source', 0.8156468868255615),\n",
       " (u'object__orient_program', 0.8154494762420654),\n",
       " (u'rabbit_mq', 0.8154045343399048),\n",
       " (u'web_tool', 0.8149875998497009),\n",
       " (u'css2', 0.8149144649505615),\n",
       " (u'handlebars', 0.8146485090255737),\n",
       " (u'api_security', 0.8146086931228638),\n",
       " (u'sql_lite', 0.8144867420196533),\n",
       " (u'html_development', 0.8143618702888489),\n",
       " (u'magento_architect', 0.8140093088150024),\n",
       " (u'ux_lead', 0.8139671087265015),\n",
       " (u'no__sql', 0.8139121532440186),\n",
       " (u'build_tool', 0.813889741897583),\n",
       " (u'sr_software_engineer', 0.8136160373687744),\n",
       " (u'graphical_user_interface_design', 0.8132694959640503),\n",
       " (u'gui_developer', 0.8130440711975098),\n",
       " (u'web_ux_design', 0.8128427267074585),\n",
       " (u'rabbitmq', 0.812441885471344),\n",
       " (u'websocket', 0.8123241066932678),\n",
       " (u'nsql', 0.8123050928115845),\n",
       " (u'memcache', 0.8122237324714661),\n",
       " (u'chief_architect', 0.8121933937072754),\n",
       " (u'flux', 0.8116445541381836),\n",
       " (u'solr', 0.8115056753158569),\n",
       " (u'responsive/adaptive_design', 0.8112978339195251),\n",
       " (u'nosql_database', 0.8111300468444824),\n",
       " (u'reactos', 0.8111294507980347),\n",
       " (u'serverside', 0.8111244440078735),\n",
       " (u'extract', 0.8107157945632935),\n",
       " (u'spring_developer', 0.8106698989868164),\n",
       " (u'scala', 0.8106129765510559),\n",
       " (u'model_view_controller', 0.8105576038360596),\n",
       " (u'nosql_databases', 0.8104938268661499),\n",
       " (u'object__orient_javascript', 0.8099992275238037),\n",
       " (u'feed', 0.8099280595779419),\n",
       " (u'framework_development', 0.8098911046981812),\n",
       " (u'adobe_experience_management_suite', 0.8097846508026123),\n",
       " (u'http_web_service', 0.8097734451293945),\n",
       " (u'full_stack_engineer', 0.8097597360610962),\n",
       " (u'javascrip', 0.8095368146896362),\n",
       " (u'geodistributed', 0.809277355670929),\n",
       " (u'laravel5', 0.8092212080955505),\n",
       " (u'tdd', 0.8088719844818115),\n",
       " (u'design_architecture', 0.8083182573318481),\n",
       " (u'senior_php_developer', 0.8083043694496155),\n",
       " (u'code_ignitor', 0.808241605758667),\n",
       " (u'paypal', 0.8079877495765686),\n",
       " (u'postgres', 0.8079557418823242),\n",
       " (u'twist', 0.807365894317627),\n",
       " (u'cucumberjvm', 0.8069980144500732),\n",
       " (u'distribute_databases', 0.8069478273391724),\n",
       " (u'scalable_distribute', 0.806801438331604),\n",
       " (u'technology_project_and_manager', 0.8067919015884399),\n",
       " (u'j_script', 0.8066891431808472),\n",
       " (u'principal_engineer', 0.806373655796051),\n",
       " (u'bigquery', 0.8063719272613525),\n",
       " (u'js_framworks', 0.8062817454338074),\n",
       " (u'scalatest', 0.8060450553894043),\n",
       " (u'canvas', 0.8059965372085571),\n",
       " (u'revolution_analytics', 0.8058544993400574),\n",
       " (u'mysql_db', 0.8058278560638428),\n",
       " (u'css_30', 0.8056614398956299),\n",
       " (u'postgress', 0.8055301308631897),\n",
       " (u'senior_web_developer', 0.8054022789001465),\n",
       " (u'stash', 0.805294394493103)]"
      ]
     },
     "execution_count": 395,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model.similar_by_word(skill_transform('node_js'), 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'react', 0.9322503805160522),\n",
       " (u'backbone', 0.9266782999038696),\n",
       " (u'mean_stack', 0.9251165986061096),\n",
       " (u'angular', 0.9170751571655273),\n",
       " (u'mean', 0.9123091697692871),\n",
       " (u'grunt', 0.9012008905410767),\n",
       " (u'mean_stack_developer', 0.8988714218139648),\n",
       " (u'express', 0.89606773853302),\n",
       " (u'bower', 0.8914059996604919),\n",
       " (u'npm', 0.8903093338012695),\n",
       " (u'loopback', 0.8901658058166504),\n",
       " (u'ui_architect', 0.890097439289093),\n",
       " (u'gulp', 0.88966965675354),\n",
       " (u'mongodb', 0.8894267082214355),\n",
       " (u'full_stack_developer', 0.8890975713729858),\n",
       " (u'extension', 0.8877204656600952),\n",
       " (u'javascript_developer', 0.8874136209487915),\n",
       " (u'bootstrap', 0.8830759525299072),\n",
       " (u'meanstack', 0.8820235729217529),\n",
       " (u'ruby_rail', 0.8796218633651733),\n",
       " (u'mongo_db', 0.8789615631103516),\n",
       " (u'front__end_developer', 0.878875195980072),\n",
       " (u'backend_developer', 0.8750494122505188),\n",
       " (u'html5_and_css3', 0.8740687370300293),\n",
       " (u'mocha', 0.8730786442756653),\n",
       " (u'api_design', 0.8726017475128174),\n",
       " (u'firebase', 0.8722734451293945),\n",
       " (u'require', 0.8716689348220825),\n",
       " (u'full_stack_development', 0.8713763952255249),\n",
       " (u'frontend', 0.8712540864944458),\n",
       " (u'less', 0.8711784482002258),\n",
       " (u'front_end_web_developer', 0.871110200881958),\n",
       " (u'oo_javascript', 0.8706681132316589),\n",
       " (u'mustache', 0.870209813117981),\n",
       " (u'ui_frameworks', 0.8697702884674072),\n",
       " (u'ionic_framework', 0.8691895604133606),\n",
       " (u'scss', 0.8691741228103638),\n",
       " (u'front_end_engineer', 0.8686078786849976),\n",
       " (u'fullstack_developer', 0.868307888507843),\n",
       " (u'advance_javascript', 0.8678364753723145),\n",
       " (u'front__end_development', 0.8675538301467896),\n",
       " (u'web_sockets', 0.867329478263855),\n",
       " (u'karma', 0.8666224479675293),\n",
       " (u'highcharts', 0.8652176260948181),\n",
       " (u'javascript_frameworks', 0.864690899848938),\n",
       " (u'front_end_developer', 0.8641008138656616),\n",
       " (u'front_end_programmer', 0.8635581135749817),\n",
       " (u'frontend_developer', 0.8632322549819946),\n",
       " (u'object_orient_javascript', 0.8632142543792725),\n",
       " (u'ember', 0.862052321434021),\n",
       " (u'full_stack', 0.8614175319671631),\n",
       " (u'durandal', 0.8601962327957153),\n",
       " (u'single_page', 0.859358012676239),\n",
       " (u'backend_engineer', 0.8582956790924072),\n",
       " (u'rest_api', 0.8580957651138306),\n",
       " (u'ui_developer', 0.8566715121269226),\n",
       " (u'behaviour_drive_development', 0.8565572500228882),\n",
       " (u'ionic', 0.8559154272079468),\n",
       " (u'jasmine', 0.8556338548660278),\n",
       " (u'handlebar', 0.8554569482803345),\n",
       " (u'ruby_on_rail_developer', 0.8553227186203003),\n",
       " (u'es6', 0.8551881909370422),\n",
       " (u'pixi', 0.8547605276107788),\n",
       " (u'transform', 0.8544202446937561),\n",
       " (u'knockout', 0.8540791273117065),\n",
       " (u'ui_developers', 0.8540389537811279),\n",
       " (u'three', 0.8534829616546631),\n",
       " (u'html5/css3/javascript', 0.8526034355163574),\n",
       " (u'restful_api', 0.8514331579208374),\n",
       " (u'fullstack', 0.8507369756698608),\n",
       " (u'backend_development', 0.8500568866729736),\n",
       " (u'server_side', 0.8499314785003662),\n",
       " (u'ccs3', 0.8496192693710327),\n",
       " (u'user_interface_development', 0.849193811416626),\n",
       " (u'front__end', 0.8484450578689575),\n",
       " (u'net_angular', 0.8459886908531189),\n",
       " (u'lamp_stack', 0.8447449207305908),\n",
       " (u'frontend_development', 0.844132661819458),\n",
       " (u'cordova', 0.8439915180206299),\n",
       " (u'ux_programmer', 0.8438774347305298),\n",
       " (u'rubyonrails', 0.8431482315063477),\n",
       " (u'meteor', 0.8430627584457397),\n",
       " (u'javascript_development', 0.8425732254981995),\n",
       " (u'ui_development', 0.8417836427688599),\n",
       " (u'ror_developer', 0.8403751850128174),\n",
       " (u'bdd', 0.8399432897567749),\n",
       " (u'ui_programmer', 0.8398774862289429),\n",
       " (u'sass', 0.8394087553024292),\n",
       " (u'ror', 0.8392817974090576),\n",
       " (u'curl', 0.8392391204833984),\n",
       " (u'ruby_rail_developer', 0.8389467000961304),\n",
       " (u'opensource', 0.8389145731925964),\n",
       " (u'oo', 0.8382019400596619),\n",
       " (u'css5', 0.8379584550857544),\n",
       " (u'user_interface_developer', 0.8374938368797302),\n",
       " (u'rabbit_mq', 0.8373412489891052),\n",
       " (u'or_ruby_on_rail', 0.8371344804763794),\n",
       " (u'django', 0.8370762467384338),\n",
       " (u'twitter_bootstrap', 0.8370622992515564),\n",
       " (u'websockets', 0.836702823638916),\n",
       " (u'ux_developer', 0.8366588354110718),\n",
       " (u'hybrid_mobile', 0.8361978530883789),\n",
       " (u'test_drive_development', 0.8358559608459473),\n",
       " (u'api_development', 0.8354291915893555),\n",
       " (u'hybrid_mobile_developer', 0.8352605104446411),\n",
       " (u'ui_lead', 0.835061252117157),\n",
       " (u'responsive', 0.8349193334579468),\n",
       " (u'odoo', 0.8346350789070129),\n",
       " (u'mvc_frameworks', 0.8336235284805298),\n",
       " (u'ui_engineer', 0.8335695266723633),\n",
       " (u'technology_lead', 0.8332920074462891),\n",
       " (u'java_ui_developer', 0.8332258462905884),\n",
       " (u'mongo', 0.8329493999481201),\n",
       " (u'html4', 0.832949161529541),\n",
       " (u'mcv', 0.832800030708313),\n",
       " (u'compass', 0.8327038884162903),\n",
       " (u'ruby_on_rail', 0.832459568977356),\n",
       " (u'restful_webservices', 0.8324102759361267),\n",
       " (u'rest_apis', 0.8322353363037109),\n",
       " (u'protractor', 0.831700325012207),\n",
       " (u'webdeveloper', 0.8316474556922913),\n",
       " (u'postgre_sql', 0.8303982019424438),\n",
       " (u'object__orient_program', 0.8301312923431396),\n",
       " (u'us_military', 0.8294525146484375),\n",
       " (u'coast_guard', 0.8288537859916687),\n",
       " (u'marine_corps', 0.8288424015045166),\n",
       " (u'frontends', 0.8288372159004211),\n",
       " (u'front_end_development', 0.8288000822067261),\n",
       " (u'principal_engineer', 0.8282302618026733),\n",
       " (u'senior_php_developer', 0.8282297849655151),\n",
       " (u'restapi', 0.8282023072242737),\n",
       " (u'javascrip', 0.8280785083770752),\n",
       " (u'dynamodb', 0.8280426859855652),\n",
       " (u'responsive_web_design', 0.8271723985671997),\n",
       " (u'nosql_databases', 0.8267852067947388),\n",
       " (u'code_ignitor', 0.8267363905906677),\n",
       " (u'phython', 0.8263670802116394),\n",
       " (u'akka_or_erlang', 0.8244451880455017),\n",
       " (u'restful_web_service', 0.8240536451339722),\n",
       " (u'typescript', 0.8239384889602661),\n",
       " (u'sql_lite', 0.8237389326095581),\n",
       " (u'extract', 0.823691725730896),\n",
       " (u'build_tool', 0.8234655261039734),\n",
       " (u'yui', 0.8229801058769226),\n",
       " (u'memcache', 0.8228806257247925),\n",
       " (u'feed', 0.8226337432861328),\n",
       " (u'or', 0.822172224521637),\n",
       " (u'handlebars', 0.8221023082733154),\n",
       " (u'no_sql', 0.8219031095504761),\n",
       " (u'nosql_db', 0.8215437531471252),\n",
       " (u'ui/ux', 0.8214239478111267),\n",
       " (u'play', 0.8214013576507568),\n",
       " (u'markup_languages', 0.8213678598403931),\n",
       " (u'gui_developer', 0.8209846019744873),\n",
       " (u'air_national_guard', 0.8208048939704895),\n",
       " (u'rest_web_service', 0.8206841945648193),\n",
       " (u'object_orient_concepts', 0.820545494556427),\n",
       " (u'microservices', 0.820486307144165),\n",
       " (u'sr_software_engineer', 0.8197553157806396),\n",
       " (u'back_end_developer', 0.8192589282989502),\n",
       " (u'phone_gap', 0.8192075490951538),\n",
       " (u'javascript_framework', 0.8191279172897339),\n",
       " (u'nosql_database', 0.8190893530845642),\n",
       " (u'dojo', 0.8190855979919434),\n",
       " (u'serverside', 0.8189901113510132),\n",
       " (u'veteran', 0.8188142776489258),\n",
       " (u'restful_service', 0.8187569975852966),\n",
       " (u'magento_programmer', 0.8169186115264893),\n",
       " (u'ms_azure', 0.8167128562927246),\n",
       " (u'spring_boot', 0.8164030313491821),\n",
       " (u'html5', 0.8156924247741699),\n",
       " (u'playstore', 0.8156911730766296),\n",
       " (u'appstore', 0.8154234290122986),\n",
       " (u'web_ux_design', 0.8154138326644897),\n",
       " (u'no__sql', 0.8151313066482544),\n",
       " (u'tdd', 0.8147590756416321),\n",
       " (u'hybrid_developer', 0.8144230842590332),\n",
       " (u'senior_software_engineer', 0.8143936991691589),\n",
       " (u'air_force', 0.8143590092658997),\n",
       " (u'redis', 0.8128796815872192),\n",
       " (u'socket', 0.8128024339675903),\n",
       " (u'scalable_distribute', 0.8121465444564819),\n",
       " (u'adobe_experience_management_suite', 0.8119006752967834),\n",
       " (u'paypal', 0.8117127418518066),\n",
       " (u'software_development_kit', 0.8117092847824097),\n",
       " (u'css3', 0.8115301132202148),\n",
       " (u'or_design', 0.8114418983459473),\n",
       " (u'senior_programmer', 0.8113954663276672),\n",
       " (u'springboot', 0.8113210201263428),\n",
       " (u'web_tool', 0.8109619617462158),\n",
       " (u'full_stack_engineer', 0.810425877571106),\n",
       " (u'framework_development', 0.8101856112480164),\n",
       " (u'kubernetes', 0.810155987739563),\n",
       " (u'web_architect', 0.8099838495254517),\n",
       " (u'nosql', 0.8099304437637329),\n",
       " (u'magento_architect', 0.8095452785491943),\n",
       " (u'user_journey', 0.8094909191131592),\n",
       " (u'woocommerce', 0.8093488216400146),\n",
       " (u'cloud_platform', 0.809120774269104),\n",
       " (u'php_lead', 0.80845046043396),\n",
       " (u'model_view_controller', 0.8083219528198242),\n",
       " (u'stash', 0.8082432746887207),\n",
       " (u'ext', 0.8080920577049255),\n",
       " (u'responsive/adaptive_design', 0.8079089522361755),\n",
       " (u'front_end', 0.8077892065048218),\n",
       " (u'zeromq', 0.807259202003479),\n",
       " (u'js_frameworks', 0.8070929050445557),\n",
       " (u'mysql_db', 0.807024359703064),\n",
       " (u'cascade_style_sheet', 0.8062223792076111),\n",
       " (u'laravel_framework', 0.8059934377670288),\n",
       " (u'codeigniter_framework', 0.8057296872138977),\n",
       " (u'grails', 0.804581880569458),\n",
       " (u'crawl', 0.8041242361068726),\n",
       " (u'multitiered', 0.8040775060653687),\n",
       " (u'js_framework', 0.8039942979812622),\n",
       " (u'html_developer', 0.8034518361091614),\n",
       " (u'rabbitmq', 0.8033852577209473),\n",
       " (u'react_native', 0.8032656908035278),\n",
       " (u'codeigniter_developer', 0.8031542301177979),\n",
       " (u'css2', 0.8029035925865173),\n",
       " (u'python_developer', 0.8025425672531128),\n",
       " (u'play_and_javascript', 0.8024415969848633),\n",
       " (u'3d_engine', 0.8018772602081299),\n",
       " (u'bluemix', 0.8018579483032227),\n",
       " (u'geodistributed', 0.8017860651016235),\n",
       " (u'native_android', 0.8017847537994385),\n",
       " (u'cors', 0.8017616271972656),\n",
       " (u'asf', 0.8017589449882507),\n",
       " (u'websocket', 0.8014872670173645),\n",
       " (u'core_php_developer', 0.8004641532897949),\n",
       " (u'stream_protocol', 0.8000181317329407),\n",
       " (u'php_development', 0.7999439835548401),\n",
       " (u'java_frameworks', 0.7998257875442505),\n",
       " (u'css_30', 0.7997254133224487),\n",
       " (u'codeignitor', 0.7994517087936401),\n",
       " (u'mobile_ui', 0.7986682057380676),\n",
       " (u'canvas', 0.7983762621879578),\n",
       " (u'advance_php', 0.7981869578361511),\n",
       " (u'postgres', 0.7980055809020996),\n",
       " (u'kinesis', 0.7979010343551636),\n",
       " (u'route_53', 0.7977803945541382),\n",
       " (u'ux_development', 0.7972257137298584),\n",
       " (u'postgress', 0.7972134947776794),\n",
       " (u'springmvc', 0.7970743179321289),\n",
       " (u'senior_web_developer', 0.7963873147964478),\n",
       " (u'senior_software_developer', 0.7963151931762695),\n",
       " (u'rest_service', 0.7959240674972534),\n",
       " (u'or_nosql', 0.795629620552063),\n",
       " (u'artifactory', 0.7955926060676575),\n",
       " (u'shopify', 0.7953507900238037),\n",
       " (u'google_api', 0.7951990962028503),\n",
       " (u'intellij', 0.7950568795204163),\n",
       " (u'ux_design', 0.7949963212013245),\n",
       " (u'bigquery', 0.7949681282043457),\n",
       " (u'html_50', 0.7942996621131897),\n",
       " (u'elastic_search', 0.7942856550216675),\n",
       " (u'native_ios', 0.794273853302002),\n",
       " (u'large_scale', 0.7940057516098022),\n",
       " (u'wordpress_programmer', 0.7932824492454529),\n",
       " (u'amazon_cloud', 0.7929429411888123),\n",
       " (u'web_program', 0.7928417921066284),\n",
       " (u'coffeescript', 0.7927553653717041),\n",
       " (u'solr', 0.7925401329994202),\n",
       " (u'scala', 0.7922948002815247),\n",
       " (u'open_source_technology', 0.7922738194465637),\n",
       " (u'pyramid', 0.7922403812408447),\n",
       " (u'php5', 0.7922391891479492),\n",
       " (u'5', 0.7920769453048706),\n",
       " (u'distribute_databases', 0.7919602394104004),\n",
       " (u'php_mysql_ajax', 0.791711688041687),\n",
       " (u'cq', 0.791693925857544),\n",
       " (u'frame', 0.7912594079971313),\n",
       " (u'lead_developer', 0.7910438776016235),\n",
       " (u'rundeck', 0.790835440158844),\n",
       " (u'object__orient_design', 0.7904046177864075),\n",
       " (u'css_3', 0.7900117039680481),\n",
       " (u'laravel_programmer', 0.7897717952728271),\n",
       " (u'singleton', 0.7896918058395386),\n",
       " (u'senior_developer', 0.789678692817688),\n",
       " (u'elk', 0.7895768284797668),\n",
       " (u'html/css', 0.7894167900085449),\n",
       " (u'elastic', 0.7890506386756897),\n",
       " (u'high_available', 0.7890384197235107),\n",
       " (u'memory_database', 0.7890005707740784),\n",
       " (u'spock', 0.788647472858429),\n",
       " (u'template', 0.7884982824325562),\n",
       " (u'plugins', 0.7884854674339294),\n",
       " (u'corephp', 0.7884300351142883),\n",
       " (u'sns', 0.7880092859268188),\n",
       " (u'restful_apis', 0.7877841591835022),\n",
       " (u'cloudbase', 0.7875716686248779),\n",
       " (u'toolkits', 0.7873758673667908),\n",
       " (u'code_igniter', 0.786882221698761),\n",
       " (u'agile_software_development', 0.7865825891494751),\n",
       " (u'software_engineer_manager', 0.7861847877502441),\n",
       " (u'php_5', 0.7860729098320007),\n",
       " (u'payment_solutions', 0.7858583927154541),\n",
       " (u'riak', 0.7857495546340942),\n",
       " (u'vs', 0.785552442073822),\n",
       " (u'mobile_ux', 0.7854920625686646)]"
      ]
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model_100d.similar_by_word(skill_transform('node_js'), 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'angular'"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(r'^angular.*$', 'angular', \"angular.js\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}