skill2vec_ver4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import cPickle \n",
    "from nltk.corpus import stopwords\n",
    "from gensim.models import word2vec\n",
    "import nltk.data\n",
    "import re\n",
    "import logging\n",
    "from nltk.stem.snowball import *\n",
    "import itertools\n",
    "\n",
    "from skill_transform import *\n",
    "\n",
    "# Python 2.x:\n",
    "import HTMLParser\n",
    "html_parser = HTMLParser.HTMLParser()\n",
    "\n",
    "import multiprocessing\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)   \n",
    "\n",
    "from nltk.stem import WordNetLemmatizer, PorterStemmer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()\n",
    "porter = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "TEST_CASE = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'html5_and_css3'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skill_transform(\"html5_and_css3_or_html6\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_case(model, model_name):\n",
    "    cases = [ \"machine learning\", \"js\", \"javascript\", \"python\", \"html\", \"html5\", \"css\", \"angular\", \"nodejs\" ]\n",
    "    TEST_CASE[model_name] = []\n",
    "    for case in cases:\n",
    "        try:\n",
    "            TEST_CASE[model_name].append(model.similar_by_word(skill_transform(case)))\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cPickle as pickle\n",
    "musthave_dice_naruki_data = pickle.load( open( \"musthave_dice_naruki_data.p\", \"rb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "allFiles = glob.glob(\"dice-full*.json\")\n",
    "frame = pd.DataFrame()\n",
    "list_ = []\n",
    "for file_ in allFiles:\n",
    "    df = pd.read_json(file_)\n",
    "    list_.append(df)\n",
    "skill_dice_jd = pd.concat(list_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1697363"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(skill_dice_jd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u\"Essential Functions:  * Independently perform various complex analyses and audits relating to Medicaid or other reimbursement-related engagements, with a focus on identification of fraud, waste, abuse or misuse of Medicaid or other payer funds * Develop and maintain general knowledge of Medicaid and/or other payer statutes, regulations, provider billing manuals and other healthcare reimbursement policies * Review applicable Federal and State Medicaid policies and regulations relating to healthcare reimbursement including alternate payment methodologies * Conduct research to determine the standards applicable to billing compliance with Medicaid or other government program regulations * Review, summarize, and apply policies and regulations to specific analyses, reviews and audits * Perform data analytics to identify fraud, waste or abuse of Medicaid or other health care system funds * Interpret and analyze healthcare data, to include review and analysis of medical, financial or other records, to identify inconsistencies, anomalies, abnormal billing patterns and other indicators of suspected fraud, waste or abuse (e.g. services not rendered, up-coding, un-bundling, etc.) * Prepare and distribute analytical data reports and/or records * Work cooperatively and constructively with team members, including mentoring, training and assisting team members as required * Maintain security and confidentiality of all protected health information encountered in performance of duties * Participate in client conference calls as appropriate * Handle phone calls and e-mails from providers to answer any questions * Evaluate business processes to identify opportunities for improvement * Work within the defined scope of assigned projectsQualifications:  * Bachelor's degree in Health Information Administration, Accounting, Finance, Healthcare, Criminal Justice or related field required * CPA, CFE, AHFI, RHIT or RHIA preferred * 3 years of related experience required * Must have strong ability to analyze data to identify issues relating to health care fraud, waste and abuse * Knowledge of the healthcare industry; fraud, waste and abuse audit operations, and claims and payment processing preferred * Experience in detecting and investigating health insurance fraud, waste, and abuse preferred * Knowledge of managed care rules and regulations * Auditing experience preferred * Proficient use of applicable software programs, including Microsoft Office software (Excel, Word, PowerPoint, Access, etc.) * Knowledge of and experience with SQL and report writing preferred * Organized and detail oriented with ability to think independently and problem solve effectively * Demonstrated ability to communicate verbally and in writing with all levels of an organization, both internally and externally * Effectively multi-task, prioritize assignments, manage multiple deadlines * Ability to work in a team environment * Must be able to travel based on client and business needsAdditional Qualifications:  * Communication and presentation skills crucial * Very professional environment * Looking for someone with high level of SQL skills that can not only pull data, but can also analyze and audit it * The Senior Healthcare Data Analyst will work with a multi-disciplinary team to identify through data analytics provider claims involving potential fraud, waste or abuse of Medicaid or other health care system funds; * Will review and analyze such claims to determine if over-payments or underpayments occurred in light of applicable policies; * Will perform reimbursement calculations to determine the amounts of such miss-payments for the purpose of preparing provider notifications; and will prepare supporting documentation and reports as required.Irvens Andre, Technical Recruiter, The Judge Group 1117 Perimeter Center West, Suite W502, Atlanta, GA 30338Office:  678.987.9894| iandre@judge.com | www.judge.com  ID:  521342 Type:  Contract Salary:  To Be Discussed\",\n",
       " u\"Essential Job Functions   Identifies customer purchasing requirements and provides technical specifications and related documentation to assist the business development staff in the sale of products and systems.   Assists in creating demonstration software and systems to showcase products and systems.   Responds to customer inquiries concerning system software and applications, pre-sale and post-sale. Refers as necessary to appropriate personnel.   Assists in testing user configurations for system compatibility. Provides support in the resolution of system malfunction and/or refers as appropriate.   Monitors and records system installation/maintenance progress to document progress and matters of significance.   Interacts with engineering groups to assist in basic design changes to equipment. Assists in tracking the progress of the projects with accountability to the customer, project manager and management to ensure an on-time and smooth product launch for the customer.  Qualifications  Basic Qualifications   Bachelor's degree or equivalent combination of education and experience   Bachelor's degree in computer science, engineering or related field preferred   Zero or more years of applications engineering, field support or technical sales experience   Experience working with overall department/account business objectives and goals   Experience working with operating systems   Experience working with software tools including compilers, assemblers, debuggers and simulators   Experience working with assembly languages such as HTML programming, MySQL, C, C++, etc   Other Qualifications  3 + years experience Technologist with exposure to a variety of shop floor Manufacturing Applications, with a focus on Manufacturing, Machine Tool, or Robotics.  3 + years experience in Machine Tool PLC and SCADA software programming 3 + years experience in Data collection and reporting Basic project management.  Good communication skills.  Communication skills   Interpersonal skills to interact with customers and team members   Organization and time management skills   Analytical and problem solving skills   Ability to listen and comprehend   Ability and willingness to learn    Department:  Applications Clearance:  None\",\n",
       " u\"Entwine is a global digital innovation agency specialized in healthcare and scientific communications. Working in a close partnership with Chameleon Communications, we are a member of Healthcare Consultancy Group and Omnicom Group, with offices in London, New York, Los Angeles and Manchester. Our clients represent the top blue-chip global pharmaceutical companies. We having an exciting opportunity for a Designer to join our team to help us build highly creative communications for leaders in the industry! We believe that in today's world the ability to clearly communicate through great conceptual thinking, strong modern branding and cross-channel design and information design application and information design, are at the core of modern communications. This is a great opportunity to assume a key position in the expansion and growth of creative team and ultimately become a leading creative innovation influencer in the industry. Requirements5+ years of agency experience as a designerPharma and healthcare experience desired, medical communications preferred OverviewDesigner will be as a part of our creative team responsible for designing, developing and delivering a wide range of communication materials for our pharmaceutical clients across all media platforms. The right candidate will be a key contributor to the team, juggling multiple visual communication design requests in a supremely organized fashion with creative flair and a devilish eye for detail. He/she will work alongside Digital Director and internal account teams, maintaining high level of communication and collaboration. Ideal candidate has to be able to demonstrate a proven track record of delivering high level of creative contributions and design quality, ability to creatively solve challenges and making things happen. Experience and expertiseThe breadth of experience should include most of:Top quality design skills (we would love to see your portfolio)Great understanding of design principles and approach to current design trendsStrong infographics and data visualization development experience: distilling content and developing storyflow and information hierarchyAbility to communicate the story and design powerful presentations, whether it's a PowerPoint, Keynote or other media from the ground upAbility to create visual assets for video, audio and 2D/3D animations, experience with storyboardingUnderstanding of HTML5 and CSSUI design experience with an top level understanding of UX principlesiBooks Author development or experience producing non-standard content delivery formats welcomeUnderstanding of the print desktop and prepress processes, ability to contribute or help overseeing the deliveryImpressive conceptual design skills and an innovative mindset, generating concepts and ideas for delivery to our clientsAbility to think creatively beyond design; generate innovative ideas for online applications and websites, apps, printed materials, congresses, exhibitions or live meetingsGreat presentation skills, oral communication and ability to convey creative concepts and design workProject management experience, leading on design projects through to final execution, working on collaborative projects, coordinating deliverables and deadlines with colleagues and clients throughout all phases of a projectExperience working to tight deadlines while working across multiple and diverse projectsExceptional Adobe CS, presentation media (PowerPoint/Keynote) skillsIn many cases the candidate will need to lead design projects and work alongside Digital Director and wider account teams to liaise with clients. Personal Skills-We have high standards. You will be bringing a creative spark to a high-achieving in-house design team with your solid integrated understanding of print and digital, including branding, comms, motion graphics, illustration, web and interactive.-Must be motivated with a can-do attitude, a yes-man, flexible, with excellent communication skills. Should show ability to overcome challenging requirements and demonstrate creativity and innovation in finding ways to execute never-been-done-before ideas. Keeping abreast of the latest creative and digital trends is an absolute must.-The candidate will be reporting to the core team in London and is expected to be well organized, with the ability to work efficiently with remote teams. Will need to possess great communication skills over videoconference, TC and email communication. Travel across US and Europe might be required per team, business or client needs. Those who wish to apply for this position: Please submit your resume online to the Careers section of the company website (http://www.hcg-int.com/) or (https://hcg-int-careers.silkroad.com/)A member of the Healthcare Consultancy Group (HCG) which serves the pharmaceutical, biotech, and device industries by providing strategic, scientific communications solutions at all stages of the product lifecycle. Part of Omnicom's Diversified Agency Services (DAS), HCG is a leader in the healthcare segment, with over 350 employees, more than 90 with advanced degrees (MD, PhD, or PharmD). We are a scientifically rigorous, strategically focused, creatively balanced agency built for the future.The companies within the Healthcare Consultancy Group are Equal Opportunity Employers. All applicants will be considered without regard to race, color, religion, sex, age, national origin, citizenship status, sexual orientation, disability, veteran status or any category or class of person protected by law.\",\n",
       " u'Hi, We are looking for SAP PP Functional Consultant. If you are available please send me your updated resume or I would appreciate if you can refer someone who is available for this job. Full Name:Mobile No:Expected hourly rate on C2C all-inclusive:Email Id:Current Location:Availability:Work authorization status ? Role: SAP PP Functional ConsultantDuration: 6+ months Location: Houston TX Client is opening a new plan in Oklahoma. Project is about IT enablement.Following are three Business scenarios from Blue print till deployment by SAP PP/WM consultant. requires travel to Oklahoma and working with Houston IT team. Integrate SAP WM with AGV (Automated Guided Vehicle)Integrate SAP PP with AGV for work centersQuality Management process Having 8+ years of experience in SAP Functional experience in Production Planning, Warehouse Management and Quality management Thanks &amp; Regards,  SAMOslo Solutions LLCPhone: 253 - 336 - 5347Email Id: sam@oslosolutions.comWebsite: http://oslosolutions.com4080 148th Avenue , Building M , Suite GRedmond, WA 98052, USA. ',\n",
       " u'We have a client in the Madison area in need of a strong Business Objects Analyst/Developer to help design and build-out new functionality within a BObj 4.0 and 4.1 environment.  This is a new project for supporting new areas of their business as well as develop improved functionality for what is already in production.  There is a great deal of work to be done over the long-term as the company has experienced great benefits from BObj. Requirements- Experience with planning, implementing, testing and training across Business Objects projects.- Knowledge of best practices and excellent experience providing innovative ideas.- Due to the amount of work, we need someone who is very hands-on and open to contributing designs through to end-user support.  In addition to providing your design expertise, you will have the opportunity for much hands-on work.  - BObj Administration skills are helpful, but not required.- Experience in costing very helpful.- Ability to read and/or debug ABAP code is also helpful.- At a minimum, you must be strong in the Sales and Manufacturing domains.- Mobile experience would be very helpful, but not required.- Experience working with multiple company codes and across facilities very helpful.- Excellent communication skills are a must! This is a long-term, direct, client of ours and we are interviewing people immediately.  Please apply today if you are interested and we will be in touch. Thank you!  ']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[s[0] for s in skill_dice_jd.loc[0, 'job_description'][0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "company_info       [Working at the crossroads of people and trans...\n",
       "crawl_date                                                25/02/2017\n",
       "fullpart           Contract Corp-To-Corp, Contract Independent, C...\n",
       "job_description    [Essential Functions:  * Independently perform...\n",
       "salary                                                    Negotiable\n",
       "since                                             Posted 6 hours ago\n",
       "skills             Accounting, Analysis, Analyst, Data Analysis, ...\n",
       "title                                   Sr. Healthcare Data Analysis\n",
       "url                https://www.dice.com/jobs/detail/Sr.-Healthcar...\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skill_dice_jd.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dice_JD = pd.read_json(\"dice-full-2.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-07 10:32:22,460 : INFO : collecting all words and their counts\n",
      "2017-05-07 10:32:22,461 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-05-07 10:32:22,540 : INFO : PROGRESS: at sentence #10000, processed 216479 words, keeping 11609 word types\n",
      "2017-05-07 10:32:22,555 : INFO : collected 12535 word types from a corpus of 254463 raw words and 12795 sentences\n",
      "2017-05-07 10:32:22,556 : INFO : Loading a fresh vocabulary\n",
      "2017-05-07 10:32:22,588 : INFO : min_count=1 retains 12535 unique words (100% of original 12535, drops 0)\n",
      "2017-05-07 10:32:22,588 : INFO : min_count=1 leaves 254463 word corpus (100% of original 254463, drops 0)\n",
      "2017-05-07 10:32:22,617 : INFO : deleting the raw counts dictionary of 12535 items\n",
      "2017-05-07 10:32:22,618 : INFO : sample=0.001 downsamples 1 most-common words\n",
      "2017-05-07 10:32:22,618 : INFO : downsampling leaves estimated 254448 word corpus (100.0% of prior 254463)\n",
      "2017-05-07 10:32:22,619 : INFO : estimated required memory for 12535 words and 300 dimensions: 36351500 bytes\n",
      "2017-05-07 10:32:22,650 : INFO : resetting layer weights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12795\n",
      "Training model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-07 10:32:22,828 : INFO : training model with 32 workers on 12535 vocabulary and 300 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-05-07 10:32:23,955 : INFO : PROGRESS: at 13.80% examples, 133942 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:32:25,019 : INFO : PROGRESS: at 28.65% examples, 131158 words/s, in_qsize 63, out_qsize 1\n",
      "2017-05-07 10:32:25,995 : INFO : PROGRESS: at 54.61% examples, 180172 words/s, in_qsize 43, out_qsize 2\n",
      "2017-05-07 10:32:26,976 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-05-07 10:32:26,989 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-05-07 10:32:27,069 : INFO : PROGRESS: at 69.94% examples, 172171 words/s, in_qsize 29, out_qsize 1\n",
      "2017-05-07 10:32:27,074 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-05-07 10:32:27,118 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-05-07 10:32:27,176 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-05-07 10:32:27,241 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-05-07 10:32:27,333 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-05-07 10:32:27,337 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-05-07 10:32:27,341 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-05-07 10:32:27,347 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-05-07 10:32:27,366 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-05-07 10:32:27,373 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-05-07 10:32:27,381 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-05-07 10:32:27,391 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-05-07 10:32:27,433 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-05-07 10:32:27,496 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-05-07 10:32:27,500 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-05-07 10:32:27,502 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-05-07 10:32:27,578 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-05-07 10:32:27,594 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-05-07 10:32:27,607 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-05-07 10:32:27,624 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-05-07 10:32:27,639 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-05-07 10:32:27,644 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-05-07 10:32:27,675 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-05-07 10:32:27,680 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-07 10:32:27,700 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-07 10:32:27,709 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-07 10:32:27,718 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-07 10:32:27,726 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-07 10:32:27,735 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-07 10:32:27,739 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-07 10:32:27,740 : INFO : training on 1017852 raw words (1017801 effective words) took 4.9s, 207560 effective words/s\n",
      "2017-05-07 10:32:27,741 : WARNING : under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2017-05-07 10:32:27,750 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    }
   ],
   "source": [
    "with open(\"prep_data_tokens_underscore_1\", \"rb\") as g:\n",
    "    data_dice = cPickle.load(g)\n",
    "dice_data = [ [ skill_transform(d.replace(\"_\", \" \")) for d in dice if d is not None ] for dice in data_dice if dice is not None ]\n",
    "\n",
    "\n",
    "musthave_dice_data = dice_data + musthave_data\n",
    "musthave_dice_data = [ list(set([ d for d in data if d is not None and len(d) > 0 ])) for data in musthave_dice_data  ]\n",
    "\n",
    "print len(musthave_dice_data)\n",
    "musthave_dice_data\n",
    "\n",
    "print (\"Training model...\")\n",
    "musthave_dice_model = word2vec.Word2Vec(musthave_dice_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=300, \n",
    "          min_count=1, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )\n",
    "\n",
    "musthave_dice_model.init_sims(replace=True)\n",
    "test_case(musthave_dice_model, \"musthave_dice_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# model.similar_by_word(skill_transform('machine learning'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# model.similar_by_word('python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120107\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "110660"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_data = musthave_dice_data\n",
    "data_naruki = pd.read_csv('naukri_skill_full', header = 0, encoding='ISO-8859-1')\n",
    "data_naruki.drop_duplicates(subset=['id', 'skill'], keep='last')\n",
    "\n",
    "data_naruki['skill'] = data_naruki['skill'].apply(skill_transform)\n",
    "data_naruki_final = data_naruki.groupby('id')['skill'].apply(list)\n",
    "print len(data_naruki_final)\n",
    "\n",
    "for skills in data_naruki_final:\n",
    "    if len(skills) > 2 and skills not in musthave_dice_naruki_data:\n",
    "        musthave_dice_naruki_data.append(skills)\n",
    "        \n",
    "len(musthave_dice_naruki_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import cPickle as pickle\n",
    "pickle.dump( musthave_dice_naruki_data, open( \"musthave_dice_naruki_data.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-07 10:36:14,874 : INFO : collecting all words and their counts\n",
      "2017-05-07 10:36:14,875 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-05-07 10:36:14,953 : INFO : PROGRESS: at sentence #10000, processed 216479 words, keeping 11609 word types\n",
      "2017-05-07 10:36:14,998 : INFO : PROGRESS: at sentence #20000, processed 316511 words, keeping 21557 word types\n",
      "2017-05-07 10:36:15,038 : INFO : PROGRESS: at sentence #30000, processed 403006 words, keeping 30521 word types\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-07 10:36:15,078 : INFO : PROGRESS: at sentence #40000, processed 489603 words, keeping 38336 word types\n",
      "2017-05-07 10:36:15,119 : INFO : PROGRESS: at sentence #50000, processed 579752 words, keeping 45956 word types\n",
      "2017-05-07 10:36:15,157 : INFO : PROGRESS: at sentence #60000, processed 666564 words, keeping 52001 word types\n",
      "2017-05-07 10:36:15,197 : INFO : PROGRESS: at sentence #70000, processed 751300 words, keeping 58857 word types\n",
      "2017-05-07 10:36:15,236 : INFO : PROGRESS: at sentence #80000, processed 836205 words, keeping 65151 word types\n",
      "2017-05-07 10:36:15,273 : INFO : PROGRESS: at sentence #90000, processed 923058 words, keeping 70256 word types\n",
      "2017-05-07 10:36:15,312 : INFO : PROGRESS: at sentence #100000, processed 1013120 words, keeping 74899 word types\n",
      "2017-05-07 10:36:15,349 : INFO : PROGRESS: at sentence #110000, processed 1099320 words, keeping 80588 word types\n",
      "2017-05-07 10:36:15,354 : INFO : collected 80695 word types from a corpus of 1105249 raw words and 110660 sentences\n",
      "2017-05-07 10:36:15,355 : INFO : Loading a fresh vocabulary\n",
      "2017-05-07 10:36:15,455 : INFO : min_count=3 retains 23483 unique words (29% of original 80695, drops 57212)\n",
      "2017-05-07 10:36:15,456 : INFO : min_count=3 leaves 1039163 word corpus (94% of original 1105249, drops 66086)\n",
      "2017-05-07 10:36:15,520 : INFO : deleting the raw counts dictionary of 80695 items\n",
      "2017-05-07 10:36:15,523 : INFO : sample=0.001 downsamples 21 most-common words\n",
      "2017-05-07 10:36:15,524 : INFO : downsampling leaves estimated 1014556 word corpus (97.6% of prior 1039163)\n",
      "2017-05-07 10:36:15,525 : INFO : estimated required memory for 23483 words and 300 dimensions: 68100700 bytes\n",
      "2017-05-07 10:36:15,588 : INFO : resetting layer weights\n",
      "2017-05-07 10:36:15,922 : INFO : training model with 32 workers on 23483 vocabulary and 300 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-05-07 10:36:17,058 : INFO : PROGRESS: at 0.81% examples, 79100 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:36:18,092 : INFO : PROGRESS: at 7.11% examples, 167041 words/s, in_qsize 63, out_qsize 2\n",
      "2017-05-07 10:36:19,088 : INFO : PROGRESS: at 14.98% examples, 211851 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:36:20,248 : INFO : PROGRESS: at 23.58% examples, 223396 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:36:21,390 : INFO : PROGRESS: at 27.35% examples, 216863 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:36:22,477 : INFO : PROGRESS: at 35.18% examples, 231742 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:36:23,480 : INFO : PROGRESS: at 45.58% examples, 248413 words/s, in_qsize 37, out_qsize 0\n",
      "2017-05-07 10:36:24,592 : INFO : PROGRESS: at 50.30% examples, 238668 words/s, in_qsize 37, out_qsize 0\n",
      "2017-05-07 10:36:25,642 : INFO : PROGRESS: at 54.76% examples, 238236 words/s, in_qsize 31, out_qsize 0\n",
      "2017-05-07 10:36:26,681 : INFO : PROGRESS: at 62.55% examples, 243296 words/s, in_qsize 31, out_qsize 0\n",
      "2017-05-07 10:36:27,742 : INFO : PROGRESS: at 71.16% examples, 247411 words/s, in_qsize 13, out_qsize 0\n",
      "2017-05-07 10:36:28,910 : INFO : PROGRESS: at 75.88% examples, 242413 words/s, in_qsize 49, out_qsize 0\n",
      "2017-05-07 10:36:29,943 : INFO : PROGRESS: at 80.92% examples, 241820 words/s, in_qsize 63, out_qsize 0\n",
      "2017-05-07 10:36:30,957 : INFO : PROGRESS: at 88.43% examples, 243595 words/s, in_qsize 45, out_qsize 0\n",
      "2017-05-07 10:36:31,364 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-05-07 10:36:31,417 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-05-07 10:36:31,431 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-05-07 10:36:31,437 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-05-07 10:36:31,491 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-05-07 10:36:31,511 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-05-07 10:36:31,553 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-05-07 10:36:31,558 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-05-07 10:36:31,561 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-05-07 10:36:31,579 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-05-07 10:36:31,607 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-05-07 10:36:31,640 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-05-07 10:36:31,676 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-05-07 10:36:31,710 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-05-07 10:36:31,720 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-05-07 10:36:31,723 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-05-07 10:36:31,773 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-05-07 10:36:31,796 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-05-07 10:36:31,819 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-05-07 10:36:31,824 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-05-07 10:36:31,826 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-05-07 10:36:31,833 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-05-07 10:36:31,852 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-05-07 10:36:31,856 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-05-07 10:36:31,873 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-05-07 10:36:31,882 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-07 10:36:31,884 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-07 10:36:31,888 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-07 10:36:31,891 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-07 10:36:31,892 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-07 10:36:31,898 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-07 10:36:31,899 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-07 10:36:31,900 : INFO : training on 4420996 raw words (4058185 effective words) took 16.0s, 254149 effective words/s\n",
      "2017-05-07 10:36:31,917 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    }
   ],
   "source": [
    "print (\"Training model...\")\n",
    "musthave_dice_naruki_model = word2vec.Word2Vec(musthave_dice_naruki_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=300, \n",
    "          min_count=3, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )\n",
    "\n",
    "musthave_dice_naruki_model.init_sims(replace=True)\n",
    "test_case(musthave_dice_naruki_model, \"musthave_dice_naruki_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'musthave_dice_model': [[(u'doctoral', 0.9242714047431946),\n",
       "   (u'postdoc', 0.9213629961013794),\n",
       "   ('bigdata', 0.9166296124458313),\n",
       "   (u'doctor', 0.9145913124084473),\n",
       "   (u'hadoop', 0.9049304127693176),\n",
       "   (u'dsp', 0.9031425714492798),\n",
       "   (u'weka', 0.8997036218643188),\n",
       "   (u'data_science', 0.8971030116081238),\n",
       "   (u'hdfs', 0.8913494348526001),\n",
       "   (u'recommendation', 0.89049232006073)],\n",
       "  [(u'css', 0.9650534987449646),\n",
       "   (u'css3', 0.9572659134864807),\n",
       "   (u'html5', 0.949856162071228),\n",
       "   (u'prototype', 0.9491482377052307),\n",
       "   (u'react', 0.9414901733398438),\n",
       "   (u'html5/css3/javascript', 0.9389783143997192),\n",
       "   (u'html', 0.9377188682556152),\n",
       "   (u'user_journey', 0.9332085251808167),\n",
       "   (u'cascade_style_sheet', 0.9284156560897827),\n",
       "   (u'adobe_aem', 0.9280628561973572)],\n",
       "  [(u'css', 0.9650534987449646),\n",
       "   (u'css3', 0.9572659134864807),\n",
       "   (u'html5', 0.949856162071228),\n",
       "   (u'prototype', 0.9491482377052307),\n",
       "   (u'react', 0.9414901733398438),\n",
       "   (u'html5/css3/javascript', 0.9389783143997192),\n",
       "   (u'html', 0.9377188682556152),\n",
       "   (u'user_journey', 0.9332085251808167),\n",
       "   (u'cascade_style_sheet', 0.9284156560897827),\n",
       "   (u'adobe_aem', 0.9280628561973572)],\n",
       "  [(u'perl', 0.9459517002105713),\n",
       "   (u'aws', 0.9320412278175354),\n",
       "   (u'tool', 0.9314518570899963),\n",
       "   (u'administrator', 0.9179779291152954),\n",
       "   (u'stream_audio', 0.9100028276443481),\n",
       "   (u'c/c++', 0.9074482917785645),\n",
       "   (u'hadoop', 0.9057959318161011),\n",
       "   (u'map_reduce', 0.9057800769805908),\n",
       "   (u'c', 0.9022610187530518),\n",
       "   (u'kinesis', 0.9019452929496765)],\n",
       "  [(u'css', 0.9720954298973083),\n",
       "   (u'ued', 0.9662882089614868),\n",
       "   (u'prototype', 0.9607166647911072),\n",
       "   (u'xd', 0.9601514339447021),\n",
       "   (u'uxpin', 0.9587953090667725),\n",
       "   (u'ui_designer', 0.9568859934806824),\n",
       "   (u'invision', 0.9565894603729248),\n",
       "   (u'template', 0.9563713073730469),\n",
       "   (u'user_experience_designer', 0.9553638100624084),\n",
       "   (u'framer', 0.954596757888794)],\n",
       "  [(u'interaction', 0.9727522730827332),\n",
       "   (u'visual', 0.9684633016586304),\n",
       "   (u'frontends', 0.9682190418243408),\n",
       "   (u'user_experience_designer', 0.9670733213424683),\n",
       "   (u'quartz_composer', 0.9666818976402283),\n",
       "   (u'invision', 0.9655629992485046),\n",
       "   (u'css3', 0.9643071889877319),\n",
       "   (u'ui_design', 0.9638136625289917),\n",
       "   (u'react', 0.9631743431091309),\n",
       "   (u'xd', 0.9624539017677307)],\n",
       "  [(u'html', 0.9720954298973083),\n",
       "   (u'prototype', 0.9685227274894714),\n",
       "   (u'javascript', 0.9650534987449646),\n",
       "   (u'ued', 0.9639898538589478),\n",
       "   (u'invision', 0.961262583732605),\n",
       "   (u'xd', 0.9608720541000366),\n",
       "   (u'css3', 0.9607824087142944),\n",
       "   (u'html5', 0.9607509970664978),\n",
       "   (u'quartz_composer', 0.9607073068618774),\n",
       "   (u'ui_designer', 0.9598108530044556)],\n",
       "  [(u'less', 0.9821205139160156),\n",
       "   (u'template', 0.9758352637290955),\n",
       "   (u'responsive', 0.9707397222518921),\n",
       "   (u'cs', 0.9693396091461182),\n",
       "   (u'npm', 0.9682244062423706),\n",
       "   (u'frontend', 0.9673136472702026),\n",
       "   (u'bower', 0.9668498039245605),\n",
       "   (u'dojo', 0.9654262065887451),\n",
       "   (u'load', 0.9652680158615112),\n",
       "   (u'three', 0.9652514457702637)],\n",
       "  [(u'npm', 0.9865400195121765),\n",
       "   (u'bower', 0.9837674498558044),\n",
       "   (u'adobe_aem', 0.9833667874336243),\n",
       "   (u'backbone', 0.9830563068389893),\n",
       "   (u'prototype', 0.9816056489944458),\n",
       "   (u'or_ruby_on_rail', 0.9813099503517151),\n",
       "   (u'3d_graphics', 0.981156587600708),\n",
       "   (u'template', 0.9810644388198853),\n",
       "   (u'3d_engine', 0.9807811975479126),\n",
       "   (u'jsf', 0.9805082678794861)]],\n",
       " 'musthave_dice_naruki_model': [[(u'data_scientist', 0.9070005416870117),\n",
       "   ('nlp', 0.8931161165237427),\n",
       "   (u'anomaly_detection', 0.8719435930252075),\n",
       "   (u'deep_learn', 0.8702843189239502),\n",
       "   (u'random_forest', 0.862106204032898),\n",
       "   (u'recommendation', 0.8620985746383667),\n",
       "   (u'scala_language', 0.8575278520584106),\n",
       "   (u'k__mean', 0.8546137809753418),\n",
       "   (u'tensorflow', 0.8538155555725098),\n",
       "   (u'time_series_analysis', 0.853800892829895)],\n",
       "  [(u'html5', 0.8463349342346191),\n",
       "   (u'ajax', 0.8372377753257751),\n",
       "   (u'css3', 0.8362160921096802),\n",
       "   (u'mvc_architecture', 0.8164827227592468),\n",
       "   (u'mvc_frameworks', 0.8085825443267822),\n",
       "   (u'web_tool', 0.8079936504364014),\n",
       "   (u'oops_concept', 0.8062907457351685),\n",
       "   (u'bootstrap', 0.8028784394264221),\n",
       "   (u'senior_web_developer', 0.7993264198303223),\n",
       "   (u'webdeveloper', 0.7979726791381836)],\n",
       "  [(u'html5', 0.8463349342346191),\n",
       "   (u'ajax', 0.8372377753257751),\n",
       "   (u'css3', 0.8362160921096802),\n",
       "   (u'mvc_architecture', 0.8164827227592468),\n",
       "   (u'mvc_frameworks', 0.8085825443267822),\n",
       "   (u'web_tool', 0.8079936504364014),\n",
       "   (u'oops_concept', 0.8062907457351685),\n",
       "   (u'bootstrap', 0.8028784394264221),\n",
       "   (u'senior_web_developer', 0.7993264198303223),\n",
       "   (u'webdeveloper', 0.7979726791381836)],\n",
       "  [(u'ruby', 0.7712863087654114),\n",
       "   (u'perl', 0.7577237486839294),\n",
       "   (u'relational_databases', 0.7391830086708069),\n",
       "   (u'test_frameworks', 0.734647274017334),\n",
       "   (u'map_reduce', 0.7312558889389038),\n",
       "   (u'devop', 0.7296180725097656),\n",
       "   (u'kinesis', 0.7290434837341309),\n",
       "   (u'bigquery', 0.7273452281951904),\n",
       "   (u'administrations', 0.7257336378097534),\n",
       "   (u'python_programming/scripting_language', 0.7253988981246948)],\n",
       "  [(u'xhtml', 0.7922053337097168),\n",
       "   (u'html5', 0.790575385093689),\n",
       "   (u'javascript', 0.7776061296463013),\n",
       "   (u'web_design', 0.771491289138794),\n",
       "   (u'ajax', 0.7691584825515747),\n",
       "   (u'css3', 0.7676531076431274),\n",
       "   (u'html_xhtml', 0.7560790777206421),\n",
       "   (u'my_sql', 0.754533052444458),\n",
       "   (u'web_tool', 0.7539231777191162),\n",
       "   (u'dreamweaver', 0.7496887445449829)],\n",
       "  [(u'css3', 0.9375057220458984),\n",
       "   (u'ui_developer', 0.9244715571403503),\n",
       "   (u'bootstrap', 0.9231293797492981),\n",
       "   (u'front__end', 0.9156382083892822),\n",
       "   (u'css_3', 0.9136257171630859),\n",
       "   (u'ui_engineer', 0.9069639444351196),\n",
       "   (u'responsive', 0.9055514931678772),\n",
       "   (u'html4', 0.9040734171867371),\n",
       "   (u'ui_lead', 0.9039298892021179),\n",
       "   (u'front_end_engineer', 0.9039292335510254)],\n",
       "  [(u'html5', 0.8562525510787964),\n",
       "   (u'css3', 0.8111556768417358),\n",
       "   (u'web_tool', 0.8096526861190796),\n",
       "   (u'ui_developer', 0.8071935176849365),\n",
       "   (u'bootstrap', 0.8052152395248413),\n",
       "   (u'senior_web_developer', 0.8026396036148071),\n",
       "   (u'mvc_frameworks', 0.7995671033859253),\n",
       "   (u'css_3', 0.7958999872207642),\n",
       "   (u'webdeveloper', 0.7954410314559937),\n",
       "   (u'ui_developers', 0.7929912209510803)],\n",
       "  [(u'node', 0.9219822883605957),\n",
       "   (u'bootstrap', 0.9215775728225708),\n",
       "   (u'backbone', 0.8942737579345703),\n",
       "   (u'react', 0.8819767832756042),\n",
       "   (u'ui_developer', 0.8729913234710693),\n",
       "   (u'html5', 0.8692876100540161),\n",
       "   (u'knockout', 0.8687394857406616),\n",
       "   (u'javascript_developer', 0.8667021989822388),\n",
       "   (u'bower', 0.8658115863800049),\n",
       "   (u'full_stack_developer', 0.8650622367858887)],\n",
       "  [(u'react', 0.9388986825942993),\n",
       "   (u'backbone', 0.9247055053710938),\n",
       "   (u'mean_stack', 0.9243057370185852),\n",
       "   (u'angular', 0.9219822883605957),\n",
       "   (u'mean', 0.9090983867645264),\n",
       "   (u'bower', 0.9059317111968994),\n",
       "   (u'full_stack_developer', 0.9024332165718079),\n",
       "   (u'npm', 0.9016261100769043),\n",
       "   (u'frontend', 0.9010946750640869),\n",
       "   (u'grunt', 0.8981549143791199)]],\n",
       " 'musthave_model': [[(u'data_mine', 0.9465877413749695),\n",
       "   (u'log_mine', 0.9132214188575745),\n",
       "   (u'parallel', 0.9121681451797485),\n",
       "   (u'algorithms', 0.9117794036865234),\n",
       "   (u'anomaly_detection', 0.9075648784637451),\n",
       "   (u'large_scale', 0.894648551940918),\n",
       "   (u'statistics', 0.8905445337295532),\n",
       "   (u'odesign', 0.887861430644989),\n",
       "   (u'mapreduce', 0.8788114190101624),\n",
       "   (u'recommendation', 0.8707009553909302)],\n",
       "  [(u'drupal', 0.9664486646652222),\n",
       "   (u'jquery', 0.9654090404510498),\n",
       "   (u'less', 0.9651084542274475),\n",
       "   (u'behaviour_drive', 0.965004563331604),\n",
       "   (u'bdd', 0.9646888971328735),\n",
       "   (u'sass', 0.9643263220787048),\n",
       "   (u'bootstrap', 0.9638028740882874),\n",
       "   (u'grunt', 0.9623281359672546),\n",
       "   (u'jasmine', 0.9620272517204285),\n",
       "   (u'mvw', 0.9599791765213013)],\n",
       "  [(u'drupal', 0.9664486646652222),\n",
       "   (u'jquery', 0.9654090404510498),\n",
       "   (u'less', 0.9651084542274475),\n",
       "   (u'behaviour_drive', 0.965004563331604),\n",
       "   (u'bdd', 0.9646888971328735),\n",
       "   (u'sass', 0.9643263220787048),\n",
       "   (u'bootstrap', 0.9638028740882874),\n",
       "   (u'grunt', 0.9623281359672546),\n",
       "   (u'jasmine', 0.9620272517204285),\n",
       "   (u'mvw', 0.9599791765213013)],\n",
       "  [(u'ruby_on_rail', 0.9440964460372925),\n",
       "   (u'ror', 0.9440122842788696),\n",
       "   (u'engineer_in_test', 0.9433156847953796),\n",
       "   (u'blackbox_test', 0.9289750456809998),\n",
       "   (u'80211', 0.9241687059402466),\n",
       "   (u'ieee_80211', 0.9238948225975037),\n",
       "   (u'omnipeek', 0.9238286018371582),\n",
       "   (u'https', 0.9230294227600098),\n",
       "   (u'single_player', 0.922428548336029),\n",
       "   (u'iperf', 0.9222717881202698)],\n",
       "  [(u'css', 0.9720334410667419),\n",
       "   (u'template', 0.968834638595581),\n",
       "   (u'adobe_experience_management_suite', 0.9643564820289612),\n",
       "   (u'cq', 0.9640279412269592),\n",
       "   (u'css3', 0.9639554023742676),\n",
       "   (u'xhtml', 0.9628666639328003),\n",
       "   (u'adobe_cq', 0.9583250284194946),\n",
       "   (u'aem', 0.9560385346412659),\n",
       "   (u'foundation', 0.955864429473877),\n",
       "   (u'css2', 0.9552405476570129)],\n",
       "  [(u'css3', 0.9702982902526855),\n",
       "   (u'aem', 0.9533524513244629),\n",
       "   (u'jquery', 0.9530197978019714),\n",
       "   (u'less', 0.9452440142631531),\n",
       "   (u'css', 0.9413914680480957),\n",
       "   (u'adobe_experience_management_suite', 0.9404764771461487),\n",
       "   (u'template', 0.940237283706665),\n",
       "   (u'angular', 0.9391735792160034),\n",
       "   (u'html', 0.9371805191040039),\n",
       "   (u'cq', 0.9347214698791504)],\n",
       "  [(u'html', 0.9720335006713867),\n",
       "   (u'jquery', 0.9490579962730408),\n",
       "   (u'javascript', 0.9436420798301697),\n",
       "   (u'html5', 0.9413914680480957),\n",
       "   (u'css3', 0.9393692016601562),\n",
       "   (u'template', 0.9392268657684326),\n",
       "   (u'adobe_experience_management_suite', 0.9372433423995972),\n",
       "   (u'aem', 0.936206579208374),\n",
       "   (u'cq', 0.9354153871536255),\n",
       "   (u'sass', 0.9351266622543335)],\n",
       "  [(u'knockout', 0.9931827783584595),\n",
       "   (u'backbone', 0.9923273324966431),\n",
       "   (u'ember', 0.9898574352264404),\n",
       "   (u'jquerry', 0.9885340929031372),\n",
       "   (u'spa', 0.9874025583267212),\n",
       "   (u'durandal', 0.9869586825370789),\n",
       "   (u'html/css', 0.986136794090271),\n",
       "   (u'three', 0.9853281378746033),\n",
       "   (u'foundation', 0.9839178323745728),\n",
       "   (u'npm', 0.9837029576301575)],\n",
       "  [(u'ember', 0.9777774810791016),\n",
       "   (u'etl', 0.9768304824829102),\n",
       "   (u'webgl', 0.9761102199554443),\n",
       "   (u'svg', 0.9758523106575012),\n",
       "   (u'responsive/adaptive_design', 0.9733733534812927),\n",
       "   (u'spa', 0.9723876118659973),\n",
       "   (u'backbone', 0.9715735912322998),\n",
       "   (u'npm', 0.9712855815887451),\n",
       "   (u'extract', 0.9701659679412842),\n",
       "   (u'three', 0.969876766204834)]]}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST_CASE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "TEST_CASE_TABLE = [ pd.DataFrame(TEST_CASE[t]) for t in TEST_CASE ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['musthave_dice_model', 'musthave_model', 'musthave_dice_naruki_model']"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ t for t in TEST_CASE ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(doctoral, 0.924271404743)</td>\n",
       "      <td>(postdoc, 0.921362996101)</td>\n",
       "      <td>(bigdata, 0.916629612446)</td>\n",
       "      <td>(doctor, 0.914591312408)</td>\n",
       "      <td>(hadoop, 0.904930412769)</td>\n",
       "      <td>(dsp, 0.903142571449)</td>\n",
       "      <td>(weka, 0.899703621864)</td>\n",
       "      <td>(data_science, 0.897103011608)</td>\n",
       "      <td>(hdfs, 0.891349434853)</td>\n",
       "      <td>(recommendation, 0.890492320061)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(css, 0.965053498745)</td>\n",
       "      <td>(css3, 0.957265913486)</td>\n",
       "      <td>(html5, 0.949856162071)</td>\n",
       "      <td>(prototype, 0.949148237705)</td>\n",
       "      <td>(react, 0.94149017334)</td>\n",
       "      <td>(html5/css3/javascript, 0.9389783144)</td>\n",
       "      <td>(html, 0.937718868256)</td>\n",
       "      <td>(user_journey, 0.933208525181)</td>\n",
       "      <td>(cascade_style_sheet, 0.92841565609)</td>\n",
       "      <td>(adobe_aem, 0.928062856197)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(css, 0.965053498745)</td>\n",
       "      <td>(css3, 0.957265913486)</td>\n",
       "      <td>(html5, 0.949856162071)</td>\n",
       "      <td>(prototype, 0.949148237705)</td>\n",
       "      <td>(react, 0.94149017334)</td>\n",
       "      <td>(html5/css3/javascript, 0.9389783144)</td>\n",
       "      <td>(html, 0.937718868256)</td>\n",
       "      <td>(user_journey, 0.933208525181)</td>\n",
       "      <td>(cascade_style_sheet, 0.92841565609)</td>\n",
       "      <td>(adobe_aem, 0.928062856197)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(perl, 0.945951700211)</td>\n",
       "      <td>(aws, 0.932041227818)</td>\n",
       "      <td>(tool, 0.93145185709)</td>\n",
       "      <td>(administrator, 0.917977929115)</td>\n",
       "      <td>(stream_audio, 0.910002827644)</td>\n",
       "      <td>(c/c++, 0.907448291779)</td>\n",
       "      <td>(hadoop, 0.905795931816)</td>\n",
       "      <td>(map_reduce, 0.905780076981)</td>\n",
       "      <td>(c, 0.902261018753)</td>\n",
       "      <td>(kinesis, 0.90194529295)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(css, 0.972095429897)</td>\n",
       "      <td>(ued, 0.966288208961)</td>\n",
       "      <td>(prototype, 0.960716664791)</td>\n",
       "      <td>(xd, 0.960151433945)</td>\n",
       "      <td>(uxpin, 0.958795309067)</td>\n",
       "      <td>(ui_designer, 0.956885993481)</td>\n",
       "      <td>(invision, 0.956589460373)</td>\n",
       "      <td>(template, 0.956371307373)</td>\n",
       "      <td>(user_experience_designer, 0.955363810062)</td>\n",
       "      <td>(framer, 0.954596757889)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(interaction, 0.972752273083)</td>\n",
       "      <td>(visual, 0.968463301659)</td>\n",
       "      <td>(frontends, 0.968219041824)</td>\n",
       "      <td>(user_experience_designer, 0.967073321342)</td>\n",
       "      <td>(quartz_composer, 0.96668189764)</td>\n",
       "      <td>(invision, 0.965562999249)</td>\n",
       "      <td>(css3, 0.964307188988)</td>\n",
       "      <td>(ui_design, 0.963813662529)</td>\n",
       "      <td>(react, 0.963174343109)</td>\n",
       "      <td>(xd, 0.962453901768)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(html, 0.972095429897)</td>\n",
       "      <td>(prototype, 0.968522727489)</td>\n",
       "      <td>(javascript, 0.965053498745)</td>\n",
       "      <td>(ued, 0.963989853859)</td>\n",
       "      <td>(invision, 0.961262583733)</td>\n",
       "      <td>(xd, 0.9608720541)</td>\n",
       "      <td>(css3, 0.960782408714)</td>\n",
       "      <td>(html5, 0.960750997066)</td>\n",
       "      <td>(quartz_composer, 0.960707306862)</td>\n",
       "      <td>(ui_designer, 0.959810853004)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(less, 0.982120513916)</td>\n",
       "      <td>(template, 0.975835263729)</td>\n",
       "      <td>(responsive, 0.970739722252)</td>\n",
       "      <td>(cs, 0.969339609146)</td>\n",
       "      <td>(npm, 0.968224406242)</td>\n",
       "      <td>(frontend, 0.96731364727)</td>\n",
       "      <td>(bower, 0.966849803925)</td>\n",
       "      <td>(dojo, 0.965426206589)</td>\n",
       "      <td>(load, 0.965268015862)</td>\n",
       "      <td>(three, 0.96525144577)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(npm, 0.986540019512)</td>\n",
       "      <td>(bower, 0.983767449856)</td>\n",
       "      <td>(adobe_aem, 0.983366787434)</td>\n",
       "      <td>(backbone, 0.983056306839)</td>\n",
       "      <td>(prototype, 0.981605648994)</td>\n",
       "      <td>(or_ruby_on_rail, 0.981309950352)</td>\n",
       "      <td>(3d_graphics, 0.981156587601)</td>\n",
       "      <td>(template, 0.98106443882)</td>\n",
       "      <td>(3d_engine, 0.980781197548)</td>\n",
       "      <td>(jsf, 0.980508267879)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               0                            1  \\\n",
       "0     (doctoral, 0.924271404743)    (postdoc, 0.921362996101)   \n",
       "1          (css, 0.965053498745)       (css3, 0.957265913486)   \n",
       "2          (css, 0.965053498745)       (css3, 0.957265913486)   \n",
       "3         (perl, 0.945951700211)        (aws, 0.932041227818)   \n",
       "4          (css, 0.972095429897)        (ued, 0.966288208961)   \n",
       "5  (interaction, 0.972752273083)     (visual, 0.968463301659)   \n",
       "6         (html, 0.972095429897)  (prototype, 0.968522727489)   \n",
       "7         (less, 0.982120513916)   (template, 0.975835263729)   \n",
       "8          (npm, 0.986540019512)      (bower, 0.983767449856)   \n",
       "\n",
       "                              2                                           3  \\\n",
       "0     (bigdata, 0.916629612446)                    (doctor, 0.914591312408)   \n",
       "1       (html5, 0.949856162071)                 (prototype, 0.949148237705)   \n",
       "2       (html5, 0.949856162071)                 (prototype, 0.949148237705)   \n",
       "3         (tool, 0.93145185709)             (administrator, 0.917977929115)   \n",
       "4   (prototype, 0.960716664791)                        (xd, 0.960151433945)   \n",
       "5   (frontends, 0.968219041824)  (user_experience_designer, 0.967073321342)   \n",
       "6  (javascript, 0.965053498745)                       (ued, 0.963989853859)   \n",
       "7  (responsive, 0.970739722252)                        (cs, 0.969339609146)   \n",
       "8   (adobe_aem, 0.983366787434)                  (backbone, 0.983056306839)   \n",
       "\n",
       "                                  4                                      5  \\\n",
       "0          (hadoop, 0.904930412769)                  (dsp, 0.903142571449)   \n",
       "1            (react, 0.94149017334)  (html5/css3/javascript, 0.9389783144)   \n",
       "2            (react, 0.94149017334)  (html5/css3/javascript, 0.9389783144)   \n",
       "3    (stream_audio, 0.910002827644)                (c/c++, 0.907448291779)   \n",
       "4           (uxpin, 0.958795309067)          (ui_designer, 0.956885993481)   \n",
       "5  (quartz_composer, 0.96668189764)             (invision, 0.965562999249)   \n",
       "6        (invision, 0.961262583733)                     (xd, 0.9608720541)   \n",
       "7             (npm, 0.968224406242)              (frontend, 0.96731364727)   \n",
       "8       (prototype, 0.981605648994)      (or_ruby_on_rail, 0.981309950352)   \n",
       "\n",
       "                               6                               7  \\\n",
       "0         (weka, 0.899703621864)  (data_science, 0.897103011608)   \n",
       "1         (html, 0.937718868256)  (user_journey, 0.933208525181)   \n",
       "2         (html, 0.937718868256)  (user_journey, 0.933208525181)   \n",
       "3       (hadoop, 0.905795931816)    (map_reduce, 0.905780076981)   \n",
       "4     (invision, 0.956589460373)      (template, 0.956371307373)   \n",
       "5         (css3, 0.964307188988)     (ui_design, 0.963813662529)   \n",
       "6         (css3, 0.960782408714)         (html5, 0.960750997066)   \n",
       "7        (bower, 0.966849803925)          (dojo, 0.965426206589)   \n",
       "8  (3d_graphics, 0.981156587601)       (template, 0.98106443882)   \n",
       "\n",
       "                                            8  \\\n",
       "0                      (hdfs, 0.891349434853)   \n",
       "1        (cascade_style_sheet, 0.92841565609)   \n",
       "2        (cascade_style_sheet, 0.92841565609)   \n",
       "3                         (c, 0.902261018753)   \n",
       "4  (user_experience_designer, 0.955363810062)   \n",
       "5                     (react, 0.963174343109)   \n",
       "6           (quartz_composer, 0.960707306862)   \n",
       "7                      (load, 0.965268015862)   \n",
       "8                 (3d_engine, 0.980781197548)   \n",
       "\n",
       "                                  9  \n",
       "0  (recommendation, 0.890492320061)  \n",
       "1       (adobe_aem, 0.928062856197)  \n",
       "2       (adobe_aem, 0.928062856197)  \n",
       "3          (kinesis, 0.90194529295)  \n",
       "4          (framer, 0.954596757889)  \n",
       "5              (xd, 0.962453901768)  \n",
       "6     (ui_designer, 0.959810853004)  \n",
       "7            (three, 0.96525144577)  \n",
       "8             (jsf, 0.980508267879)  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST_CASE_TABLE[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(data_mine, 0.946587741375)</td>\n",
       "      <td>(log_mine, 0.913221418858)</td>\n",
       "      <td>(parallel, 0.91216814518)</td>\n",
       "      <td>(algorithms, 0.911779403687)</td>\n",
       "      <td>(anomaly_detection, 0.907564878464)</td>\n",
       "      <td>(large_scale, 0.894648551941)</td>\n",
       "      <td>(statistics, 0.89054453373)</td>\n",
       "      <td>(odesign, 0.887861430645)</td>\n",
       "      <td>(mapreduce, 0.87881141901)</td>\n",
       "      <td>(recommendation, 0.870700955391)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(drupal, 0.966448664665)</td>\n",
       "      <td>(jquery, 0.965409040451)</td>\n",
       "      <td>(less, 0.965108454227)</td>\n",
       "      <td>(behaviour_drive, 0.965004563332)</td>\n",
       "      <td>(bdd, 0.964688897133)</td>\n",
       "      <td>(sass, 0.964326322079)</td>\n",
       "      <td>(bootstrap, 0.963802874088)</td>\n",
       "      <td>(grunt, 0.962328135967)</td>\n",
       "      <td>(jasmine, 0.96202725172)</td>\n",
       "      <td>(mvw, 0.959979176521)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(drupal, 0.966448664665)</td>\n",
       "      <td>(jquery, 0.965409040451)</td>\n",
       "      <td>(less, 0.965108454227)</td>\n",
       "      <td>(behaviour_drive, 0.965004563332)</td>\n",
       "      <td>(bdd, 0.964688897133)</td>\n",
       "      <td>(sass, 0.964326322079)</td>\n",
       "      <td>(bootstrap, 0.963802874088)</td>\n",
       "      <td>(grunt, 0.962328135967)</td>\n",
       "      <td>(jasmine, 0.96202725172)</td>\n",
       "      <td>(mvw, 0.959979176521)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(ruby_on_rail, 0.944096446037)</td>\n",
       "      <td>(ror, 0.944012284279)</td>\n",
       "      <td>(engineer_in_test, 0.943315684795)</td>\n",
       "      <td>(blackbox_test, 0.928975045681)</td>\n",
       "      <td>(80211, 0.92416870594)</td>\n",
       "      <td>(ieee_80211, 0.923894822598)</td>\n",
       "      <td>(omnipeek, 0.923828601837)</td>\n",
       "      <td>(https, 0.92302942276)</td>\n",
       "      <td>(single_player, 0.922428548336)</td>\n",
       "      <td>(iperf, 0.92227178812)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(css, 0.972033441067)</td>\n",
       "      <td>(template, 0.968834638596)</td>\n",
       "      <td>(adobe_experience_management_suite, 0.96435648...</td>\n",
       "      <td>(cq, 0.964027941227)</td>\n",
       "      <td>(css3, 0.963955402374)</td>\n",
       "      <td>(xhtml, 0.962866663933)</td>\n",
       "      <td>(adobe_cq, 0.958325028419)</td>\n",
       "      <td>(aem, 0.956038534641)</td>\n",
       "      <td>(foundation, 0.955864429474)</td>\n",
       "      <td>(css2, 0.955240547657)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(css3, 0.970298290253)</td>\n",
       "      <td>(aem, 0.953352451324)</td>\n",
       "      <td>(jquery, 0.953019797802)</td>\n",
       "      <td>(less, 0.945244014263)</td>\n",
       "      <td>(css, 0.941391468048)</td>\n",
       "      <td>(adobe_experience_management_suite, 0.94047647...</td>\n",
       "      <td>(template, 0.940237283707)</td>\n",
       "      <td>(angular, 0.939173579216)</td>\n",
       "      <td>(html, 0.937180519104)</td>\n",
       "      <td>(cq, 0.934721469879)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(html, 0.972033500671)</td>\n",
       "      <td>(jquery, 0.949057996273)</td>\n",
       "      <td>(javascript, 0.94364207983)</td>\n",
       "      <td>(html5, 0.941391468048)</td>\n",
       "      <td>(css3, 0.93936920166)</td>\n",
       "      <td>(template, 0.939226865768)</td>\n",
       "      <td>(adobe_experience_management_suite, 0.9372433424)</td>\n",
       "      <td>(aem, 0.936206579208)</td>\n",
       "      <td>(cq, 0.935415387154)</td>\n",
       "      <td>(sass, 0.935126662254)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(knockout, 0.993182778358)</td>\n",
       "      <td>(backbone, 0.992327332497)</td>\n",
       "      <td>(ember, 0.989857435226)</td>\n",
       "      <td>(jquerry, 0.988534092903)</td>\n",
       "      <td>(spa, 0.987402558327)</td>\n",
       "      <td>(durandal, 0.986958682537)</td>\n",
       "      <td>(html/css, 0.98613679409)</td>\n",
       "      <td>(three, 0.985328137875)</td>\n",
       "      <td>(foundation, 0.983917832375)</td>\n",
       "      <td>(npm, 0.98370295763)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(ember, 0.977777481079)</td>\n",
       "      <td>(etl, 0.976830482483)</td>\n",
       "      <td>(webgl, 0.976110219955)</td>\n",
       "      <td>(svg, 0.975852310658)</td>\n",
       "      <td>(responsive/adaptive_design, 0.973373353481)</td>\n",
       "      <td>(spa, 0.972387611866)</td>\n",
       "      <td>(backbone, 0.971573591232)</td>\n",
       "      <td>(npm, 0.971285581589)</td>\n",
       "      <td>(extract, 0.970165967941)</td>\n",
       "      <td>(three, 0.969876766205)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                0                           1  \\\n",
       "0     (data_mine, 0.946587741375)  (log_mine, 0.913221418858)   \n",
       "1        (drupal, 0.966448664665)    (jquery, 0.965409040451)   \n",
       "2        (drupal, 0.966448664665)    (jquery, 0.965409040451)   \n",
       "3  (ruby_on_rail, 0.944096446037)       (ror, 0.944012284279)   \n",
       "4           (css, 0.972033441067)  (template, 0.968834638596)   \n",
       "5          (css3, 0.970298290253)       (aem, 0.953352451324)   \n",
       "6          (html, 0.972033500671)    (jquery, 0.949057996273)   \n",
       "7      (knockout, 0.993182778358)  (backbone, 0.992327332497)   \n",
       "8         (ember, 0.977777481079)       (etl, 0.976830482483)   \n",
       "\n",
       "                                                   2  \\\n",
       "0                          (parallel, 0.91216814518)   \n",
       "1                             (less, 0.965108454227)   \n",
       "2                             (less, 0.965108454227)   \n",
       "3                 (engineer_in_test, 0.943315684795)   \n",
       "4  (adobe_experience_management_suite, 0.96435648...   \n",
       "5                           (jquery, 0.953019797802)   \n",
       "6                        (javascript, 0.94364207983)   \n",
       "7                            (ember, 0.989857435226)   \n",
       "8                            (webgl, 0.976110219955)   \n",
       "\n",
       "                                   3  \\\n",
       "0       (algorithms, 0.911779403687)   \n",
       "1  (behaviour_drive, 0.965004563332)   \n",
       "2  (behaviour_drive, 0.965004563332)   \n",
       "3    (blackbox_test, 0.928975045681)   \n",
       "4               (cq, 0.964027941227)   \n",
       "5             (less, 0.945244014263)   \n",
       "6            (html5, 0.941391468048)   \n",
       "7          (jquerry, 0.988534092903)   \n",
       "8              (svg, 0.975852310658)   \n",
       "\n",
       "                                              4  \\\n",
       "0           (anomaly_detection, 0.907564878464)   \n",
       "1                         (bdd, 0.964688897133)   \n",
       "2                         (bdd, 0.964688897133)   \n",
       "3                        (80211, 0.92416870594)   \n",
       "4                        (css3, 0.963955402374)   \n",
       "5                         (css, 0.941391468048)   \n",
       "6                         (css3, 0.93936920166)   \n",
       "7                         (spa, 0.987402558327)   \n",
       "8  (responsive/adaptive_design, 0.973373353481)   \n",
       "\n",
       "                                                   5  \\\n",
       "0                      (large_scale, 0.894648551941)   \n",
       "1                             (sass, 0.964326322079)   \n",
       "2                             (sass, 0.964326322079)   \n",
       "3                       (ieee_80211, 0.923894822598)   \n",
       "4                            (xhtml, 0.962866663933)   \n",
       "5  (adobe_experience_management_suite, 0.94047647...   \n",
       "6                         (template, 0.939226865768)   \n",
       "7                         (durandal, 0.986958682537)   \n",
       "8                              (spa, 0.972387611866)   \n",
       "\n",
       "                                                   6  \\\n",
       "0                        (statistics, 0.89054453373)   \n",
       "1                        (bootstrap, 0.963802874088)   \n",
       "2                        (bootstrap, 0.963802874088)   \n",
       "3                         (omnipeek, 0.923828601837)   \n",
       "4                         (adobe_cq, 0.958325028419)   \n",
       "5                         (template, 0.940237283707)   \n",
       "6  (adobe_experience_management_suite, 0.9372433424)   \n",
       "7                          (html/css, 0.98613679409)   \n",
       "8                         (backbone, 0.971573591232)   \n",
       "\n",
       "                           7                                8  \\\n",
       "0  (odesign, 0.887861430645)       (mapreduce, 0.87881141901)   \n",
       "1    (grunt, 0.962328135967)         (jasmine, 0.96202725172)   \n",
       "2    (grunt, 0.962328135967)         (jasmine, 0.96202725172)   \n",
       "3     (https, 0.92302942276)  (single_player, 0.922428548336)   \n",
       "4      (aem, 0.956038534641)     (foundation, 0.955864429474)   \n",
       "5  (angular, 0.939173579216)           (html, 0.937180519104)   \n",
       "6      (aem, 0.936206579208)             (cq, 0.935415387154)   \n",
       "7    (three, 0.985328137875)     (foundation, 0.983917832375)   \n",
       "8      (npm, 0.971285581589)        (extract, 0.970165967941)   \n",
       "\n",
       "                                  9  \n",
       "0  (recommendation, 0.870700955391)  \n",
       "1             (mvw, 0.959979176521)  \n",
       "2             (mvw, 0.959979176521)  \n",
       "3            (iperf, 0.92227178812)  \n",
       "4            (css2, 0.955240547657)  \n",
       "5              (cq, 0.934721469879)  \n",
       "6            (sass, 0.935126662254)  \n",
       "7              (npm, 0.98370295763)  \n",
       "8           (three, 0.969876766205)  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST_CASE_TABLE[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(data_scientist, 0.907000541687)</td>\n",
       "      <td>(nlp, 0.893116116524)</td>\n",
       "      <td>(anomaly_detection, 0.871943593025)</td>\n",
       "      <td>(deep_learn, 0.870284318924)</td>\n",
       "      <td>(random_forest, 0.862106204033)</td>\n",
       "      <td>(recommendation, 0.862098574638)</td>\n",
       "      <td>(scala_language, 0.857527852058)</td>\n",
       "      <td>(k__mean, 0.854613780975)</td>\n",
       "      <td>(tensorflow, 0.853815555573)</td>\n",
       "      <td>(time_series_analysis, 0.85380089283)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(html5, 0.846334934235)</td>\n",
       "      <td>(ajax, 0.837237775326)</td>\n",
       "      <td>(css3, 0.83621609211)</td>\n",
       "      <td>(mvc_architecture, 0.816482722759)</td>\n",
       "      <td>(mvc_frameworks, 0.808582544327)</td>\n",
       "      <td>(web_tool, 0.807993650436)</td>\n",
       "      <td>(oops_concept, 0.806290745735)</td>\n",
       "      <td>(bootstrap, 0.802878439426)</td>\n",
       "      <td>(senior_web_developer, 0.79932641983)</td>\n",
       "      <td>(webdeveloper, 0.797972679138)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(html5, 0.846334934235)</td>\n",
       "      <td>(ajax, 0.837237775326)</td>\n",
       "      <td>(css3, 0.83621609211)</td>\n",
       "      <td>(mvc_architecture, 0.816482722759)</td>\n",
       "      <td>(mvc_frameworks, 0.808582544327)</td>\n",
       "      <td>(web_tool, 0.807993650436)</td>\n",
       "      <td>(oops_concept, 0.806290745735)</td>\n",
       "      <td>(bootstrap, 0.802878439426)</td>\n",
       "      <td>(senior_web_developer, 0.79932641983)</td>\n",
       "      <td>(webdeveloper, 0.797972679138)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(ruby, 0.771286308765)</td>\n",
       "      <td>(perl, 0.757723748684)</td>\n",
       "      <td>(relational_databases, 0.739183008671)</td>\n",
       "      <td>(test_frameworks, 0.734647274017)</td>\n",
       "      <td>(map_reduce, 0.731255888939)</td>\n",
       "      <td>(devop, 0.72961807251)</td>\n",
       "      <td>(kinesis, 0.729043483734)</td>\n",
       "      <td>(bigquery, 0.727345228195)</td>\n",
       "      <td>(administrations, 0.72573363781)</td>\n",
       "      <td>(python_programming/scripting_language, 0.7253...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(xhtml, 0.79220533371)</td>\n",
       "      <td>(html5, 0.790575385094)</td>\n",
       "      <td>(javascript, 0.777606129646)</td>\n",
       "      <td>(web_design, 0.771491289139)</td>\n",
       "      <td>(ajax, 0.769158482552)</td>\n",
       "      <td>(css3, 0.767653107643)</td>\n",
       "      <td>(html_xhtml, 0.756079077721)</td>\n",
       "      <td>(my_sql, 0.754533052444)</td>\n",
       "      <td>(web_tool, 0.753923177719)</td>\n",
       "      <td>(dreamweaver, 0.749688744545)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(css3, 0.937505722046)</td>\n",
       "      <td>(ui_developer, 0.92447155714)</td>\n",
       "      <td>(bootstrap, 0.923129379749)</td>\n",
       "      <td>(front__end, 0.915638208389)</td>\n",
       "      <td>(css_3, 0.913625717163)</td>\n",
       "      <td>(ui_engineer, 0.906963944435)</td>\n",
       "      <td>(responsive, 0.905551493168)</td>\n",
       "      <td>(html4, 0.904073417187)</td>\n",
       "      <td>(ui_lead, 0.903929889202)</td>\n",
       "      <td>(front_end_engineer, 0.903929233551)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(html5, 0.856252551079)</td>\n",
       "      <td>(css3, 0.811155676842)</td>\n",
       "      <td>(web_tool, 0.809652686119)</td>\n",
       "      <td>(ui_developer, 0.807193517685)</td>\n",
       "      <td>(bootstrap, 0.805215239525)</td>\n",
       "      <td>(senior_web_developer, 0.802639603615)</td>\n",
       "      <td>(mvc_frameworks, 0.799567103386)</td>\n",
       "      <td>(css_3, 0.795899987221)</td>\n",
       "      <td>(webdeveloper, 0.795441031456)</td>\n",
       "      <td>(ui_developers, 0.792991220951)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(node, 0.921982288361)</td>\n",
       "      <td>(bootstrap, 0.921577572823)</td>\n",
       "      <td>(backbone, 0.894273757935)</td>\n",
       "      <td>(react, 0.881976783276)</td>\n",
       "      <td>(ui_developer, 0.872991323471)</td>\n",
       "      <td>(html5, 0.869287610054)</td>\n",
       "      <td>(knockout, 0.868739485741)</td>\n",
       "      <td>(javascript_developer, 0.866702198982)</td>\n",
       "      <td>(bower, 0.86581158638)</td>\n",
       "      <td>(full_stack_developer, 0.865062236786)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(react, 0.938898682594)</td>\n",
       "      <td>(backbone, 0.924705505371)</td>\n",
       "      <td>(mean_stack, 0.924305737019)</td>\n",
       "      <td>(angular, 0.921982288361)</td>\n",
       "      <td>(mean, 0.909098386765)</td>\n",
       "      <td>(bower, 0.905931711197)</td>\n",
       "      <td>(full_stack_developer, 0.902433216572)</td>\n",
       "      <td>(npm, 0.901626110077)</td>\n",
       "      <td>(frontend, 0.901094675064)</td>\n",
       "      <td>(grunt, 0.898154914379)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  0                              1  \\\n",
       "0  (data_scientist, 0.907000541687)          (nlp, 0.893116116524)   \n",
       "1           (html5, 0.846334934235)         (ajax, 0.837237775326)   \n",
       "2           (html5, 0.846334934235)         (ajax, 0.837237775326)   \n",
       "3            (ruby, 0.771286308765)         (perl, 0.757723748684)   \n",
       "4            (xhtml, 0.79220533371)        (html5, 0.790575385094)   \n",
       "5            (css3, 0.937505722046)  (ui_developer, 0.92447155714)   \n",
       "6           (html5, 0.856252551079)         (css3, 0.811155676842)   \n",
       "7            (node, 0.921982288361)    (bootstrap, 0.921577572823)   \n",
       "8           (react, 0.938898682594)     (backbone, 0.924705505371)   \n",
       "\n",
       "                                        2                                   3  \\\n",
       "0     (anomaly_detection, 0.871943593025)        (deep_learn, 0.870284318924)   \n",
       "1                   (css3, 0.83621609211)  (mvc_architecture, 0.816482722759)   \n",
       "2                   (css3, 0.83621609211)  (mvc_architecture, 0.816482722759)   \n",
       "3  (relational_databases, 0.739183008671)   (test_frameworks, 0.734647274017)   \n",
       "4            (javascript, 0.777606129646)        (web_design, 0.771491289139)   \n",
       "5             (bootstrap, 0.923129379749)        (front__end, 0.915638208389)   \n",
       "6              (web_tool, 0.809652686119)      (ui_developer, 0.807193517685)   \n",
       "7              (backbone, 0.894273757935)             (react, 0.881976783276)   \n",
       "8            (mean_stack, 0.924305737019)           (angular, 0.921982288361)   \n",
       "\n",
       "                                  4                                       5  \\\n",
       "0   (random_forest, 0.862106204033)        (recommendation, 0.862098574638)   \n",
       "1  (mvc_frameworks, 0.808582544327)              (web_tool, 0.807993650436)   \n",
       "2  (mvc_frameworks, 0.808582544327)              (web_tool, 0.807993650436)   \n",
       "3      (map_reduce, 0.731255888939)                  (devop, 0.72961807251)   \n",
       "4            (ajax, 0.769158482552)                  (css3, 0.767653107643)   \n",
       "5           (css_3, 0.913625717163)           (ui_engineer, 0.906963944435)   \n",
       "6       (bootstrap, 0.805215239525)  (senior_web_developer, 0.802639603615)   \n",
       "7    (ui_developer, 0.872991323471)                 (html5, 0.869287610054)   \n",
       "8            (mean, 0.909098386765)                 (bower, 0.905931711197)   \n",
       "\n",
       "                                        6  \\\n",
       "0        (scala_language, 0.857527852058)   \n",
       "1          (oops_concept, 0.806290745735)   \n",
       "2          (oops_concept, 0.806290745735)   \n",
       "3               (kinesis, 0.729043483734)   \n",
       "4            (html_xhtml, 0.756079077721)   \n",
       "5            (responsive, 0.905551493168)   \n",
       "6        (mvc_frameworks, 0.799567103386)   \n",
       "7              (knockout, 0.868739485741)   \n",
       "8  (full_stack_developer, 0.902433216572)   \n",
       "\n",
       "                                        7  \\\n",
       "0               (k__mean, 0.854613780975)   \n",
       "1             (bootstrap, 0.802878439426)   \n",
       "2             (bootstrap, 0.802878439426)   \n",
       "3              (bigquery, 0.727345228195)   \n",
       "4                (my_sql, 0.754533052444)   \n",
       "5                 (html4, 0.904073417187)   \n",
       "6                 (css_3, 0.795899987221)   \n",
       "7  (javascript_developer, 0.866702198982)   \n",
       "8                   (npm, 0.901626110077)   \n",
       "\n",
       "                                       8  \\\n",
       "0           (tensorflow, 0.853815555573)   \n",
       "1  (senior_web_developer, 0.79932641983)   \n",
       "2  (senior_web_developer, 0.79932641983)   \n",
       "3       (administrations, 0.72573363781)   \n",
       "4             (web_tool, 0.753923177719)   \n",
       "5              (ui_lead, 0.903929889202)   \n",
       "6         (webdeveloper, 0.795441031456)   \n",
       "7                 (bower, 0.86581158638)   \n",
       "8             (frontend, 0.901094675064)   \n",
       "\n",
       "                                                   9  \n",
       "0              (time_series_analysis, 0.85380089283)  \n",
       "1                     (webdeveloper, 0.797972679138)  \n",
       "2                     (webdeveloper, 0.797972679138)  \n",
       "3  (python_programming/scripting_language, 0.7253...  \n",
       "4                      (dreamweaver, 0.749688744545)  \n",
       "5               (front_end_engineer, 0.903929233551)  \n",
       "6                    (ui_developers, 0.792991220951)  \n",
       "7             (full_stack_developer, 0.865062236786)  \n",
       "8                            (grunt, 0.898154914379)  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST_CASE_TABLE[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-07 10:36:32,347 : INFO : storing 23483x300 projection weights into musthave_dice_naruki.model\n"
     ]
    }
   ],
   "source": [
    "musthave_dice_naruki_model.wv.save_word2vec_format(\"musthave_dice_naruki.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 146332\r\n",
      "-rw-r--r-- 1 root root     1069 13:19 17 Th04 LICENSE\r\n",
      "-rw-r--r-- 1 root root 16656175 10:24  7 Th05 musthave_dice_naruki_100d.model\r\n",
      "-rw-r--r-- 1 root root 33111241 10:24  7 Th05 musthave_dice_naruki_200d.model\r\n",
      "-rw-r--r-- 1 root root 67163623 10:36  7 Th05 musthave_dice_naruki.model\r\n",
      "-rw-r--r-- 1 root root  8885546 15:44 16 Th04 mustHaveSkills-2.csv\r\n",
      "-rw-r--r-- 1 root root 18240737 11:17 17 Th04 naukri_skill_full\r\n",
      "-rw-r--r-- 1 root root  1276232 15:57 16 Th04 prepare_data.ipynb\r\n",
      "-rw-r--r-- 1 root root  3972850 15:54 16 Th04 prep_data_tokens_underscore_1\r\n",
      "-rw-r--r-- 1 root root       42 13:19 17 Th04 README.md\r\n",
      "-rw-r--r-- 1 root root   139130 13:53 17 Th04 skill2vec_ver2.ipynb\r\n",
      "-rw-r--r-- 1 root root    32300 10:33  7 Th05 skill2vec_ver3.ipynb\r\n",
      "drwxr-xr-x 2 root root       46 13:19 17 Th04 test\r\n",
      "-rw-r--r-- 1 root root   246074 08:58 20 Th04 tfidf.ipynb\r\n",
      "-rw-r--r-- 1 root root        0 10:04 18 Th04 untitled.txt\r\n",
      "drwxr-xr-x 2 root root       63 13:19 17 Th04 word2vec_skills\r\n",
      "-rw-r--r-- 1 root root    77333 15:55 16 Th04 word2vec_skills.ipynb\r\n",
      "-rw-r--r-- 1 root root    12329 16:33 16 Th04 word2vec_skills_nltk.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "!ls -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-07 10:36:37,044 : INFO : collecting all words and their counts\n",
      "2017-05-07 10:36:37,046 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-05-07 10:36:37,214 : INFO : PROGRESS: at sentence #10000, processed 216479 words, keeping 11609 word types\n",
      "2017-05-07 10:36:37,266 : INFO : PROGRESS: at sentence #20000, processed 316511 words, keeping 21557 word types\n",
      "2017-05-07 10:36:37,310 : INFO : PROGRESS: at sentence #30000, processed 403006 words, keeping 30521 word types\n",
      "2017-05-07 10:36:37,358 : INFO : PROGRESS: at sentence #40000, processed 489603 words, keeping 38336 word types\n",
      "2017-05-07 10:36:37,407 : INFO : PROGRESS: at sentence #50000, processed 579752 words, keeping 45956 word types\n",
      "2017-05-07 10:36:37,447 : INFO : PROGRESS: at sentence #60000, processed 666564 words, keeping 52001 word types\n",
      "2017-05-07 10:36:37,493 : INFO : PROGRESS: at sentence #70000, processed 751300 words, keeping 58857 word types\n",
      "2017-05-07 10:36:37,533 : INFO : PROGRESS: at sentence #80000, processed 836205 words, keeping 65151 word types\n",
      "2017-05-07 10:36:37,572 : INFO : PROGRESS: at sentence #90000, processed 923058 words, keeping 70256 word types\n",
      "2017-05-07 10:36:37,611 : INFO : PROGRESS: at sentence #100000, processed 1013120 words, keeping 74899 word types\n",
      "2017-05-07 10:36:37,655 : INFO : PROGRESS: at sentence #110000, processed 1099320 words, keeping 80588 word types\n",
      "2017-05-07 10:36:37,660 : INFO : collected 80695 word types from a corpus of 1105249 raw words and 110660 sentences\n",
      "2017-05-07 10:36:37,660 : INFO : Loading a fresh vocabulary\n",
      "2017-05-07 10:36:37,752 : INFO : min_count=5 retains 17151 unique words (21% of original 80695, drops 63544)\n",
      "2017-05-07 10:36:37,753 : INFO : min_count=5 leaves 1017848 word corpus (92% of original 1105249, drops 87401)\n",
      "2017-05-07 10:36:37,807 : INFO : deleting the raw counts dictionary of 80695 items\n",
      "2017-05-07 10:36:37,810 : INFO : sample=0.001 downsamples 22 most-common words\n",
      "2017-05-07 10:36:37,811 : INFO : downsampling leaves estimated 992315 word corpus (97.5% of prior 1017848)\n",
      "2017-05-07 10:36:37,811 : INFO : estimated required memory for 17151 words and 100 dimensions: 22296300 bytes\n",
      "2017-05-07 10:36:37,861 : INFO : resetting layer weights\n",
      "2017-05-07 10:36:38,086 : INFO : training model with 32 workers on 17151 vocabulary and 100 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-05-07 10:36:39,108 : INFO : PROGRESS: at 12.29% examples, 565331 words/s, in_qsize 56, out_qsize 2\n",
      "2017-05-07 10:36:40,140 : INFO : PROGRESS: at 28.29% examples, 606725 words/s, in_qsize 57, out_qsize 0\n",
      "2017-05-07 10:36:41,186 : INFO : PROGRESS: at 50.33% examples, 655919 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:36:42,194 : INFO : PROGRESS: at 69.13% examples, 678792 words/s, in_qsize 62, out_qsize 2\n",
      "2017-05-07 10:36:43,203 : INFO : PROGRESS: at 85.38% examples, 681587 words/s, in_qsize 57, out_qsize 0\n",
      "2017-05-07 10:36:43,564 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-05-07 10:36:43,601 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-05-07 10:36:43,605 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-05-07 10:36:43,607 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-05-07 10:36:43,608 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-05-07 10:36:43,609 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-05-07 10:36:43,610 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-05-07 10:36:43,611 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-05-07 10:36:43,612 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-05-07 10:36:43,614 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-05-07 10:36:43,615 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-05-07 10:36:43,616 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-05-07 10:36:43,617 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-05-07 10:36:43,619 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-05-07 10:36:43,620 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-05-07 10:36:43,621 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-05-07 10:36:43,622 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-05-07 10:36:43,623 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-05-07 10:36:43,632 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-05-07 10:36:43,652 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-05-07 10:36:43,653 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-05-07 10:36:43,661 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-05-07 10:36:43,666 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-05-07 10:36:43,672 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-05-07 10:36:43,679 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-05-07 10:36:43,682 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-07 10:36:43,683 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-07 10:36:43,690 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-07 10:36:43,694 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-07 10:36:43,699 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-07 10:36:43,707 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-07 10:36:43,715 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-07 10:36:43,716 : INFO : training on 4420996 raw words (3968872 effective words) took 5.6s, 707103 effective words/s\n",
      "2017-05-07 10:36:43,730 : INFO : collecting all words and their counts\n",
      "2017-05-07 10:36:43,731 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-05-07 10:36:43,828 : INFO : PROGRESS: at sentence #10000, processed 216479 words, keeping 11609 word types\n",
      "2017-05-07 10:36:43,874 : INFO : PROGRESS: at sentence #20000, processed 316511 words, keeping 21557 word types\n",
      "2017-05-07 10:36:43,914 : INFO : PROGRESS: at sentence #30000, processed 403006 words, keeping 30521 word types\n",
      "2017-05-07 10:36:43,952 : INFO : PROGRESS: at sentence #40000, processed 489603 words, keeping 38336 word types\n",
      "2017-05-07 10:36:43,993 : INFO : PROGRESS: at sentence #50000, processed 579752 words, keeping 45956 word types\n",
      "2017-05-07 10:36:44,030 : INFO : PROGRESS: at sentence #60000, processed 666564 words, keeping 52001 word types\n",
      "2017-05-07 10:36:44,067 : INFO : PROGRESS: at sentence #70000, processed 751300 words, keeping 58857 word types\n",
      "2017-05-07 10:36:44,106 : INFO : PROGRESS: at sentence #80000, processed 836205 words, keeping 65151 word types\n",
      "2017-05-07 10:36:44,141 : INFO : PROGRESS: at sentence #90000, processed 923058 words, keeping 70256 word types\n",
      "2017-05-07 10:36:44,183 : INFO : PROGRESS: at sentence #100000, processed 1013120 words, keeping 74899 word types\n",
      "2017-05-07 10:36:44,222 : INFO : PROGRESS: at sentence #110000, processed 1099320 words, keeping 80588 word types\n",
      "2017-05-07 10:36:44,226 : INFO : collected 80695 word types from a corpus of 1105249 raw words and 110660 sentences\n",
      "2017-05-07 10:36:44,227 : INFO : Loading a fresh vocabulary\n",
      "2017-05-07 10:36:44,302 : INFO : min_count=5 retains 17151 unique words (21% of original 80695, drops 63544)\n",
      "2017-05-07 10:36:44,302 : INFO : min_count=5 leaves 1017848 word corpus (92% of original 1105249, drops 87401)\n",
      "2017-05-07 10:36:44,345 : INFO : deleting the raw counts dictionary of 80695 items\n",
      "2017-05-07 10:36:44,347 : INFO : sample=0.001 downsamples 22 most-common words\n",
      "2017-05-07 10:36:44,348 : INFO : downsampling leaves estimated 992315 word corpus (97.5% of prior 1017848)\n",
      "2017-05-07 10:36:44,349 : INFO : estimated required memory for 17151 words and 200 dimensions: 36017100 bytes\n",
      "2017-05-07 10:36:44,397 : INFO : resetting layer weights\n",
      "2017-05-07 10:36:44,643 : INFO : training model with 32 workers on 17151 vocabulary and 200 features, using sg=1 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-05-07 10:36:45,679 : INFO : PROGRESS: at 3.24% examples, 233738 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:36:46,688 : INFO : PROGRESS: at 17.64% examples, 364186 words/s, in_qsize 22, out_qsize 1\n",
      "2017-05-07 10:36:47,731 : INFO : PROGRESS: at 27.17% examples, 376961 words/s, in_qsize 63, out_qsize 0\n",
      "2017-05-07 10:36:48,761 : INFO : PROGRESS: at 42.73% examples, 422811 words/s, in_qsize 63, out_qsize 0\n",
      "2017-05-07 10:36:49,763 : INFO : PROGRESS: at 50.94% examples, 409228 words/s, in_qsize 63, out_qsize 0\n",
      "2017-05-07 10:36:50,769 : INFO : PROGRESS: at 63.32% examples, 422677 words/s, in_qsize 64, out_qsize 0\n",
      "2017-05-07 10:36:51,812 : INFO : PROGRESS: at 75.27% examples, 420165 words/s, in_qsize 63, out_qsize 0\n",
      "2017-05-07 10:36:52,842 : INFO : PROGRESS: at 87.36% examples, 433001 words/s, in_qsize 49, out_qsize 0\n",
      "2017-05-07 10:36:53,146 : INFO : worker thread finished; awaiting finish of 31 more threads\n",
      "2017-05-07 10:36:53,184 : INFO : worker thread finished; awaiting finish of 30 more threads\n",
      "2017-05-07 10:36:53,268 : INFO : worker thread finished; awaiting finish of 29 more threads\n",
      "2017-05-07 10:36:53,279 : INFO : worker thread finished; awaiting finish of 28 more threads\n",
      "2017-05-07 10:36:53,284 : INFO : worker thread finished; awaiting finish of 27 more threads\n",
      "2017-05-07 10:36:53,287 : INFO : worker thread finished; awaiting finish of 26 more threads\n",
      "2017-05-07 10:36:53,289 : INFO : worker thread finished; awaiting finish of 25 more threads\n",
      "2017-05-07 10:36:53,290 : INFO : worker thread finished; awaiting finish of 24 more threads\n",
      "2017-05-07 10:36:53,293 : INFO : worker thread finished; awaiting finish of 23 more threads\n",
      "2017-05-07 10:36:53,296 : INFO : worker thread finished; awaiting finish of 22 more threads\n",
      "2017-05-07 10:36:53,299 : INFO : worker thread finished; awaiting finish of 21 more threads\n",
      "2017-05-07 10:36:53,307 : INFO : worker thread finished; awaiting finish of 20 more threads\n",
      "2017-05-07 10:36:53,331 : INFO : worker thread finished; awaiting finish of 19 more threads\n",
      "2017-05-07 10:36:53,334 : INFO : worker thread finished; awaiting finish of 18 more threads\n",
      "2017-05-07 10:36:53,341 : INFO : worker thread finished; awaiting finish of 17 more threads\n",
      "2017-05-07 10:36:53,382 : INFO : worker thread finished; awaiting finish of 16 more threads\n",
      "2017-05-07 10:36:53,400 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-05-07 10:36:53,409 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-05-07 10:36:53,420 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-05-07 10:36:53,435 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-05-07 10:36:53,446 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-05-07 10:36:53,454 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-05-07 10:36:53,455 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-05-07 10:36:53,458 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-05-07 10:36:53,459 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-05-07 10:36:53,467 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-05-07 10:36:53,477 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-05-07 10:36:53,480 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-05-07 10:36:53,483 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-05-07 10:36:53,494 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-05-07 10:36:53,496 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-05-07 10:36:53,504 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-05-07 10:36:53,504 : INFO : training on 4420996 raw words (3969228 effective words) took 8.9s, 448392 effective words/s\n"
     ]
    }
   ],
   "source": [
    "musthave_dice_naruki_model_100d = word2vec.Word2Vec(musthave_dice_naruki_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=100, \n",
    "          min_count=5, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )\n",
    "\n",
    "musthave_dice_naruki_model_200d = word2vec.Word2Vec(musthave_dice_naruki_data, \n",
    "          workers=multiprocessing.cpu_count(),  # Number of threads to run in parallel\n",
    "          size=200, \n",
    "          min_count=5, \n",
    "          window=10, \n",
    "          sample = 1e-3,  # Downsample setting for frequent words\n",
    "          iter=4,\n",
    "          sg =1\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-05-07 10:36:53,535 : INFO : storing 17151x100 projection weights into musthave_dice_naruki_100d.model\n",
      "2017-05-07 10:36:54,597 : INFO : storing 17151x200 projection weights into musthave_dice_naruki_200d.model\n"
     ]
    }
   ],
   "source": [
    "musthave_dice_naruki_model_100d.wv.save_word2vec_format(\"musthave_dice_naruki_100d.model\")\n",
    "musthave_dice_naruki_model_200d.wv.save_word2vec_format(\"musthave_dice_naruki_200d.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 145924\n",
      "-rw-r--r-- 1 root root     1069 13:19 17 Th04 LICENSE\n",
      "-rw-r--r-- 1 root root 16517779 10:36  7 Th05 musthave_dice_naruki_100d.model\n",
      "-rw-r--r-- 1 root root 32832888 10:36  7 Th05 musthave_dice_naruki_200d.model\n",
      "-rw-r--r-- 1 root root 67163623 10:36  7 Th05 musthave_dice_naruki.model\n",
      "-rw-r--r-- 1 root root  8885546 15:44 16 Th04 mustHaveSkills-2.csv\n",
      "-rw-r--r-- 1 root root 18240737 11:17 17 Th04 naukri_skill_full\n",
      "-rw-r--r-- 1 root root  1276232 15:57 16 Th04 prepare_data.ipynb\n",
      "-rw-r--r-- 1 root root  3972850 15:54 16 Th04 prep_data_tokens_underscore_1\n",
      "-rw-r--r-- 1 root root       42 13:19 17 Th04 README.md\n",
      "-rw-r--r-- 1 root root   139130 13:53 17 Th04 skill2vec_ver2.ipynb\n",
      "-rw-r--r-- 1 root root    32300 10:33  7 Th05 skill2vec_ver3.ipynb\n",
      "drwxr-xr-x 2 root root       46 13:19 17 Th04 test\n",
      "-rw-r--r-- 1 root root   246074 08:58 20 Th04 tfidf.ipynb\n",
      "-rw-r--r-- 1 root root        0 10:04 18 Th04 untitled.txt\n",
      "drwxr-xr-x 2 root root       63 13:19 17 Th04 word2vec_skills\n",
      "-rw-r--r-- 1 root root    77333 15:55 16 Th04 word2vec_skills.ipynb\n",
      "-rw-r--r-- 1 root root    12329 16:33 16 Th04 word2vec_skills_nltk.ipynb\n",
      "              total        used        free      shared  buff/cache   available\n",
      "Mem:         128673       11659      108258          43        8755      116236\n",
      "Swap:             0           0           0\n"
     ]
    }
   ],
   "source": [
    "!ls -l\n",
    "!free -m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'html5', 0.8463349342346191),\n",
       " (u'ajax', 0.8372377753257751),\n",
       " (u'css3', 0.8362160921096802),\n",
       " (u'mvc_architecture', 0.8164827227592468),\n",
       " (u'mvc_frameworks', 0.8085825443267822),\n",
       " (u'web_tool', 0.8079936504364014),\n",
       " (u'oops_concept', 0.8062907457351685),\n",
       " (u'bootstrap', 0.8028784394264221),\n",
       " (u'senior_web_developer', 0.7993264198303223),\n",
       " (u'webdeveloper', 0.7979726791381836)]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model.wv.similar_by_word(skill_transform('javascript'),topn=10)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'css3', 0.9307963252067566),\n",
       " (u'bootstrap', 0.9234713912010193),\n",
       " (u'ui_developer', 0.9192823171615601),\n",
       " (u'front_end_engineer', 0.9077576398849487),\n",
       " (u'front__end', 0.9058327674865723),\n",
       " (u'css_3', 0.9039251208305359),\n",
       " (u'ui_engineer', 0.9014566540718079),\n",
       " (u'adobe_experience_management_suite', 0.9006956815719604),\n",
       " (u'html4', 0.9003456234931946),\n",
       " (u'user_interface_developer', 0.8981920480728149)]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model_200d.wv.similar_by_word(skill_transform('html5'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'visual_design', 0.8798125982284546),\n",
       " (u'user_interface_design', 0.8789986371994019),\n",
       " (u'ux_design', 0.8746244311332703),\n",
       " (u'ui', 0.8447457551956177),\n",
       " (u'interaction_designer', 0.8409931063652039),\n",
       " (u'interaction_design', 0.8397772908210754),\n",
       " (u'user_interaction', 0.8348618745803833),\n",
       " (u'wireframe', 0.8340160250663757),\n",
       " (u'transform', 0.8149587512016296),\n",
       " (u'mobile_design', 0.8148149251937866)]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# [(u'css3', 0.9307963252067566),\n",
    "#  (u'bootstrap', 0.9234713912010193),\n",
    "#  (u'ui_developer', 0.9192823171615601),\n",
    "#  (u'front_end_engineer', 0.9077576398849487),\n",
    "#  (u'front__end', 0.9058327674865723),\n",
    "#  (u'css_3', 0.9039251208305359),\n",
    "#  (u'ui_engineer', 0.9014566540718079),\n",
    "#  (u'adobe_experience_management_suite', 0.9006956815719604),\n",
    "#  (u'html4', 0.9003456234931946),\n",
    "#  (u'user_interface_developer', 0.8981920480728149)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'html5', 0.8463349342346191),\n",
       " (u'ajax', 0.8372377753257751),\n",
       " (u'css3', 0.8362160921096802),\n",
       " (u'mvc_architecture', 0.8164827227592468),\n",
       " (u'mvc_frameworks', 0.8085825443267822),\n",
       " (u'web_tool', 0.8079936504364014),\n",
       " (u'oops_concept', 0.8062907457351685),\n",
       " (u'bootstrap', 0.8028784394264221),\n",
       " (u'senior_web_developer', 0.7993264198303223),\n",
       " (u'webdeveloper', 0.7979726791381836)]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model.similar_by_word(skill_transform('javascript'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'react', 0.9388986825942993),\n",
       " (u'backbone', 0.9247055053710938),\n",
       " (u'mean_stack', 0.9243057370185852),\n",
       " (u'angular', 0.9219822883605957),\n",
       " (u'mean', 0.9090983867645264),\n",
       " (u'bower', 0.9059317111968994),\n",
       " (u'full_stack_developer', 0.9024332165718079),\n",
       " (u'npm', 0.9016261100769043),\n",
       " (u'frontend', 0.9010946750640869),\n",
       " (u'grunt', 0.8981549143791199),\n",
       " (u'css/javascript', 0.8971441984176636),\n",
       " (u'gulp', 0.8954569697380066),\n",
       " (u'bootstrap', 0.8946527242660522),\n",
       " (u'mongodb', 0.894255518913269),\n",
       " (u'javascript_developer', 0.8926291465759277),\n",
       " (u'mean_stack_developer', 0.8926271200180054),\n",
       " (u'mongo_db', 0.8904428482055664),\n",
       " (u'front_end_web_developer', 0.8894308805465698),\n",
       " (u'service_workers', 0.8887876272201538),\n",
       " (u'ui_architect', 0.8884862661361694),\n",
       " (u'less', 0.888136088848114),\n",
       " (u'loopback', 0.8872900009155273),\n",
       " (u'firebase', 0.8807092905044556),\n",
       " (u'mocha', 0.8802285194396973),\n",
       " (u'meanstack', 0.880145788192749),\n",
       " (u'front__end', 0.8791353106498718),\n",
       " (u'javascript_mvc', 0.8780139684677124),\n",
       " (u'front__end_developer', 0.8763250708580017),\n",
       " (u'ruby_rail', 0.875716507434845),\n",
       " (u'redux', 0.8757087588310242),\n",
       " (u'frontend_developer', 0.8744759559631348),\n",
       " (u'progressive_web_apps', 0.8743990659713745),\n",
       " (u'responsive_web', 0.8740561008453369),\n",
       " (u'ui_frameworks', 0.8734986782073975),\n",
       " (u'scss', 0.8733119964599609),\n",
       " (u'ui_developement', 0.8728358149528503),\n",
       " (u'oo_javascript', 0.8728337287902832),\n",
       " (u'yeoman', 0.8724195957183838),\n",
       " (u'extension', 0.8723046779632568),\n",
       " (u'backend_developer', 0.8721400499343872),\n",
       " (u'java_ui', 0.8712367415428162),\n",
       " (u'ui_developer', 0.8705049157142639),\n",
       " (u'full_stack', 0.8704302906990051),\n",
       " (u'api_design', 0.8697232007980347),\n",
       " (u'require', 0.8695825338363647),\n",
       " (u'mongo_database', 0.8692529201507568),\n",
       " (u'express', 0.8689918518066406),\n",
       " (u'front_end_engineer', 0.8679870963096619),\n",
       " (u'ionic', 0.8674670457839966),\n",
       " (u'html5_apis', 0.8669593334197998),\n",
       " (u'rest_api', 0.8667384386062622),\n",
       " (u'vanilla', 0.8662528991699219),\n",
       " (u'javascript_frameworks', 0.8661893606185913),\n",
       " (u'front_end_developer', 0.8660576343536377),\n",
       " (u'angula', 0.8660332560539246),\n",
       " (u'socketio', 0.8656952381134033),\n",
       " (u'web_sockets', 0.865507185459137),\n",
       " (u'knockout', 0.8648263216018677),\n",
       " (u'fullstack', 0.8647933006286621),\n",
       " (u'ractive', 0.8647328615188599),\n",
       " (u'handlebar', 0.8638430833816528),\n",
       " (u'ember', 0.8634741306304932),\n",
       " (u'karma', 0.8630586862564087),\n",
       " (u'userinterface', 0.8628602027893066),\n",
       " (u'transform', 0.8626048564910889),\n",
       " (u'aurelia', 0.8621917963027954),\n",
       " (u'anguler', 0.8620923757553101),\n",
       " (u'vue', 0.8616383671760559),\n",
       " (u'mustache', 0.8614273071289062),\n",
       " (u'mock', 0.8613691329956055),\n",
       " (u'three', 0.8611224889755249),\n",
       " (u'ruby_on_rail_developer', 0.8604821562767029),\n",
       " (u'postcss', 0.8601441383361816),\n",
       " (u'durandal', 0.8589529991149902),\n",
       " (u'full_stack_software_developer', 0.8588634729385376),\n",
       " (u'single_page', 0.8583961725234985),\n",
       " (u'sass', 0.8582544922828674),\n",
       " (u'angular_', 0.8575234413146973),\n",
       " (u'highcharts', 0.8572196364402771),\n",
       " (u'phython', 0.856558084487915),\n",
       " (u'rubyonrails', 0.8562724590301514),\n",
       " (u'server_side', 0.8562562465667725),\n",
       " (u'type_script', 0.8559668064117432),\n",
       " (u'fullstack_developer', 0.8557930588722229),\n",
       " (u'backend_engineer', 0.8557524085044861),\n",
       " (u'rabbit_mq', 0.8551300764083862),\n",
       " (u'protractor', 0.8548111319541931),\n",
       " (u'pixi', 0.853692889213562),\n",
       " (u'postgre_sql', 0.8536221385002136),\n",
       " (u'lamp_stack', 0.8534302711486816),\n",
       " (u'jasmine', 0.8533531427383423),\n",
       " (u'web_engineer', 0.8526561260223389),\n",
       " (u'anjular', 0.8521955013275146),\n",
       " (u'ui_engineer', 0.8517796993255615),\n",
       " (u'java_ui_developer', 0.8506346940994263),\n",
       " (u'reactjs_developer', 0.8504369854927063),\n",
       " (u'css/sass', 0.8501026630401611),\n",
       " (u'responsive', 0.8494596481323242),\n",
       " (u'frontends', 0.8493925929069519),\n",
       " (u'user_interface_developer', 0.8493410348892212),\n",
       " (u'mvc_frameworks', 0.8492969870567322),\n",
       " (u'ruby_rail_developer', 0.8491718769073486),\n",
       " (u'ux_developer', 0.8489862680435181),\n",
       " (u'jqueryui', 0.8482282161712646),\n",
       " (u'full__stack', 0.8481844067573547),\n",
       " (u'es6', 0.8477033972740173),\n",
       " (u'odoo', 0.8470433950424194),\n",
       " (u'restful_webservices', 0.8469909429550171),\n",
       " (u'oo', 0.8467534184455872),\n",
       " (u'behaviour_drive', 0.8467332124710083),\n",
       " (u'spring_data', 0.8464245200157166),\n",
       " (u'nosql_db', 0.8459915518760681),\n",
       " (u'technology_lead', 0.8453523516654968),\n",
       " (u'adobe_experience_management_suite', 0.8451828360557556),\n",
       " (u'django', 0.8448779582977295),\n",
       " (u'restful_api', 0.8447527885437012),\n",
       " (u'ccs3', 0.8447158932685852),\n",
       " (u'domain_drive_design', 0.8445805311203003),\n",
       " (u'marine_corps', 0.8445587754249573),\n",
       " (u'ui_specialist', 0.8444175720214844),\n",
       " (u'dynamodb', 0.8443212509155273),\n",
       " (u'data_source', 0.8435825109481812),\n",
       " (u'ror', 0.8434613943099976),\n",
       " (u'bootstrap_3', 0.8434313535690308),\n",
       " (u'memcache', 0.8433147072792053),\n",
       " (u'fontend', 0.8432695865631104),\n",
       " (u'phonegap_developer', 0.8432431221008301),\n",
       " (u'back_end_developer', 0.8430757522583008),\n",
       " (u'java_j2ee_developer', 0.8429073095321655),\n",
       " (u'css5', 0.8427987098693848),\n",
       " (u'build_tool', 0.84251868724823),\n",
       " (u'ui_developers', 0.8424967527389526),\n",
       " (u'javascrip', 0.8422414064407349),\n",
       " (u'cordova', 0.8421887159347534),\n",
       " (u'js_angular', 0.8419628143310547),\n",
       " (u'mysql_db', 0.8415170311927795),\n",
       " (u'or_ruby_on_rail', 0.8413493633270264),\n",
       " (u'j_script', 0.8411810398101807),\n",
       " (u'php_architect', 0.8409843444824219),\n",
       " (u'full_stack_web_developer', 0.840727686882019),\n",
       " (u'rich_internet_applications', 0.8406217098236084),\n",
       " (u'aws_service', 0.8397541046142578),\n",
       " (u'hybrid_apps', 0.8394625186920166),\n",
       " (u'hybrid_mobile', 0.8392165899276733),\n",
       " (u'api_security', 0.8390585780143738),\n",
       " (u'yii2', 0.8388251066207886),\n",
       " (u'bs/ms_in_computer_science', 0.8387699127197266),\n",
       " (u'restful_web_service', 0.838039219379425),\n",
       " (u'sap_ui5', 0.8376845717430115),\n",
       " (u'aws_sdk', 0.8376805782318115),\n",
       " (u'ror_developer', 0.8376057744026184),\n",
       " (u'cascade_style_sheet', 0.837382972240448),\n",
       " (u'mcv', 0.8373094797134399),\n",
       " (u'html5', 0.8372753262519836),\n",
       " (u'compass', 0.8370662331581116),\n",
       " (u'sql_lite', 0.8370129466056824),\n",
       " (u'full_stack_engineer', 0.8368926644325256),\n",
       " (u'object_oriented', 0.8368857502937317),\n",
       " (u'hybrid_mobile_developer', 0.8361007571220398),\n",
       " (u'microservices', 0.8359015583992004),\n",
       " (u'object_relational_mapper', 0.8358584642410278),\n",
       " (u'mvw', 0.8358190059661865),\n",
       " (u'git_hub', 0.8356602787971497),\n",
       " (u'markup_languages', 0.8355462551116943),\n",
       " (u'serverside', 0.8354949951171875),\n",
       " (u'html5/css3/javascript', 0.8354354500770569),\n",
       " (u'curl', 0.8351383209228516),\n",
       " (u'rest_apis', 0.8351231813430786),\n",
       " (u'air_national_guard', 0.8348999619483948),\n",
       " (u'css3', 0.8347866535186768),\n",
       " (u'full_stack_software_engineer', 0.8342705368995667),\n",
       " (u'us_military', 0.8338086605072021),\n",
       " (u'no_sql', 0.8336770534515381),\n",
       " (u'net_angular', 0.8335667848587036),\n",
       " (u'ui_lead', 0.8332997560501099),\n",
       " (u'hybrid_applications', 0.8321267366409302),\n",
       " (u'vs_2010', 0.8321205377578735),\n",
       " (u'opensource', 0.8316771984100342),\n",
       " (u'twitter_bootstrap', 0.831123948097229),\n",
       " (u'css2', 0.8309839963912964),\n",
       " (u'webdeveloper', 0.830945611000061),\n",
       " (u'reactos', 0.8305810689926147),\n",
       " (u'ext_', 0.8301624655723572),\n",
       " (u'client_side', 0.8300262689590454),\n",
       " (u'websockets', 0.8299471139907837),\n",
       " (u'ui_architecture', 0.829939603805542),\n",
       " (u'cloudbased', 0.8299084901809692),\n",
       " (u'javscript', 0.8298561573028564),\n",
       " (u'mvcs', 0.8293628692626953),\n",
       " (u'web_ux_design', 0.8290427923202515),\n",
       " (u'paypal', 0.8289146423339844),\n",
       " (u'gui_developer', 0.8288257122039795),\n",
       " (u'react_native_developer', 0.8283089399337769),\n",
       " (u'jav', 0.8281422853469849),\n",
       " (u'responsive/adaptive_design', 0.8280612230300903),\n",
       " (u'veteran', 0.8278810977935791),\n",
       " (u'ux_lead', 0.8275170922279358),\n",
       " (u'test_drive', 0.8269290924072266),\n",
       " (u'yui', 0.8268765211105347),\n",
       " (u'web_ui_developer', 0.8268682956695557),\n",
       " (u'amazon_kinesis', 0.8263774514198303),\n",
       " (u'css_30', 0.8259981870651245),\n",
       " (u'sr_software_engineer', 0.8256813287734985),\n",
       " (u'meteor', 0.8255735635757446),\n",
       " (u'or', 0.8255723714828491),\n",
       " (u'laravel_developer', 0.8255006074905396),\n",
       " (u'feed', 0.8253114223480225),\n",
       " (u'nosql_database', 0.8251196146011353),\n",
       " (u'jquerymobile', 0.8250206708908081),\n",
       " (u'php_laravel', 0.824870228767395),\n",
       " (u'game_pc', 0.8246759176254272),\n",
       " (u'html4', 0.8242911100387573),\n",
       " (u'mongo', 0.8241134285926819),\n",
       " (u'boostrap', 0.8238417506217957),\n",
       " (u'cloudfront', 0.8236427903175354),\n",
       " (u'oauth2', 0.8234416842460632),\n",
       " (u'extract', 0.8229417204856873),\n",
       " (u'object', 0.822860062122345),\n",
       " (u'bdd', 0.8226581811904907),\n",
       " (u'ruby_on_rail', 0.8226355314254761),\n",
       " (u'web_scrap', 0.8224507570266724),\n",
       " (u'phone_gap', 0.8223198652267456),\n",
       " (u'cq', 0.8222280740737915),\n",
       " (u'php_web', 0.8220747709274292),\n",
       " (u'cq5_developer', 0.8219730257987976),\n",
       " (u'js_framworks', 0.8217878341674805),\n",
       " (u'design_architecture', 0.8215535879135132),\n",
       " (u'ojavascript', 0.8214572668075562),\n",
       " (u'restful_service', 0.8208134174346924),\n",
       " (u'redis', 0.8207763433456421),\n",
       " (u'coast_guard', 0.8207489848136902),\n",
       " (u'elastic_search', 0.8207370042800903),\n",
       " (u'nosql_databases', 0.8205721378326416),\n",
       " (u'management_experience', 0.8204604387283325),\n",
       " (u'php_engineer', 0.8204389810562134),\n",
       " (u'google_api', 0.8202437162399292),\n",
       " (u'magento_2', 0.820149302482605),\n",
       " (u'web_skills', 0.8195937871932983),\n",
       " (u'playstore', 0.8194507360458374),\n",
       " (u'celery', 0.8193916082382202),\n",
       " (u'senior_software_developer', 0.81935054063797),\n",
       " (u'cloudfoundry', 0.8191690444946289),\n",
       " (u'data_algorithms', 0.8191555738449097),\n",
       " (u'full_stack_java_developer', 0.8190546035766602),\n",
       " (u'consumer_brand', 0.8190178871154785),\n",
       " (u'scalatest', 0.8189647197723389),\n",
       " (u'kendo', 0.8188273906707764),\n",
       " (u'magento_architect', 0.8186784386634827),\n",
       " (u'devtool', 0.8184027671813965),\n",
       " (u'web_tool', 0.818313479423523),\n",
       " (u'experience_with_selenium', 0.8178790211677551),\n",
       " (u'stream_technology', 0.8178411722183228),\n",
       " (u'typescript', 0.8178083300590515),\n",
       " (u'ajex', 0.8175716400146484),\n",
       " (u'amazon_cloud', 0.8174740076065063),\n",
       " (u'cors', 0.8174416422843933),\n",
       " (u'javascript_cod', 0.8167667388916016),\n",
       " (u'stream_protocol', 0.8167084455490112),\n",
       " (u'stash', 0.8166012763977051),\n",
       " (u'html/css', 0.8165947198867798),\n",
       " (u'laravel5', 0.8165886998176575),\n",
       " (u'insightdb', 0.8164784908294678),\n",
       " (u'asf', 0.8164197206497192),\n",
       " (u'nosql', 0.8163759708404541),\n",
       " (u'drupal_architect', 0.8161864280700684),\n",
       " (u'angualr', 0.816053032875061),\n",
       " (u'responsive_web_design', 0.8160463571548462),\n",
       " (u'html_developer', 0.8157740831375122),\n",
       " (u'adobe_aem', 0.8157471418380737),\n",
       " (u'senior_php_developer', 0.8157331347465515),\n",
       " (u'principal_architect', 0.8155913352966309),\n",
       " (u'senior_developer', 0.8154444098472595),\n",
       " (u'gae', 0.8154246807098389),\n",
       " (u'uml_tool', 0.8154241442680359),\n",
       " (u'mobile_analytics', 0.8151251673698425),\n",
       " (u'php_lead', 0.8150041699409485),\n",
       " (u'drupal_architecture', 0.8149550557136536),\n",
       " (u'java_spring', 0.8148590326309204),\n",
       " (u'technology_project', 0.8146462440490723),\n",
       " (u'mango', 0.8146111965179443),\n",
       " (u'user_journey', 0.8145466446876526),\n",
       " (u'senior_software_engineer', 0.8145338892936707),\n",
       " (u'hybrid_mobile_apps', 0.814515233039856),\n",
       " (u'no__sql', 0.81451416015625),\n",
       " (u'js_frameworks', 0.8145068287849426),\n",
       " (u'postgres', 0.8143495321273804),\n",
       " (u'spring_developer', 0.8142673969268799),\n",
       " (u'head_technology', 0.8141673803329468),\n",
       " (u'php_frameworks', 0.8141087889671326),\n",
       " (u'principal_engineer', 0.8140419125556946),\n",
       " (u'springmvc', 0.8139918446540833),\n",
       " (u'open_source_technology', 0.8139768838882446),\n",
       " (u'kony', 0.8136768937110901),\n",
       " (u'aws_cloud', 0.8135985136032104),\n",
       " (u'bluemix', 0.8134545087814331),\n",
       " (u'flux', 0.8131818771362305),\n",
       " (u'lead_developer', 0.813156008720398),\n",
       " (u'internal_apps', 0.813065767288208),\n",
       " (u'software_engineer_manager', 0.8130375146865845),\n",
       " (u'kendoui', 0.8127765655517578)]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model.similar_by_word(skill_transform('node_js'), 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'react', 0.9410043358802795),\n",
       " (u'backbone', 0.9235603213310242),\n",
       " (u'angular', 0.9139069318771362),\n",
       " (u'mean_stack', 0.9107990860939026),\n",
       " (u'npm', 0.8981935977935791),\n",
       " (u'mean', 0.8961061239242554),\n",
       " (u'mongodb', 0.8947668075561523),\n",
       " (u'bower', 0.89299476146698),\n",
       " (u'gulp', 0.8898729085922241),\n",
       " (u'full_stack_developer', 0.8898143768310547),\n",
       " (u'less', 0.8879895806312561),\n",
       " (u'grunt', 0.8874234557151794),\n",
       " (u'bootstrap', 0.8813268542289734),\n",
       " (u'javascript_mvc', 0.8796409368515015),\n",
       " (u'ui_architect', 0.878745436668396),\n",
       " (u'express', 0.8783759474754333),\n",
       " (u'javascript_developer', 0.8779905438423157),\n",
       " (u'loopback', 0.8772644996643066),\n",
       " (u'scss', 0.876909077167511),\n",
       " (u'meanstack', 0.8767061233520508),\n",
       " (u'fullstack_developer', 0.8745248317718506),\n",
       " (u'frontend_developer', 0.8743304014205933),\n",
       " (u'oo_javascript', 0.8734292984008789),\n",
       " (u'frontend', 0.8726052641868591),\n",
       " (u'mean_stack_developer', 0.8722931146621704),\n",
       " (u'mocha', 0.8718781471252441),\n",
       " (u'ionic', 0.8692560195922852),\n",
       " (u'karma', 0.8691719770431519),\n",
       " (u'mongo_db', 0.869140088558197),\n",
       " (u'ruby_rail', 0.8685640096664429),\n",
       " (u'javascript_frameworks', 0.8681174516677856),\n",
       " (u'front__end_developer', 0.8658348321914673),\n",
       " (u'firebase', 0.8657660484313965),\n",
       " (u'front_end_engineer', 0.865490198135376),\n",
       " (u'web_sockets', 0.865413248538971),\n",
       " (u'front_end_web_developer', 0.8645513653755188),\n",
       " (u'ui_developer', 0.8639869689941406),\n",
       " (u'api_design', 0.8633240461349487),\n",
       " (u'fullstack', 0.8624975085258484),\n",
       " (u'responsive_web', 0.8604957461357117),\n",
       " (u'full_stack', 0.8601876497268677),\n",
       " (u'backend_developer', 0.8590860366821289),\n",
       " (u'ember', 0.858986496925354),\n",
       " (u'angular_', 0.8588980436325073),\n",
       " (u'single_page', 0.8582155704498291),\n",
       " (u'highcharts', 0.8563627004623413),\n",
       " (u'require', 0.856257975101471),\n",
       " (u'extension', 0.8553192019462585),\n",
       " (u'compass', 0.8549339771270752),\n",
       " (u'front__end', 0.8544724583625793),\n",
       " (u'sass', 0.8542588949203491),\n",
       " (u'durandal', 0.8531215190887451),\n",
       " (u'net_angular', 0.8520907759666443),\n",
       " (u'ror_developer', 0.8518674373626709),\n",
       " (u'rest_api', 0.8515506982803345),\n",
       " (u'ui_frameworks', 0.8505377769470215),\n",
       " (u'jasmine', 0.8499050140380859),\n",
       " (u'restful_webservices', 0.8487206697463989),\n",
       " (u'rabbit_mq', 0.8482152819633484),\n",
       " (u'mcv', 0.8479892611503601),\n",
       " (u'ruby_on_rail_developer', 0.8479012250900269),\n",
       " (u'three', 0.8476657867431641),\n",
       " (u'ui_engineer', 0.8473669290542603),\n",
       " (u'es6', 0.8466081619262695),\n",
       " (u'front_end_developer', 0.8456935286521912),\n",
       " (u'ui_developers', 0.8455706238746643),\n",
       " (u'transform', 0.8446633219718933),\n",
       " (u'backend_engineer', 0.8440500497817993),\n",
       " (u'twitter_bootstrap', 0.8437660932540894),\n",
       " (u'oo', 0.8437266945838928),\n",
       " (u'mustache', 0.8428046703338623),\n",
       " (u'server_side', 0.8427610397338867),\n",
       " (u'knockout', 0.8426931500434875),\n",
       " (u'veteran', 0.8412466049194336),\n",
       " (u'ruby_rail_developer', 0.8410531878471375),\n",
       " (u'marine_corps', 0.8409768342971802),\n",
       " (u'ccs3', 0.8406050801277161),\n",
       " (u'behaviour_drive', 0.8397578001022339),\n",
       " (u'rubyonrails', 0.8394588232040405),\n",
       " (u'technology_lead', 0.8387559652328491),\n",
       " (u'restful_api', 0.8387421369552612),\n",
       " (u'cordova', 0.8376701474189758),\n",
       " (u'responsive', 0.8371790647506714),\n",
       " (u'lamp_stack', 0.8371149301528931),\n",
       " (u'user_interface_developer', 0.8369914293289185),\n",
       " (u'django', 0.8368545770645142),\n",
       " (u'phython', 0.8365734815597534),\n",
       " (u'ror', 0.8360055685043335),\n",
       " (u'air_national_guard', 0.8355268239974976),\n",
       " (u'opensource', 0.8353366255760193),\n",
       " (u'java_ui_developer', 0.8345665335655212),\n",
       " (u'css5', 0.8340780735015869),\n",
       " (u'mvc_frameworks', 0.8340442776679993),\n",
       " (u'dynamodb', 0.8339343070983887),\n",
       " (u'websockets', 0.8332309126853943),\n",
       " (u'us_military', 0.8328179717063904),\n",
       " (u'full_stack_engineer', 0.832779586315155),\n",
       " (u'protractor', 0.8322723507881165),\n",
       " (u'postgre_sql', 0.8318290710449219),\n",
       " (u'html5/css3/javascript', 0.8316993117332458),\n",
       " (u'restful_web_service', 0.831649899482727),\n",
       " (u'coast_guard', 0.8311233520507812),\n",
       " (u'feed', 0.830828070640564),\n",
       " (u'rest_apis', 0.8300957679748535),\n",
       " (u'gui_developer', 0.8295877575874329),\n",
       " (u'ux_developer', 0.8295469284057617),\n",
       " (u'or_ruby_on_rail', 0.8285820484161377),\n",
       " (u'mysql_db', 0.8280737996101379),\n",
       " (u'hybrid_mobile_developer', 0.8280222415924072),\n",
       " (u'pixi', 0.8276479244232178),\n",
       " (u'build_tool', 0.8270093202590942),\n",
       " (u'frontends', 0.8267995119094849),\n",
       " (u'css_30', 0.8267189264297485),\n",
       " (u'redis', 0.8265978097915649),\n",
       " (u'curl', 0.8263528943061829),\n",
       " (u'html4', 0.8255170583724976),\n",
       " (u'nosql_db', 0.8250864148139954),\n",
       " (u'paypal', 0.8247868418693542),\n",
       " (u'webdeveloper', 0.8242667317390442),\n",
       " (u'web_tool', 0.8239617347717285),\n",
       " (u'data_algorithms', 0.8238608837127686),\n",
       " (u'serverside', 0.8236905336380005),\n",
       " (u'bdd', 0.8233084678649902),\n",
       " (u'memcache', 0.8231711387634277),\n",
       " (u'phone_gap', 0.8226145505905151),\n",
       " (u'ruby_on_rail', 0.821251392364502),\n",
       " (u'javascrip', 0.8205989599227905),\n",
       " (u'nosql_database', 0.8204118013381958),\n",
       " (u'database_knowledge', 0.8191187381744385),\n",
       " (u'client_side', 0.8188855648040771),\n",
       " (u'ui_lead', 0.8186686038970947),\n",
       " (u'back_end_developer', 0.8185954689979553),\n",
       " (u'html5', 0.8185789585113525),\n",
       " (u'cascade_style_sheet', 0.8184218406677246),\n",
       " (u'adobe_experience_management_suite', 0.8177361488342285),\n",
       " (u'principal_engineer', 0.8167839646339417),\n",
       " (u'mongo', 0.816455066204071),\n",
       " (u'object', 0.815758466720581),\n",
       " (u'meteor', 0.8157100677490234),\n",
       " (u'tdd', 0.8151615858078003),\n",
       " (u'sql_lite', 0.8151118755340576),\n",
       " (u'ms_azure', 0.8150578737258911),\n",
       " (u'restapi', 0.8144571781158447),\n",
       " (u'nosql_databases', 0.8143872022628784),\n",
       " (u'css2', 0.8143463730812073),\n",
       " (u'or', 0.8143424391746521),\n",
       " (u'stash', 0.814171314239502),\n",
       " (u'web_architect', 0.8131659030914307),\n",
       " (u'rabbitmq', 0.8120532035827637),\n",
       " (u'hybrid_mobile', 0.8119121193885803),\n",
       " (u'singleton', 0.8117045164108276),\n",
       " (u'bluemix', 0.8116636276245117),\n",
       " (u'web_ux_design', 0.8116440773010254),\n",
       " (u'test_drive', 0.8115922808647156),\n",
       " (u'hybrid_mobile_apps', 0.8111529350280762),\n",
       " (u'grails', 0.8101392984390259),\n",
       " (u'cors', 0.8100817203521729),\n",
       " (u'asf', 0.8100420236587524),\n",
       " (u'markup_languages', 0.809999942779541),\n",
       " (u'distributedsystems', 0.8099850416183472),\n",
       " (u'spring_boot', 0.8091616034507751),\n",
       " (u'cloudbased', 0.8090344667434692),\n",
       " (u'woocommerce', 0.8088372349739075),\n",
       " (u'twist', 0.8085389137268066),\n",
       " (u'odoo', 0.8083859086036682),\n",
       " (u'handlebars', 0.808367133140564),\n",
       " (u'responsive_web_design', 0.8081499338150024),\n",
       " (u'responsive/adaptive_design', 0.8079869151115417),\n",
       " (u'html_developer', 0.807793140411377),\n",
       " (u'amazon_cloud', 0.8074489831924438),\n",
       " (u'postgress', 0.8073288202285767),\n",
       " (u'no_sql', 0.8073278665542603),\n",
       " (u'hsts', 0.8071803450584412),\n",
       " (u'it_developer', 0.8069207072257996),\n",
       " (u'air_force', 0.8067172765731812),\n",
       " (u'extract', 0.8064073324203491),\n",
       " (u'aws_redshift', 0.8064067363739014),\n",
       " (u'senior_php_developer', 0.8055492639541626),\n",
       " (u'js_frameworks', 0.8052727580070496),\n",
       " (u'css3', 0.8050574064254761),\n",
       " (u'playstore', 0.8049640655517578),\n",
       " (u'postgres', 0.804912805557251),\n",
       " (u'play', 0.8044744729995728),\n",
       " (u'google_api', 0.8042485117912292),\n",
       " (u'or_design', 0.8036627769470215),\n",
       " (u'typescript', 0.8036567568778992),\n",
       " (u'3d_engine', 0.8034220933914185),\n",
       " (u'user_journey', 0.8029398918151855),\n",
       " (u'appstore', 0.8022555112838745),\n",
       " (u'restful_service', 0.8019024729728699),\n",
       " (u'php_mysql_ajax', 0.8013861179351807),\n",
       " (u'lead_developer', 0.8010302782058716),\n",
       " (u'senior_software_developer', 0.8009309768676758),\n",
       " (u'yui', 0.8007832765579224),\n",
       " (u'vs', 0.8000665903091431),\n",
       " (u'cloud_platform', 0.7998828887939453),\n",
       " (u'crawl', 0.7998439073562622),\n",
       " (u'magento_architect', 0.7995043992996216),\n",
       " (u'nosql', 0.7994401454925537),\n",
       " (u'route_53', 0.7992023229598999),\n",
       " (u'senior_software_engineer', 0.7990199327468872),\n",
       " (u'php_lead', 0.7989487648010254),\n",
       " (u'elastic_search', 0.7987798452377319),\n",
       " (u'artifactory', 0.7983718514442444),\n",
       " (u'model_view_controller', 0.797264575958252),\n",
       " (u'payment_solutions', 0.7971372008323669),\n",
       " (u'html_50', 0.7971060276031494),\n",
       " (u'microservices', 0.7967995405197144),\n",
       " (u'springmvc', 0.7963003516197205),\n",
       " (u'game_pc', 0.7958492040634155),\n",
       " (u'php5', 0.795848548412323),\n",
       " (u'riak', 0.795801043510437),\n",
       " (u'rest_web_service', 0.7955740690231323),\n",
       " (u'pyramid', 0.7955533266067505),\n",
       " (u'sde', 0.7954222559928894),\n",
       " (u'large_scale', 0.7949842214584351),\n",
       " (u'python_developer', 0.7948983907699585),\n",
       " (u'elk', 0.7947587966918945),\n",
       " (u'stream_protocol', 0.7945043444633484),\n",
       " (u'software_engineer_manager', 0.7937685251235962),\n",
       " (u'coaca', 0.7936199903488159),\n",
       " (u'scalable_distribute', 0.7933412790298462),\n",
       " (u'cq', 0.7930501699447632),\n",
       " (u'senior_web_developer', 0.7928750514984131),\n",
       " (u'software_kit', 0.7926671504974365),\n",
       " (u'sr_software_engineer', 0.7924463152885437),\n",
       " (u'mobile_ui', 0.792439341545105),\n",
       " (u'ux_design', 0.7917436957359314),\n",
       " (u'ui/ux', 0.7915641069412231),\n",
       " (u'canvas', 0.7915598154067993),\n",
       " (u'5', 0.7912713885307312),\n",
       " (u'code_ignitor', 0.7912570238113403),\n",
       " (u'objective', 0.791038990020752),\n",
       " (u'principal_software_engineer', 0.7904893159866333),\n",
       " (u'bigquery', 0.790488600730896),\n",
       " (u'react_native', 0.7904773354530334),\n",
       " (u'async', 0.7903505563735962),\n",
       " (u'codeigniter_developer', 0.7902258038520813),\n",
       " (u'adobe_aem', 0.7900496125221252),\n",
       " (u'codeignitor', 0.789922833442688),\n",
       " (u'ja', 0.7898610830307007),\n",
       " (u'springboot', 0.7897517681121826),\n",
       " (u'geodistributed', 0.7895814180374146),\n",
       " (u'micro_service', 0.7886703014373779),\n",
       " (u'html/css', 0.7885652184486389),\n",
       " (u'salesforces', 0.7884368896484375),\n",
       " (u'ext', 0.7882282733917236),\n",
       " (u'agile_software', 0.7881209850311279),\n",
       " (u'amazon_webservices', 0.7872484922409058),\n",
       " (u'scala', 0.7871466875076294),\n",
       " (u'plugin', 0.7866387367248535),\n",
       " (u'kendo_ui', 0.7865896224975586),\n",
       " (u'front_end', 0.7865813970565796),\n",
       " (u'corephp', 0.7863837480545044),\n",
       " (u'senior_developer', 0.786231517791748),\n",
       " (u'spring_frameworks', 0.7862160205841064),\n",
       " (u'open_source_technology', 0.7860170602798462),\n",
       " (u'hybrid_developer', 0.7858641147613525),\n",
       " (u'jss', 0.7857216000556946),\n",
       " (u'dell_boomi', 0.7852903604507446),\n",
       " (u'jee_engineer', 0.7851806879043579),\n",
       " (u'toolkits', 0.7851142883300781),\n",
       " (u'jasper', 0.7851141095161438),\n",
       " (u'jee_developer', 0.7849906086921692),\n",
       " (u'cloudformation', 0.7846865653991699),\n",
       " (u'or_nosql', 0.7845689058303833),\n",
       " (u'high_available', 0.7844181060791016),\n",
       " (u'drupal_developer', 0.7841536402702332),\n",
       " (u'kit', 0.783937931060791),\n",
       " (u'parse', 0.7835628986358643),\n",
       " (u'dojo', 0.7835083603858948),\n",
       " (u'kinesis', 0.7834018468856812),\n",
       " (u'aws_infrastructure', 0.7830744981765747),\n",
       " (u'solr', 0.782554030418396),\n",
       " (u'websocket', 0.7822335362434387),\n",
       " (u'css_3', 0.7821604013442993),\n",
       " (u'shopify', 0.7818537950515747),\n",
       " (u'native_ios', 0.7818351984024048),\n",
       " (u'h265', 0.7816279530525208),\n",
       " (u'software_applications', 0.7815268039703369),\n",
       " (u'plugins', 0.780738890171051),\n",
       " (u'php_5', 0.7806766033172607),\n",
       " (u'elastic', 0.7805345058441162),\n",
       " (u'aws_cloud', 0.7804417610168457),\n",
       " (u'deployments', 0.7803974151611328),\n",
       " (u'multitiered', 0.780363917350769),\n",
       " (u'j2ee_developer', 0.7802588939666748),\n",
       " (u'gedistributed', 0.7802466750144958),\n",
       " (u'quartz_composer', 0.7800343036651611),\n",
       " (u'symphony', 0.7798798680305481),\n",
       " (u'client_online', 0.7796136140823364),\n",
       " (u'graphical_user_interface', 0.7786712050437927),\n",
       " (u'frame', 0.7786129117012024),\n",
       " (u'java_engineer', 0.7785176038742065),\n",
       " (u'spock', 0.7784895300865173),\n",
       " (u'rest_service', 0.7784515023231506),\n",
       " (u'java_script', 0.7781447172164917),\n",
       " (u'memory_database', 0.7781147956848145),\n",
       " (u'cloudbase', 0.7779359817504883),\n",
       " (u'distribute_databases', 0.7778652906417847)]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musthave_dice_naruki_model_100d.similar_by_word(skill_transform('node_js'), 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'angular'"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(r'^angular.*$', 'angular', \"angular.js\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}