trenddit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import praw\n",
    "import nltk\n",
    "import pymongo\n",
    "import time\n",
    "import json\n",
    "import math\n",
    "from collections import defaultdict\n",
    "from operator import itemgetter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AccessDB():\n",
    "    def __init__(self):\n",
    "        self.client = pymongo.MongoClient()\n",
    "\n",
    "    def useDB(self,dbname):\n",
    "        self.db = self.client[dbname]\n",
    "\n",
    "    def createCollection(self,name):\n",
    "        try:\n",
    "            self.db.create_collection(name)\n",
    "            return True\n",
    "        except:\n",
    "            return False\n",
    "    \n",
    "    def insert(self,collection,query):\n",
    "        '''\n",
    "        para query :: dict\n",
    "        '''\n",
    "        #query = json.loads(query)\n",
    "        self.db[collection].insert_one(query)\n",
    "    \n",
    "    def update(self,collection,target,query):\n",
    "        \n",
    "        self.db[collection].update_one(target,query)\n",
    "    \n",
    "    def find(self, query = None, projection = None, collection = None):\n",
    "    \n",
    "        if collection:\n",
    "            documents = [i for i in self.db[collection].find(query, projection)]\n",
    "            return documents\n",
    "        \n",
    "        else:\n",
    "            for i in self.db.collection_names():\n",
    "                document = self.db[i].find(query, projection)\n",
    "                \n",
    "                for i in document:\n",
    "                    documents.append(i)\n",
    "            \n",
    "            return documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class controlData():\n",
    "    def __init__(self,db):\n",
    "        '''\n",
    "        parameter db is instance of AccessDB() \n",
    "        '''\n",
    "        self.DB = db\n",
    "    \n",
    "    def inputPosts(self, db, subreddit, posts):\n",
    "        \n",
    "        result = False\n",
    "        \n",
    "        if type(posts) is list:\n",
    "            for post in posts:\n",
    "                result = self.inputPost(db, subreddit, post)\n",
    "                \n",
    "            return result\n",
    "            \n",
    "        elif type(posts) is dict:\n",
    "            result = self.inputPost(db, subreddit, posts)\n",
    "            return result\n",
    "            \n",
    "        else:\n",
    "            return result\n",
    "            \n",
    "            \n",
    "            \n",
    "    def inputPost(self, db, subreddit, post):\n",
    "        self.DB.useDB(db)\n",
    "        \n",
    "        if post.has_key('id'):\n",
    "            id_query = {\"ID\" : post['id']}\n",
    "        elif post.has_key('ID'):\n",
    "            id_query = {\"ID\" : post['ID']}\n",
    "        else:\n",
    "            print \"has not key(id)\"\n",
    "            return False\n",
    "        \n",
    "        if not self.DB.find(query = id_query, collection = subreddit):\n",
    "            self.DB.insert(subreddit,id_query)\n",
    "        \n",
    "        for key in post.keys():\n",
    "            if key is 'id' or key is 'ID':\n",
    "                continue\n",
    "            \n",
    "            update_query = {'$set' : {key : post[key]}}\n",
    "            \n",
    "            self.DB.update(subreddit,id_query,update_query)\n",
    "            \n",
    "        return True\n",
    "    \n",
    "    def findPost(self, db, subreddit = None, query = None):\n",
    "        self.DB.useDB(db)\n",
    "        \n",
    "        return self.DB.find(query = query, collection = subreddit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Reddit():\n",
    "    def __init__(self, db = None):\n",
    "        '''\n",
    "        parameter db is instance of AccessDB() \n",
    "        '''\n",
    "        \n",
    "        client = open('client_key')\n",
    "        \n",
    "        self.reddit = praw.Reddit(client_id = client.readline(),\n",
    "                             client_secret = client.readline(),\n",
    "                             user_agent = 'my user agent')\n",
    "        \n",
    "        self.subreddits = ['programming',\n",
    "                           'gaming',\n",
    "                           'movies',\n",
    "                           'hacking',\n",
    "                           'worldnews',\n",
    "                           'futurology',\n",
    "                           'sports',\n",
    "                           'news',\n",
    "                           'music',\n",
    "                           'science',\n",
    "                           'technology',\n",
    "                           'politics',\n",
    "                           'nba',\n",
    "                           'soccer',\n",
    "                           'design'\n",
    "                          ]\n",
    "        if db:\n",
    "            self.db = controlData(db)\n",
    "            self.dbname = 'reddit'\n",
    "        \n",
    "    def split_date(self,date):\n",
    "        date = date.split(\":\")\n",
    "            \n",
    "        '''start_time = date[0] + \" 00:00\"\n",
    "        end_time = date[1]+\" 00:00\"\n",
    "            \n",
    "        start_time = time.mktime(time.strptime(start_time, '%Y%m%d %H:%M'))\n",
    "        end_time = time.mktime(time.strptime(end_time, '%Y%m%d %H:%M'))\n",
    "        '''\n",
    "        \n",
    "        start_time = str2stamp(date[0])\n",
    "        end_time = str2stamp(date[1])\n",
    "        \n",
    "        end_time = end_time + 86400\n",
    "        \n",
    "        return start_time,end_time\n",
    "    \n",
    "        \n",
    "    def request2dbinsert(self, date, subreddit = None):\n",
    "        '''\n",
    "        date를 1일 단위로 쪼개서 request 후 db에 넣음\n",
    "        '''\n",
    "        start_time,end_time = self.split_date(date)\n",
    "        \n",
    "        \n",
    "        s = start_time\n",
    "        while(s < end_time):\n",
    "            e = s + 86400\n",
    "                \n",
    "            request_time = stamp2str(s) + \":\" + stamp2str(e)\n",
    "            \n",
    "            print \"-\" * 10 + request_time + \" request \" + \"-\" * 10\n",
    "            \n",
    "            self.subreddits_request(request_subreddit = subreddit, date = request_time, parse = True, db = True)\n",
    "                \n",
    "            print \"-\" * 10 + request_time + \" complete \" + \"-\" * 10\n",
    "            \n",
    "            s += 86400\n",
    "    \n",
    "    def subreddits_request(self, request_subreddit = None, date=None, parse = False, db = False):\n",
    "        '''\n",
    "        self.subreddits에 있는 서브렛딧들을 한번에 요청\n",
    "        parse : parse 까지 실행\n",
    "        db : db insert까지 실행 (parse가 true 일때만)\n",
    "        '''\n",
    "        submissions = {}\n",
    "        \n",
    "        if request_subreddit:\n",
    "            if type(request_subreddit) is list:\n",
    "                subreddits = request_subreddit\n",
    "            else:\n",
    "                subreddits = [request_subreddit]\n",
    "                \n",
    "        else:\n",
    "            subreddits = self.subreddits\n",
    "            \n",
    "            \n",
    "        for subreddit in subreddits:\n",
    "            print subreddit + \" request\"\n",
    "            submissions[subreddit] = self.list_request(subreddit,date=date)\n",
    "            \n",
    "            if parse:\n",
    "                submissions[subreddit] = self.parse(submissions[subreddit])\n",
    "                \n",
    "                print subreddit + \" parse\"\n",
    "                \n",
    "                if db:\n",
    "                    self.db.inputPosts(self.dbname,subreddit,submissions[subreddit])\n",
    "                    \n",
    "                    print subreddit + \" insert to db \"\n",
    "                    \n",
    "                    submissions[subreddit] = True\n",
    "        \n",
    "        return submissions\n",
    "            \n",
    "        \n",
    "    def list_request(self,subreddit,num=None,date=None):\n",
    "        \n",
    "        if date:\n",
    "            start_time,end_time = self.split_date(date)\n",
    "            \n",
    "            print start_time\n",
    "            print end_time\n",
    "            \n",
    "            query=\"timestamp:\"+str(int(start_time))+\"..\"+str(int(end_time))\n",
    "            \n",
    "            \n",
    "            result = self.reddit.subreddit(subreddit).submissions(start_time, end_time)\n",
    "             \n",
    "            if not result:\n",
    "                raise last_exception\n",
    "        \n",
    "        else:\n",
    "            result = self.reddit.subreddit(subreddit).new(limit=num)\n",
    "            \n",
    "        return result\n",
    "        \n",
    "    \n",
    "    def parse(self,reddit_list):\n",
    "        posts = []\n",
    "        \n",
    "        time_start = int(time.time())\n",
    "        timeout=900\n",
    "        \n",
    "        while int(time.time()) < time_start + timeout:\n",
    "            try:\n",
    "\n",
    "                for submission in  reddit_list:\n",
    "                    comments = []\n",
    "                    post = {}\n",
    "\n",
    "                    post['id'] = submission.id\n",
    "                    post['title'] = submission.title\n",
    "                    post['date'] = submission.created\n",
    "                    post['vote'] = submission.score\n",
    "                    post['content'] = submission.selftext\n",
    "                    post['url'] = submission.url\n",
    "\n",
    "                    for temp_comment in list(submission.comments):\n",
    "\n",
    "                        comment = {}\n",
    "\n",
    "                        if not hasattr(temp_comment, 'body'):\n",
    "                            continue\n",
    "\n",
    "                        comment['id'] = temp_comment.id\n",
    "                        comment['content'] = temp_comment.body\n",
    "                        comment['date'] = temp_comment.created\n",
    "                        comment['vote'] = temp_comment.score\n",
    "\n",
    "                        comments.append(comment)\n",
    "\n",
    "                    post['comments'] = comments\n",
    "\n",
    "                    posts.append(post)\n",
    "                \n",
    "\n",
    "            except:\n",
    "                #wait for 30 seconds since sending more requests to overloaded server might not be helping\n",
    "                #last_exception = e\n",
    "                print \"*\" * 50\n",
    "                print \"error! waiting 30sec\"\n",
    "                print \"*\" * 50\n",
    "                time.sleep(30)\n",
    "            else:\n",
    "                break\n",
    "        \n",
    "        return posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Analyse():\n",
    "    '''def __init__(self,db):\n",
    "        self.DB = controlData(db)\n",
    "        \n",
    "    def get_posts(self,subreddit,date=None):\n",
    "        dbname = 'reddit'\n",
    "        posts = self.DB.findPost(dbname, collection = subreddit)\n",
    "        return posts\n",
    "        '''\n",
    "        \n",
    "        \n",
    "    def tokenize(self, text):\n",
    "        try:\n",
    "            tokens = nltk.word_tokenize(text)\n",
    "        except:\n",
    "            print text\n",
    "        tagged_tokens = nltk.pos_tag(tokens)\n",
    "\n",
    "        #parser_en = nltk.RegexpParser(\"NP: {<DT>?<JJ>?<NN.*>*}\")\n",
    "        #chunks_en = parser_en.parse(tagged_tokens)\n",
    "        #chunks_en.draw()\n",
    "\n",
    "        return tagged_tokens\n",
    "        \n",
    "    def posts_analyze(self, posts):\n",
    "        if type(posts) is list:\n",
    "            documents = []\n",
    "            \n",
    "            for post in posts:\n",
    "                documents.append(self.posts_analyze(post))\n",
    "            \n",
    "            return documents\n",
    "            \n",
    "        if type(posts) is dict:\n",
    "            document = {}\n",
    "            \n",
    "            document['ID'] = posts['ID']\n",
    "            title_analyse = self.tokenize(posts['title'])\n",
    "\n",
    "            document.update(self.choose_noun(title_analyse))\n",
    "\n",
    "            if posts['content']:\n",
    "                content_analyse = self.tokenize(posts['content'])\n",
    "\n",
    "                document.update(self.choose_noun(content_analyse))\n",
    "\n",
    "            if posts['comments']:\n",
    "                document['COMMENTS'] = []\n",
    "                \n",
    "                for comment in posts['comments']:\n",
    "                    comment_analyse = self.tokenize(comment['content'])\n",
    "        \n",
    "                        \n",
    "                    comment_noun = self.choose_noun(comment_analyse)\n",
    "                    comment_noun['ID'] = comment['id']\n",
    "\n",
    "                    document['COMMENTS'].append(comment_noun)\n",
    "            \n",
    "            return document\n",
    "        \n",
    "        \n",
    "        \n",
    "    def choose_noun(self,tagged_token):\n",
    "        noun_dict = {}\n",
    "        except_noun = [\".\", \"[\", \"]\", \">\", \"<\", \"/*\", \"*/\", \"*\", \"+\", \"-\", \"=\", \"%\"]\n",
    "        \n",
    "        for i in range(len(tagged_token)):\n",
    "            if \"NN\" in tagged_token[i][1]:\n",
    "                \n",
    "                noun = tagged_token[i][0].lower()\n",
    "                \n",
    "                if noun in except_noun or \".\" in noun:\n",
    "                    continue\n",
    "                \n",
    "                if noun in noun_dict.keys():\n",
    "                    noun_dict[noun] += 1\n",
    "                    \n",
    "                else:\n",
    "                    noun_dict[noun] = 1\n",
    "                    \n",
    "        if type(noun_dict) is int:\n",
    "            print tagged_token\n",
    "        return noun_dict\n",
    "        \n",
    "        \n",
    "    def token_input(self,tagged_token):  #tagged-token : nltk로 pos_tag한 token\n",
    "        input_word=set()\n",
    "        \n",
    "        for i in range(len(tagged_token)):\n",
    "            if \"NN\" in tagged_token[i][1]:\n",
    "                input_word.add(tagged_token[i][0].lower())\n",
    "            \n",
    "        for word in input_word:\n",
    "            if self.word_dict.has_key(word):\n",
    "                self.word_dict[word] += 1\n",
    "            else:\n",
    "                self.word_dict[word] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def str2stamp(date):\n",
    "    date += \" 00:00\"\n",
    "    return time.mktime(time.strptime(date, '%Y%m%d %H:%M'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stamp2str(timestamp):\n",
    "    return time.strftime(\"%Y%m%d\", time.localtime(int(timestamp)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def start():\n",
    "    db=AccessDB()\n",
    "    reddit = Reddit(db)\n",
    "    reddit.request2dbinsert(\"20170101:20170228\")\n",
    "    ana = Analyse()\n",
    "    posts = reddit.db.findPost('reddit',subreddit='futurology')\n",
    "    nouns = ana.posts_analyze(posts)\n",
    "    reddit.db.inputPosts('noun','futurology',nouns)\n",
    "    print \" complete\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_id_list(date, end_date = None):\n",
    "    '''\n",
    "    하루 이상의 데이터를 이용할 경우 끝나는 날을 end_date 입력\n",
    "    make_id_list(\"20170101\", end_date = \"20170108\")\n",
    "    Reddit.subreddits 에 있는 모든 서브레딧에서 부터 해당하는 날짜의 명사들을 가져옴\n",
    "    \n",
    "    '''\n",
    "    day = 86400\n",
    "    start_time = str2stamp(date)\n",
    "    \n",
    "    if end_date:\n",
    "        end_time = str2stamp(end_date) + day\n",
    "    else:\n",
    "        end_time = start_time + day\n",
    "    \n",
    "    query = {\"$and\" : [ \n",
    "        { \"date\" : { \"$gte\" : start_time}}, \n",
    "        { \"date\" : { \"$lt\" : end_time}}\n",
    "    ]}\n",
    "    \n",
    "    projection = {\"_id\" : 0, \"ID\" : 1}\n",
    "    \n",
    "    reddit = AccessDB()\n",
    "    reddit.useDB(\"reddit\")\n",
    "    \n",
    "    noun = AccessDB()\n",
    "    noun.useDB(\"noun\")\n",
    "    \n",
    "    subreddits = Reddit().subreddits\n",
    "    \n",
    "    id_list = {}\n",
    "    \n",
    "    for subreddit in subreddits:\n",
    "        \n",
    "        id_list[subreddit] = reddit.find(collection = subreddit, query = query, projection = projection)\n",
    "        \n",
    "        for ID in id_list[subreddit]:\n",
    "            id_query = {\"ID\":ID[\"ID\"]}\n",
    "            noun_list = noun.find(collection = subreddit, query = id_query)\n",
    "            \n",
    "            for nouns in noun_list:\n",
    "                ID.update(nouns)\n",
    "       \n",
    "    return id_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def df(documents):\n",
    "    \"\"\"\n",
    "    ID, _id, COMMENTS, 명사명단이 포함된\n",
    "    서브레딧하나만 받아야함\n",
    "    \"\"\"\n",
    "    df = {}\n",
    "    \n",
    "    exception = [\"COMMENTS\", \"ID\", \"_id\"]\n",
    "    \n",
    "    for submission in documents:\n",
    "        df[submission[\"ID\"]] = set()\n",
    "\n",
    "\n",
    "        df[submission[\"ID\"]] = set([ i for i in submission if i not in exception])\n",
    "        \n",
    "        if \"COMMENTS\" not in submission:\n",
    "            continue\n",
    "\n",
    "        for comment in submission[\"COMMENTS\"]:\n",
    "            df[submission[\"ID\"]] = df[submission[\"ID\"]].union(set(\n",
    "                [ i for i in comment if i not in exception]))\n",
    "            \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tf(documents):\n",
    "    \"\"\"\n",
    "    ID, _id, COMMENTS, 명사명단이 포함된\n",
    "    서브레딧하나만 받아야함\n",
    "    \"\"\"\n",
    "    exception = [\"COMMENTS\", \"ID\", \"_id\"]\n",
    "    \n",
    "    tf = {}\n",
    "    tf = defaultdict(lambda:0, tf)\n",
    "    \n",
    "    for submission in documents:\n",
    "        frequency = {}\n",
    "        frequency = defaultdict(lambda:0, frequency)\n",
    "        total = 0\n",
    "        \n",
    "        for noun_name in submission.keys():\n",
    "            if noun_name in exception:\n",
    "                continue\n",
    "            frequency[noun_name] += float(submission[noun_name])\n",
    "            total += submission[noun_name]\n",
    "            \n",
    "        if \"COMMENTS\" not in submission:\n",
    "            continue\n",
    "            \n",
    "        for comments_list in submission[\"COMMENTS\"]:\n",
    "            for noun_name in comments_list.keys():\n",
    "                if noun_name in exception:\n",
    "                    continue\n",
    "                    \n",
    "                frequency[noun_name] += float(comments_list[noun_name])\n",
    "                total += comments_list[noun_name]\n",
    "    \n",
    "        for noun_name in frequency.keys():\n",
    "            tf[noun_name] += frequency[noun_name] / total\n",
    "            \n",
    "    #sorted_tf = sorted(tf.items(), key=itemgetter(1), reverse = True) \n",
    "    \n",
    "    return tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score(for_tf, for_df):\n",
    "    all_df = {}\n",
    "    for subreddit in for_tf.keys():\n",
    "        all_df.update(df(for_tf[subreddit]))\n",
    "\n",
    "    for subreddit in for_df.keys():\n",
    "        all_df.update(df(for_df[subreddit]))\n",
    "        \n",
    "    noun_df = {}\n",
    "    noun_df = defaultdict(lambda:0, noun_df)\n",
    "    df_total = 0\n",
    "\n",
    "    for sub_id in all_df.keys():\n",
    "        for keyword in all_df[sub_id]:\n",
    "            noun_df[keyword] += 1\n",
    "        df_total += 1\n",
    "        \n",
    "    all_tf={}\n",
    "    \n",
    "    for i in for_tf.keys():\n",
    "        all_tf[i] = tf(for_tf[i])\n",
    "        \n",
    "    tf_idf = {}\n",
    "    for subreddit in all_tf.keys():\n",
    "        tf_idf[subreddit] = {}\n",
    "        for keyword in all_tf[subreddit]:\n",
    "            tf_idf[subreddit][keyword] = all_tf[subreddit][keyword] * math.log10(float(df_total) / noun_df[keyword])\n",
    "            \n",
    "    #sorted_tf_idf = {}\n",
    "    \n",
    "    #sorted_tf_idf['programming'] = sorted(tf_idf['programming'].items(), key=itemgetter(1), reverse = True) \n",
    "    \n",
    "    return tf_idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def date2list(start_date, end_date, pre_day = 0):\n",
    "    result = []\n",
    "    \n",
    "    day = 86400\n",
    "    \n",
    "    stamp_start_date = str2stamp(start_date)\n",
    "    stamp_end_date = str2stamp(end_date)\n",
    "    \n",
    "    date = stamp_start_date - pre_day*day\n",
    "    \n",
    "    result = [stamp2str(date)]\n",
    "    \n",
    "    while(date < stamp_end_date):\n",
    "        date += day\n",
    "        \n",
    "        result.append(stamp2str(date))\n",
    "        \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_insert_tf_idf(start_date, end_date):\n",
    "    scoreDB = AccessDB()\n",
    "    scoreDB.useDB('score')\n",
    "    \n",
    "    stamp_start_date = str2stamp(start_date)\n",
    "    stamp_end_date = str2stamp(end_date)\n",
    "    \n",
    "    date = stamp_start_date\n",
    "    \n",
    "    day = 86400\n",
    "    \n",
    "    while(date <= stamp_end_date):\n",
    "        x_week_date = stamp2str(date - day*7)\n",
    "        yesterday = stamp2str(date - day)\n",
    "        \n",
    "        today = stamp2str(date)\n",
    "        \n",
    "        \n",
    "        today_list = make_id_list(today)\n",
    "        x_week_list = make_id_list(x_week_date, end_date = yesterday)\n",
    "        \n",
    "        tf_idf = score(today_list, x_week_list)\n",
    "        \n",
    "        scoreDB.insert(\"d\"+today, tf_idf)\n",
    "            \n",
    "        date += day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_trend_score(date):\n",
    "    scoreDB = AccessDB()\n",
    "    scoreDB.useDB('score')\n",
    "    \n",
    "    documents = scoreDB.find(collection = \"d\" + date)[0]\n",
    "        \n",
    "        \n",
    "    x_week = date2list(date, date, pre_day = 7)\n",
    "    x_week.remove(date)\n",
    "        \n",
    "    x_week_document = [ scoreDB.find(collection = \"d\" + i)[0] for i in x_week]\n",
    "        \n",
    "    documents.pop(\"_id\")\n",
    "        \n",
    "    keyword_score = {}\n",
    "        \n",
    "    for subreddit in documents.keys():\n",
    "        keyword_score[subreddit] = {}\n",
    "        for keyword in documents[subreddit].keys():\n",
    "            score_sum = 0\n",
    "            for i in range(len(x_week_document)):\n",
    "                if x_week_document[i][subreddit].get(keyword):\n",
    "                    score_sum += x_week_document[i][subreddit][keyword]\n",
    "                        \n",
    "            keyword_score[subreddit][keyword] = documents[subreddit][keyword] - score_sum/len(x_week_document)\n",
    "                \n",
    "                \n",
    "    return keyword_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if __name__==\"__main__\":\n",
    "    #test_insert_score(\"20170108\", \"20170227\")\n",
    "    \n",
    "    #start_date = \"20170115\"\n",
    "    start_date = \"20170227\"\n",
    "    end_date = \"20170227\"\n",
    "    \n",
    "    \n",
    "    today = make_id_list(\"20170227\",end_date = \"20170227\")\n",
    "    x_week = make_id_list(\"20170220\",end_date = \"20170226\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss = Reddit().subreddits\n",
    "\n",
    "    tf_idf_result = {}\n",
    "    tf_idf_result = score(today,x_week)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sorted_tf_idf ={}\n",
    "    for i in sss:\n",
    "        sorted_tf_idf[i] = sorted(tf_idf_result[i].items(), key=itemgetter(1), reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf_result = {}\n",
    "    for i in sss:\n",
    "        tf_result[i] = tf(today[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sorted_tf ={}\n",
    "    for i in sss:\n",
    "        sorted_tf[i] = sorted(tf_result[i].items(), key=itemgetter(1), reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "trend_score = test_trend_score(\"20170227\")\n",
    "    sorted_trend = {}\n",
    "    for i in sss:\n",
    "        sorted_trend[i] = sorted(trend_score[i].items(), key=itemgetter(1), reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "****************************************programming****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:history0.557710420654        1:cobol:0.767344976576         1:cobol:0.732115503971\n",
      "2:time0.31985468264            2:history:0.717947803957       2:history:0.666621552419\n",
      "3:code0.259571034482           3:middlewares:0.616263733943   3:middlewares:0.616263733943\n",
      "4:something0.257827819177      4:webserver:0.616263733943     4:webserver:0.616263733943\n",
      "5:music0.222222222222          5:dynamo:0.593947144111        5:dynamo:0.593947144111\n",
      "6:monitor0.222222222222        6:monitor:0.570146246799       6:golang:0.563729685958\n",
      "7:language0.210775607314       7:golang:0.566116347575        7:monitor:0.524779811975\n",
      "8:origin0.2                    8:csv:0.516410876181           8:csv:0.490531044555\n",
      "9:fiction0.2                   9:code:0.491599087496          9:origin:0.487756974842\n",
      "10:cobol0.2                    10:origin:0.487756974842       10:vuforia:0.479316237511\n",
      "11:dynamo0.1875                11:vuforia:0.479316237511      11:imagemagick:0.479316237511\n",
      "12:https0.181083966776         12:imagemagick:0.479316237511  12:facebook-bot:0.479316237511\n",
      "13:functions0.178880982306     13:facebook-bot:0.479316237511 13:fiction:0.454490690489\n",
      "14:c0.164576279628             14:functions:0.464704670862    14:timelapse:0.445868460215\n",
      "15:edit0.163675958188          15:fiction:0.454490690489      15:prolapse:0.445868460215\n",
      "16:classes0.16358992913        16:timelapse:0.445868460215    16:echo:0.430402305991\n",
      "17:comment0.15242975758        17:prolapse:0.445868460215     17:bnf:0.420863037815\n",
      "18:part0.150591492216          18:echo:0.430402305991         18:export:0.416558018418\n",
      "19:look0.149390243902          19:bnf:0.420863037815          19:wang:0.4142307672\n",
      "20:golang0.147551978538        20:export:0.416558018418       20:interval:0.412420682919\n",
      "21:facebook0.146601903173      21:wang:0.4142307672           21:templates:0.397715654796\n",
      "22:guy0.146276595745           22:declaration:0.414124684233  22:fgrep:0.392167830691\n",
      "23:csv0.142857142857           23:interval:0.412420682919     23:declaration:0.384698398544\n",
      "24:declaration0.142857142857   24:templates:0.401281614194    24:❤️:0.378972905623\n",
      "25:echo0.142857142857          25:spam:0.401242308469         25:functions:0.378316255733\n",
      "26:class0.142857142857         26:fgrep:0.392167830691        26:ack:0.364801467449\n",
      "27:middlewares0.142857142857   27:language:0.390240106056     27:classes:0.35361360896\n",
      "28:export0.142857142857        28:classes:0.383677797881      28:ripgrep:0.349389387105\n",
      "29:spam0.142857142857          29:❤️:0.378972905623           29:dom:0.339267036601\n",
      "30:intro0.142857142857         30:vim:0.368301550752          30:fzf:0.333977306761\n",
      "\n",
      "****************************************gaming****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:game17.3502274709            1:game:12.9289585246           1:recon:3.20318494666\n",
      "2:games8.32922156671           2:games:8.17436199471          2:landing:2.67804192114\n",
      "3:time3.49452977475            3:recon:4.56144052862          3:yojimhoe:2.1569230688\n",
      "4:people3.15789982263          4:pc:3.46953089503             4:/r/im14andthisisfunny:2.1569230688\n",
      "5:i2.28515098881               5:landing:2.84293722959        5:oohhhh:2.1569230688\n",
      "6:pc2.09533645432              6:wildlands:2.66974436223      6:picross:2.1569230688\n",
      "7:recon1.98236543242           7:beta:2.61134145072           7:game:1.90867280398\n",
      "8:https1.81068082462           8:i:2.47689447016              8:shadow:1.84019026582\n",
      "9:lol1.6820228122              9:yojimhoe:2.1569230688        9:games:1.80160617449\n",
      "10:anyone1.65383350314         10:/r/im14andthisisfunny:2.1569230688 10:limbo:1.67947585266\n",
      "11:thing1.60820415212          11:oohhhh:2.1569230688         11:naughty:1.67444899262\n",
      "12:video1.53281400264          12:picross:2.1569230688        12:ohh:1.52241952385\n",
      "13:way1.46798809895            13:shadow:2.15136764524        13:wubububububub:1.43794871253\n",
      "14:something1.46237067917      14:switch:2.10574041258        14:dovision:1.43794871253\n",
      "15:world1.38898368998          15:time:2.0528439579           15:jacksmith:1.43794871253\n",
      "16:http1.34025711209           16:ps4:1.92459761605           16:luigi:1.41447547789\n",
      "17:lot1.24432191732            17:video:1.9122164499          17:silk:1.41427795489\n",
      "18:one1.23714686259            18:nintendo:1.78802234251      18:mordor:1.38587974143\n",
      "19:things1.23588350504         19:lol:1.78624882263           19:wildlands:1.35072023832\n",
      "20:beta1.19061439027           20:limbo:1.7440963768          20:psychics:1.33760538065\n",
      "21:years1.17384810526          21:witcher:1.73322957672       21:ace:1.33019886584\n",
      "22:wildlands1.15381239869      22:naughty:1.7053780753        22:beta:1.20156968356\n",
      "23:day1.14637303744            23:mod:1.65149518747           23:sequel:1.11773414297\n",
      "24:anything1.09452882393       24:luigi:1.56887743927         24:civilization:1.09348632121\n",
      "25:switch1.08077686315         25:ohh:1.55296243754           25:marill:1.0784615344\n",
      "26:guy1.04760665305            26:mordor:1.51820061875        26:teepublic:1.0784615344\n",
      "27:play1.04027639142           27:fps:1.49155187113           27:gen1:1.0784615344\n",
      "28:life1.02590593313           28:anyone:1.46508714814        28:gem:1.05842216404\n",
      "29:landing1.00724637681        29:wubububububub:1.43794871253 29:subway:1.03903921248\n",
      "30:ps40.998111102243           30:dovision:1.43794871253      30:brick:1.02727724066\n",
      "\n",
      "****************************************movies****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:movie5.32719080642           1:movie:6.84690455061          1:oscar:2.322198371\n",
      "2:film2.49965008964            2:film:3.47472403708           2:movie:1.67513206615\n",
      "3:movies1.63288058626          3:oscar:2.93791716733          3:oscars:1.22943368043\n",
      "4:oscar1.5122246345            4:movies:2.44975383609         4:cars:1.18408866986\n",
      "5:time1.30690905571            5:oscars:1.87619412704         5:yawwwnnnnn:1.0784615344\n",
      "6:people1.13733938131          6:trailer:1.6125968885         6:best:1.04458647308\n",
      "7:https1.03892823723           7:best:1.55326177344           7:b-roll:0.86276922752\n",
      "8:best0.927122926094           8:la:1.46195622571             8:actor:0.734509750908\n",
      "9:la0.926316353102             9:films:1.39200632162          9:littlefoot:0.718974356267\n",
      "10:something0.91042725643      10:cars:1.23626868042          10:nikos:0.718974356267\n",
      "11:year0.905967506468          11:actor:1.16445560391         11:koundouros:0.718974356267\n",
      "12:oscars0.890449020003        12:yawwwnnnnn:1.0784615344     12:generation”:0.677069096979\n",
      "13:films0.832354463639         13:picture:0.896620907915      13:lego:0.622144446647\n",
      "14:trailer0.808846370376       14:b-roll:0.86276922752        14:recommendations:0.620462371562\n",
      "15:http0.668925766338          15:land:0.781098680812         15:colorized:0.616263733943\n",
      "16:way0.66570868143            16:lego:0.770224132399         16:extended:0.59298134691\n",
      "17:cars0.649906162515          17:time:0.767737158209         17:razzies:0.591288899338\n",
      "18:actor0.586489665713         18:horror:0.765853108428       18:film:0.588638177362\n",
      "19:thing0.585495614391         19:year:0.744130116436         19:trailer:0.568994604093\n",
      "20:anyone0.577396862877        20:character:0.73500971802     20:winner:0.543650266887\n",
      "21:one0.563360702412           21:something:0.724571666311    21:piranha:0.528039857324\n",
      "22:picture0.552526769925       22:littlefoot:0.718974356267   22:it’s:0.512260953234\n",
      "23:fuck0.552292862076          23:nikos:0.718974356267        23:winners:0.511265282175\n",
      "24:years0.53608193966          24:koundouros:0.718974356267   24:kimmel:0.503517326865\n",
      "25:night0.517936169875         25:scene:0.71761024092         25:minute:0.492298438127\n",
      "26:anything0.497522621242      26:actors:0.690030217498       26:bollywood:0.492276908352\n",
      "27:land0.470361836353          27:hollywood:0.683951869862    27:picture:0.489881349496\n",
      "28:character0.469224725274     28:generation”:0.677069096979  28:chocolate:0.476365755258\n",
      "29:story0.467707146753         29:arrival:0.666516616457      29:omg:0.459116338845\n",
      "30:action0.463076758782        30:night:0.662760899966        30:greek:0.453607505506\n",
      "\n",
      "****************************************hacking****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:bug0.262048192771            1:/thread:0.867187024397       1:/thread:0.867187024397\n",
      "2:/thread0.25                  2:ip:0.670094420505            2:ip:0.659430756596\n",
      "3:address0.25                  3:illegal:0.647392567          3:illegal:0.644574759928\n",
      "4:illegal0.25                  4:bug:0.615828533109           4:bug:0.612742050294\n",
      "5:ip0.25                       5:address:0.579552735751       5:address:0.546343367182\n",
      "6:data0.151161328191           6:hackers:0.363326861203       6:hackers:0.32189399445\n",
      "7:hackers0.137048192771        7:cloudflare:0.35459061036     7:cloudflare:0.306844784048\n",
      "8:command0.132572431958        8:command:0.321057490581       8:command:0.297973489107\n",
      "9:example0.129695251595        9:leak:0.29435059316           9:leak:0.276480346444\n",
      "10:chances0.125                10:traffic:0.255601178411      10:wget:0.253755655153\n",
      "11:sign0.125                   11:wget:0.253755655153         11:chances:0.234314180471\n",
      "12:leak0.125                   12:data:0.249866751978         12:os:0.230349918682\n",
      "13:cloudflare0.125             13:os:0.245195647686           13:traffic:0.230061321049\n",
      "14:traffic0.119402985075       14:chances:0.234314180471      14:sign:0.220299517454\n",
      "15:os0.0921568627451           15:sign:0.220299517454         15:data:0.210961261325\n",
      "16:website0.0819006362529      16:tor:0.218804794431          16:apache:0.207982052189\n",
      "17:something0.079089391273     17:apache:0.207982052189       17:exploit:0.192497261908\n",
      "18:tor0.0779026217228          18:exploit:0.192497261908      18:upload:0.184573816385\n",
      "19:hat0.0722891566265          19:injection:0.188229575606    19:tor:0.179171852425\n",
      "20:bank0.0722891566265         20:upload:0.184573816385       20:injection:0.177033545251\n",
      "21:company0.0722891566265      21:example:0.182338245022      21:hat:0.164849810755\n",
      "22:service0.0700594844679      22:hat:0.164849810755          22:example:0.154535048935\n",
      "23:online0.0666666666667       23:learn:0.155946370936        23:learn:0.148928036204\n",
      "24:use0.063184079602           24:website:0.146397949158      24:bank:0.144795346775\n",
      "25:python0.0588235294118       25:bank:0.146299628403         25:proxygambit:0.143794871253\n",
      "26:server0.0588235294118       26:python:0.144148428087       26:greenwalds:0.143794871253\n",
      "27:apache0.0588235294118       27:proxygambit:0.143794871253  27:didnt:0.134380138819\n",
      "28:wget0.0588235294118         28:greenwalds:0.143794871253   28:scrapers:0.133760538065\n",
      "29:exploit0.0588235294118      29:didnt:0.134380138819        29:website:0.132594173125\n",
      "30:script0.0588235294118       30:scrapers:0.133760538065     30:tethering:0.128771526496\n",
      "\n",
      "****************************************worldnews****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:http4.10450256585            1:http:3.00360427636           1:vx:1.03912411926\n",
      "2:%1.83498161896               2:feedback:2.28090234661       2:rouhani:1.01518607335\n",
      "3:tl1.61295601326              3:tl:2.21431745845             3:dowry:0.958632475022\n",
      "4:feedback1.61180734193        4:israel:1.69389947552         4:khan:0.77432514375\n",
      "5:people1.23186376218          5:%:1.63183293433              5:rome:0.706529154794\n",
      "6:news0.949958221752           6:iran:1.43843318021           6:francis:0.693770492499\n",
      "7:bot0.903082432459            7:quot:1.36763083887           7:hamas:0.64891728571\n",
      "8:world0.895927024077          8:vx:1.24190223561             8:pope:0.634093092107\n",
      "9:comments0.85410797637        9:*keywords*:1.17624872112     9:carpets:0.633602548727\n",
      "10:dr0.84676772889             10:summary**:1.17624872112     10:iran:0.628700381748\n",
      "11:version0.837354528436       11:drs:1.1752710682            11:agent:0.628233413911\n",
      "12:trump0.826873107854         12:faq:1.17332391705           12:role’:0.616263733943\n",
      "13:israel0.815033949382        13:dr:1.14923109418            13:|:0.597644844997\n",
      "14:theory0.814285681965        14:theory:1.09097607647        14:indonesia:0.569877742251\n",
      "15:pm0.807359275039            15:pm:1.04936183981            15:church:0.553804444761\n",
      "16:*keywords*0.805903670963    16:rouhani:1.01518607335       16:aide:0.546187320275\n",
      "17:faq0.805903670963           17:version:0.985936967999      17:greece:0.536348470207\n",
      "18:drs0.805903670963           18:dowry:0.958632475022        18:nerve:0.517656713372\n",
      "19:summary**0.805903670963     19:news:0.942200576993         19:critics:0.507931018849\n",
      "20:quot0.791276547734          20:pope:0.852049789613         20:ties:0.501750289156\n",
      "21:iran0.692114197291          21:kim:0.844146830478          21:historian:0.499261349707\n",
      "22:state0.656186196174         22:world:0.82670279601         22:water:0.493554318256\n",
      "23:us0.629272692521            23:khan:0.81991289193          23:nokia:0.492409661127\n",
      "24:https0.510409857392         24:indonesia:0.811754241226    24:bandung:0.479316237511\n",
      "25:years0.473771963108         25:hamas:0.799428009368        25:sunday:0.476402711132\n",
      "26:russia0.460189689112        26:rome:0.76097770902          26:tekebayev:0.454089067116\n",
      "27:country0.444229190753       27:state:0.751948912237        27:neighborhood:0.450737608731\n",
      "28:time0.440475581556          28:church:0.735430157339       28:israel:0.438998522149\n",
      "29:city0.429722844384          29:mosul:0.73314076724         29:carpet:0.424931746345\n",
      "30:uk0.4178459653              30:syria:0.728937456211        30:re-eating:0.392167830691\n",
      "\n",
      "****************************************futurology****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:people0.583039423106         1:car:0.917114904658           1:villages:0.680026860134\n",
      "2:car0.557676247312            2:villages:0.764643408124      2:initiative:0.672649211801\n",
      "3:jobs0.460651007407           3:jobs:0.712142743866          3:car:0.649145498375\n",
      "4:future0.436510410696         4:species:0.698259459824       4:species:0.595613594327\n",
      "5:time0.344872935326           5:initiative:0.672649211801    5:biologists:0.441961860902\n",
      "6:species0.304012351525        6:future:0.567352579978        6:jobs:0.425649173262\n",
      "7:years0.290572927362          7:ai:0.521451901699            7:genome:0.401281614194\n",
      "8:work0.270282798952           8:robots:0.506778154906        8:cash:0.378617201703\n",
      "9:ai0.268149635954             9:humans:0.50211673474         9:screens:0.367346500528\n",
      "10:article0.265568266395       10:automation:0.500702993961   10:glasses:0.35436828384\n",
      "11:humans0.260159963314        11:cars:0.457247585992         11:maluuba:0.331834318277\n",
      "12:title0.25410940077          12:biologists:0.441961860902   12:robo-rats:0.331834318277\n",
      "13:villages0.25                13:genome:0.401281614194       13:automation’s:0.331834318277\n",
      "14:initiative0.25              14:fusion:0.389009861592       14:chipping:0.331834318277\n",
      "15:half0.25                    15:vision:0.386540540169       15:gans:0.331834318277\n",
      "16:cars0.240374951366          16:cash:0.384763652354         16:vision:0.327721962362\n",
      "17:way0.238643814675           17:screens:0.370419967029      17:mechanism:0.327266924282\n",
      "18:robots0.228234810848        18:driving:0.369896713013      18:half:0.326787587887\n",
      "19:automation0.214748834782    19:scientists:0.365463062016   19:multi-agent:0.308678164764\n",
      "20:things0.213065729781        20:glasses:0.361106904823      20:empowers:0.308678164764\n",
      "21:intelligence0.20410360722   21:half:0.35926279035          21:coupling:0.308678164764\n",
      "22:cash0.203337421696          22:intelligence:0.344007895876 22:fusion:0.295881568313\n",
      "23:information0.201509469382   23:technology:0.339978632532   23:lace:0.294613872718\n",
      "24:technology0.186975909347    24:mechanism:0.336831603396    24:regeneration:0.289182692968\n",
      "25:end0.186274509804           25:maluuba:0.331834318277      25:restriction:0.279284789152\n",
      "26:life0.184864512259          26:robo-rats:0.331834318277    26:transmit:0.274051777349\n",
      "27:video0.180423280423         27:automation’s:0.331834318277 27:backseat:0.266826776737\n",
      "28:vision0.178571428571        28:chipping:0.331834318277     28:wearer:0.265127581877\n",
      "29:scientists0.167619983604    29:gans:0.331834318277         29:automation:0.257569033449\n",
      "30:driving0.155461396051       30:regeneration:0.321426807929 30:driving:0.253308664913\n",
      "\n",
      "****************************************sports****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:year0.468822953766           1:armbar:1.43794871253         1:armbar:1.43794871253\n",
      "2:daytona0.403571428571        2:daytona:1.33737361982        2:daytona:1.33240930909\n",
      "3:someone0.381237607835        3:helicopter:0.850139381346    3:helicopter:0.850139381346\n",
      "4:ball0.339743589744           4:liverpool:0.800010761739     4:liverpool:0.800010761739\n",
      "5:play0.336538461538           5:simple:0.725708523482        5:simple:0.725708523482\n",
      "6:armbar0.333333333333         6:canelo:0.718974356267        6:canelo:0.718974356267\n",
      "7:simple0.333333333333         7:engines:0.686411103384       7:engines:0.686411103384\n",
      "8:helicopter0.333333333333     8:perch:0.639454147147         8:perch:0.639454147147\n",
      "9:liverpool0.333333333333      9:sportsmanship:0.618631024379 9:sportsmanship:0.618631024379\n",
      "10:united0.257575757576        10:drivers:0.602689037652      10:drivers:0.602689037652\n",
      "11:team0.251674779359          11:your:0.533000637414         11:your:0.529777524918\n",
      "12:your0.25                    12:khabib:0.501602017742       12:khabib:0.501602017742\n",
      "13:wow0.25                     13:jre:0.47959061036           13:wrecks:0.47959061036\n",
      "14:engines0.25                 14:wrecks:0.47959061036        14:jre:0.478781836064\n",
      "15:drivers0.25                 15:ball:0.473863502544         15:niskanen:0.417468981058\n",
      "16:rule0.213631905852          16:play:0.445112520835         16:play:0.403163315171\n",
      "17:league0.209865542067        17:niskanen:0.417468981058     17:punch:0.375218721723\n",
      "18:people0.196134335657        18:cricket:0.41111962883       18:speed:0.369865995716\n",
      "19:speed0.191056910569         19:punch:0.408420546247        19:ferguson:0.366704361986\n",
      "20:cup0.181818181818           20:rugby:0.401708818925        20:rule:0.355953190067\n",
      "21:player0.180608974359        21:rule:0.386395207614         21:tire:0.355397274851\n",
      "22:power0.178319265276         22:year:0.385074825182         22:united:0.346246684456\n",
      "23:basketball0.176997902873    23:speed:0.371890407232        23:ball:0.341945842948\n",
      "24:football0.170182595183      24:ferguson:0.366704361986     24:rugby:0.336947177842\n",
      "25:canelo0.166666666667        25:tire:0.355397274851         25:cup:0.317651837484\n",
      "26:punch0.166666666667         26:united:0.352081041475       26:sweat-flick:0.308131866972\n",
      "27:perch0.166666666667         27:cup:0.349620187363          27:cricket:0.307760359989\n",
      "28:vs0.166666666667            28:wow:0.333347061331          28:wow:0.302824742923\n",
      "29:sportsmanship0.166666666667 29:someone:0.330532406476      29:year:0.299707793771\n",
      "30:sports0.161079440037        30:hockey:0.327308642977       30:credits:0.29940893602\n",
      "\n",
      "****************************************news****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:people1.0636922706           1:water:0.564346408355         1:plots:0.532388788585\n",
      "2:news0.519957543126           2:hollywood:0.534294626696     2:ira:0.518088934429\n",
      "3:man0.419176285634            3:plots:0.533317130882         3:hollywood:0.50232992868\n",
      "4:water0.348525284211          4:ira:0.518088934429           4:clingy:0.445868460215\n",
      "5:lot0.301180893115            5:news:0.515711413332          5:water:0.422459521388\n",
      "6:hollywood0.278873071404      6:crash:0.447047578355         6:bae’:0.392167830691\n",
      "7:trump0.273339738614          7:clingy:0.445868460215        7:dubai:0.391151319452\n",
      "8:time0.272205555469           8:people:0.419522281403        8:crash:0.386924554935\n",
      "9:us0.247854666544             9:bombs:0.408420546247         9:bombs:0.382535505522\n",
      "10:name0.245147694167          10:un:0.405014365098           10:volunteer:0.355397274851\n",
      "11:police0.233455466043        11:bae’:0.392167830691         11:wwi:0.335662944492\n",
      "12:war0.23288722453            12:dubai:0.391151319452        12:un:0.332495412777\n",
      "13:way0.228392609061           13:judge:0.373738790903        13:wapner:0.331834318277\n",
      "14:car0.226276138933           14:car:0.372117730644          14:panda:0.330158161864\n",
      "15:years0.221535413275         15:police:0.363361785525       15:terror:0.316053430241\n",
      "16:year0.210463693234          16:terror:0.361063980284       16:fiery:0.315340736144\n",
      "17:court0.19896149041          17:volunteer:0.355397274851    17:moines:0.314337055067\n",
      "18:crash0.198076742659         18:county:0.346191768806       18:landslides:0.308678164764\n",
      "19:woman0.196804073323         19:man:0.344824708103          19:srinivas:0.308678164764\n",
      "20:fuck0.194321687944          20:wwi:0.335960362816          20:rainstorms:0.308678164764\n",
      "21:school0.192900339773        21:wapner:0.331834318277       21:keesler:0.308131866972\n",
      "22:un0.190476190476            22:panda:0.330158161864        22:memorial:0.300159285932\n",
      "23:crime0.187838452649         23:woman:0.324570096832        23:flop:0.297819742447\n",
      "24:threat0.184026973377        24:threat:0.323880684565       24:kuchibhotla:0.295132683299\n",
      "25:someone0.180780534228       25:crime:0.319923818026        25:isil:0.294617817702\n",
      "26:judge0.176462811273         26:streets:0.316752459092      26:airman:0.286629724424\n",
      "27:uk0.171150971599            27:fiery:0.315340736144        27:imbeciles:0.278067394867\n",
      "28:terror0.170921985816        28:moines:0.314337055067       28:des:0.275078028808\n",
      "29:guy0.170509537991           29:landslides:0.308678164764   29:organisation:0.267516687832\n",
      "30:ira0.168025362319           30:rainstorms:0.308678164764   30:mardi:0.265648275916\n",
      "\n",
      "****************************************music****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:https6.82856445516           1:song:3.89994070502           1:alt/fusion:1.0784615344\n",
      "2:song3.02855884522            2:band:3.33852156343           2:fjord:1.00320403548\n",
      "3:band2.28356712932            3:rock:3.12199533857           3:steez:0.718974356267\n",
      "4:rock2.26917719126            4:album:3.00000329999          4:songbpm:0.718974356267\n",
      "5:music2.17878857715           5:music:2.7503017002           5:punk:0.626961108869\n",
      "6:album2.06227203626           6:https:2.60076335684          6:playlist:0.613873488353\n",
      "7:songs1.25247128593           7:songs:1.94940906927          7:johnny:0.582395547176\n",
      "8:artist1.23409559473          8:spotify:1.75576262413        8:wheel:0.580968570514\n",
      "9:please1.16809832632          9:itunes:1.73257369731         9:rock:0.575083176424\n",
      "10:spotify1.11919600558        10:tidal:1.69547368717         10:goat:0.567071567216\n",
      "11:youtube1.10848871792        11:boop:1.68902427586          11:bpm:0.556173558255\n",
      "12:listeners1.09780006548      12:*if:1.68809062722           12:harrison:0.553532184345\n",
      "13:plays1.08779145022          13:beep:1.68437530308          13:hop:0.55056435414\n",
      "14:itunes1.08375348397         14:time*:1.68345125001         14:v=t3m3igjnbhs:0.5392307672\n",
      "15:^^downvote1.08154145022     15:uo=4:1.67553393283          15:'folsom:0.5392307672\n",
      "16:^^incorrect1.08154145022    16:artist:1.65743299301        16:wagon:0.498604473811\n",
      "17:^^if1.08154145022           17:platforms:1.64711301792     17:cash:0.495243689106\n",
      "18:^^self-deletes1.08154145022 18:youtube:1.57108314113       18:fretboard:0.479316237511\n",
      "19:^^01.08154145022            19:listeners:1.52888429066     19:pink:0.449423345472\n",
      "20:^^please1.0757611034        20:^^incorrect:1.5136204831    20:life:0.440083906256\n",
      "21:http1.05780237735           21:^^if:1.5136204831           21:jazz:0.428987900764\n",
      "22:tidal1.03908352299          22:^^self-deletes:1.5136204831 22:capital:0.42275795194\n",
      "23:pic1.03607906237            23:^^0:1.5136204831            23:paxton:0.418455831911\n",
      "24:*if1.03513095382            24:^^downvote:1.51190726242    24:walking:0.413149495835\n",
      "25:time*1.03513095382          25:^^please:1.50496215818      25:bro:0.412833715371\n",
      "26:bot1.03513095382            26:tags:1.44100123764          26:hip:0.407373513946\n",
      "27:beep1.03513095382           27:plays:1.4212269254          27:madrid:0.40175063957\n",
      "28:platforms1.03513095382      28:pic:1.40220387409           28:/r/newmadrid:0.392167830691\n",
      "29:boop1.03513095382           29:playlist:1.32191065913      29:adejuyigbe:0.392167830691\n",
      "30:tags1.03392240261           30:guitar:1.23878107794        30:psych-rock:0.392167830691\n",
      "\n",
      "****************************************science****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:people0.276212089952         1:entomology:0.5392307672      1:entomology:0.5392307672\n",
      "2:research0.251284842686       2:insect:0.5392307672          2:insect:0.5392307672\n",
      "3:something0.171191553544      3:bumblebees:0.463973268284    3:rodents:0.439049493988\n",
      "4:http0.169945319475           4:rodents:0.463973268284       4:bumblebees:0.412387835718\n",
      "5:paper0.162982397582          5:research:0.453142240112      5:oxidative:0.392167830691\n",
      "6:power0.125                   6:oxidative:0.392167830691     6:x-post/r/marijuana:0.392167830691\n",
      "7:bumblebees0.125              7:x-post/r/marijuana:0.392167830691 7:ogd/r-induced:0.392167830691\n",
      "8:brains0.125                  8:ogd/r-induced:0.392167830691 8:hippocampal:0.392167830691\n",
      "9:matter0.125                  9:hippocampal:0.392167830691   9:ants:0.382321704062\n",
      "10:bees0.125                   10:bees:0.385424652028         10:cannabidiol:0.364801467449\n",
      "11:rodents0.125                11:ants:0.382321704062         11:dinomyidae:0.359487178133\n",
      "12:ants0.125                   12:cannabidiol:0.364801467449  12:rodent:0.319727073573\n",
      "13:entomology0.125             13:dinomyidae:0.359487178133   13:penguin:0.308063323005\n",
      "14:insect0.125                 14:rodent:0.319727073573       14:sheets:0.303981926746\n",
      "15:link0.120409345634          15:emissions:0.316827571078    15:emissions:0.293720615614\n",
      "16:stress0.115572768039        16:sheets:0.312077351031       16:neighborhoods:0.288248237344\n",
      "17:cool0.111111111111          17:penguin:0.308063323005      17:p-wave:0.287589742507\n",
      "18:house0.111111111111         18:brains:0.30782347361        18:s-wave:0.287589742507\n",
      "19:plastic0.111111111111       19:neurons:0.301258739782      19:neat:0.281485772429\n",
      "20:sheets0.111111111111        20:neighborhoods:0.288248237344 20:plastic:0.26054869751\n",
      "21:neat0.111111111111          21:p-wave:0.287589742507       21:brains:0.258902448618\n",
      "22:front0.111111111111         22:s-wave:0.287589742507       22:magma:0.255781658859\n",
      "23:film0.111111111111          23:paper:0.284509311843        23:taipei:0.247452409752\n",
      "24:emissions0.111111111111     24:stress:0.283906081236       24:neurons:0.246991678798\n",
      "25:crime0.1                    25:neat:0.282554902884         25:dioxide:0.246694803754\n",
      "26:neighborhoods0.1            26:plastic:0.279389509794      26:vitamin:0.239658118756\n",
      "27:benefit0.0953933958418      27:magma:0.255781658859        27:ascorbic:0.239658118756\n",
      "28:penguin0.0952380952381      28:dioxide:0.255660425972      28:pauling:0.239658118756\n",
      "29:core0.09375                 29:taipei:0.247452409752       29:metropolis:0.227383743374\n",
      "30:food0.0931512433755         30:pauling:0.239658118756      30:pfass:0.224095903252\n",
      "\n",
      "****************************************technology****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:phone1.03136329929           1:nokia:2.7392952309           1:nokia:2.7392952309\n",
      "2:nokia0.902540926785          2:phone:1.68380877634          2:phone:1.3968455785\n",
      "3:google0.609158985598         3:android:1.19910621883        3:android:1.01052971782\n",
      "4:us0.501095806448             4:google:1.10763488003         4:assistant:0.994455719285\n",
      "5:android0.485714285714        5:samsung:1.07084116189        5:mwc:0.941429479879\n",
      "6:apple0.445387596899          6:assistant:0.998442835731     6:ad-company:0.86276922752\n",
      "7:assistant0.402380952381      7:apple:0.987395853012         7:phonemakers:0.86276922752\n",
      "8:samsung0.391666666667        8:mwc:0.941429479879           8:surveillance:0.849134830493\n",
      "9:phones0.375850530582         9:phones:0.866427919382        9:samsung:0.805016302315\n",
      "10:answer0.336448598131        10:ad-company:0.86276922752    10:mongolians:0.802563228387\n",
      "11:https0.333650306175         11:phonemakers:0.86276922752   11:phones:0.716435111003\n",
      "12:surveillance0.333333333333  12:surveillance:0.852657093976 12:mininova:0.681133600674\n",
      "13:new0.320674146682           13:mongolians:0.802563228387   13:marshmallow:0.593778850922\n",
      "14:years0.315231890063         14:mininova:0.681133600674     14:apple:0.588227243018\n",
      "15:news0.298918518756          15:marshmallow:0.593778850922  15:answer:0.562264088975\n",
      "16:mwc0.271403242148           16:nougat:0.574443094066       16:resurrected:0.530255163753\n",
      "17:things0.269569341145        17:answer:0.565021301195       17:nougat:0.497382692346\n",
      "18:people0.255884573969        18:us:0.550042962737           18:weakness:0.470668709821\n",
      "19:down0.252631578947          19:resurrected:0.530255163753  19:down:0.457307115426\n",
      "20:model0.247960779437         20:down:0.507409204586         20:us:0.453486715954\n",
      "21:data0.222062218345          21:weakness:0.500186556192     21:genghis:0.43138461376\n",
      "22:bbc0.201923076923           22:model:0.469610537051        22:smartphones:0.413167678941\n",
      "23:http0.201860465116          23:smartphones:0.43413615493   23:model:0.401531305628\n",
      "24:weakness0.2                 24:genghis:0.43138461376       24:'uber:0.401281614194\n",
      "25:ad-company0.2               25:uber:0.414300094347         25:re-release:0.392219359818\n",
      "26:mongolians0.2               26:'uber:0.401281614194        26:mercer:0.389932779516\n",
      "27:hit0.2                      27:bbc:0.394177733974          27:bbc:0.388978789187\n",
      "28:phonemakers0.2              28:re-release:0.392219359818   28:computer’s:0.359985192341\n",
      "29:media0.194219052543         29:mercer:0.389932779516       29:doomed:0.353569488722\n",
      "30:tool0.182748414376          30:tool:0.385054611861         30:google:0.353057270761\n",
      "\n",
      "****************************************politics****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:comments13.918837568         1:users:8.58401451067          1:navy:0.810031379608\n",
      "2:do13.8483077002              2:do:8.58030101877             2:christie:0.7719855499\n",
      "3:subreddit13.8214210283       3:subreddit:8.54667556597      3:withdraws:0.651462387444\n",
      "4:users13.813104811            4:comments:7.44920187847       4:bilden:0.553440828169\n",
      "5:https9.96587431952           5:incivility:5.90571953846     5:spicer:0.51647465452\n",
      "6:incivility9.22243517081      6:user:5.80488467104           6:sanders:0.48389839112\n",
      "7:user9.21031881915            7:trump:5.31064442983          7:bernie:0.43351515339\n",
      "8:people7.38462092598          8:https:3.79565586871          8:yemen:0.428035272254\n",
      "9:trump7.27405056129           9:speech:2.98408362581         9:dinner:0.416439963749\n",
      "10:anything5.0416611406        10:morons:2.95142501106        10:leaks:0.407305478491\n",
      "11:questions4.82117150606      11:attacks:2.95076204476       11:seal:0.402040590888\n",
      "12:attack4.78273715078         12:to=/r/politics:2.94889786069 12:sessions:0.364703257231\n",
      "13:speech4.75567822857         13:wiki_no_personal_attacks:2.94889786069 13:secretary:0.363957658912\n",
      "14:please4.72869537195         14:ban**:2.94889786069         14:oscars:0.351263462255\n",
      "15:children4.72572420574       15:explicit:2.94847371142      15:dnc:0.348069096371\n",
      "16:action4.72462201481         16:modmail:2.94804965209       16:father:0.303306176118\n",
      "17:attacks4.71531614353        17:wiki_please_be_civil:2.94804965209 17:nominee:0.299497798983\n",
      "18:ideas4.68074902382          18:implicit:2.94720180314      18:owens:0.291273511177\n",
      "19:concerns4.65169745363       19:shill:2.9436619365          19:spending:0.286002843069\n",
      "20:morons4.64218755639         20:conduct:2.94276785428       20:prosecutor:0.280404558243\n",
      "21:opinions4.63873775074       21:shills:2.94154786456        21:pleadings:0.27622738334\n",
      "22:discussion4.63192624736     22:trolls:2.93925802415        22:nyt:0.270860970323\n",
      "23:jerk4.62796000427           23:jerk:2.93774487199          23:kasich:0.260373380839\n",
      "24:trolls4.62437612653         24:*i:2.93148815609            24:ad:0.257694207137\n",
      "25:quality4.62194877431        25:reminder:2.92766907411      25:ethics:0.254141252753\n",
      "26:reminder4.6160521671        26:questions:2.9237816527      26:probe:0.247337911092\n",
      "27:conduct4.61001216202        27:/message/compose/:2.91942256996 27:correspondents:0.246852482822\n",
      "28:moderators4.60613193877     28:moderators:2.91386163555    28:investigation:0.246482873924\n",
      "29:*i4.60554335847             29:concerns:2.91385288396      29:cuts:0.230745419052\n",
      "30:bot4.60547814825            30:people:2.91250872437        30:philip:0.230086266668\n",
      "\n",
      "****************************************nba****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:game2.77139632613            1:team:2.55623434212           1:giannis:1.14462786065\n",
      "2:team2.7644527782             2:season:2.35188986862         2:kat:1.0311971573\n",
      "3:season2.06713230852          3:game:2.06517569962           3:buzzer:0.754970289076\n",
      "4:player1.85021955848          4:nba:1.96680209848            4:millsap:0.733686504332\n",
      "5:games1.73280331876           5:player:1.94136953037         5:dedmon:0.723484534912\n",
      "6:players1.56686568469         6:lebron:1.90416315145         6:morris:0.65949738662\n",
      "7:nba1.51126141677             7:giannis:1.76172464511        7:games:0.623555203589\n",
      "8:time1.49212096694            8:westbrook:1.73961614422      8:westbrook:0.598864377072\n",
      "9:year1.45981114195            9:players:1.7203312856         9:chriss:0.594305225192\n",
      "10:man1.25385862583            10:games:1.70058648095         10:career:0.573221687089\n",
      "11:league1.2144173529          11:warriors:1.59336580305      11:putback:0.569967451728\n",
      "12:lebron1.20650497953         12:league:1.40360577758        12:refs:0.562275945379\n",
      "13:guy1.11170984603            13:dunk:1.36101801138          13:denver:0.562025399603\n",
      "14:%1.03362433884              14:|:1.34981111514             14:wood:0.548398601479\n",
      "15:point1.00351301077          15:teams:1.24037890062         15:assists:0.543216943094\n",
      "16:warriors0.98932730705       16:year:1.19903796469          16:conley:0.541085704656\n",
      "17:teams0.951851508541         17:kat:1.19211195744           17:halftime:0.523228170739\n",
      "18:westbrook0.912893844234     18:ball:1.18949124174          18:highlights:0.497793086932\n",
      "19:people0.901970768086        19:points:1.16268971579        19:gobert:0.484277091487\n",
      "20:ball0.852823697681          20:career:1.1082965815         20:marcus:0.484116207151\n",
      "21:points0.824477723125        21:harden:1.08717484638        21:~:0.482792471934\n",
      "22:https0.812411779352         22:defense:1.07894225555       22:dunk:0.478095903774\n",
      "23:way0.802998824517           23:cavs:1.06747385625          23:blake:0.475494205931\n",
      "24:career0.800249703235        24:kawhi:1.06410385309         24:mason:0.472977889644\n",
      "25:|0.783506841224             25:mvp:1.05894921997           25:cp3:0.470792972758\n",
      "26:giannis0.773901531341       26:basketball:1.03965700124    26:clock:0.460878062469\n",
      "27:defense0.760662937341       27:man:1.03145490209           27:pts:0.452127366769\n",
      "28:lot0.732772089651           28:guy:0.986639780015          28:wall:0.449220929431\n",
      "29:years0.729599592488         29:hayward:0.978102067573      29:season:0.448440529362\n",
      "30:fuck0.721784693644          30:playoffs:0.975251667771     30:hayward:0.440395819158\n",
      "\n",
      "****************************************soccer****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:goal3.94587549767            1:goal:5.51185967916           1:goal:2.02185995306\n",
      "2:https2.93368381718           2:goals:2.5194280559           2:messi:1.75574662828\n",
      "3:%1.8382514595                3:messi:2.29131961061          3:marseille:1.66430727788\n",
      "4:game1.56449728671            4:madrid:1.96391620748         4:madrid:1.44596038159\n",
      "5:league1.55351538705          5:mirror:1.82954757535         5:vitesse:1.40638765888\n",
      "6:time1.48945094614            6:league:1.79553031551         6:kane:1.34403101085\n",
      "7:goals1.47460782096           7:marseille:1.71536905421      7:feyenoord:1.26029575462\n",
      "8:team1.43393245217            8:penalty:1.67108227771        8:arnhem:1.2022898412\n",
      "9:season1.29930297029          9:%:1.63474077462              9:heracles:1.16679700803\n",
      "10:man1.19207784106            10:cup:1.61914726433           10:psv:1.055480838\n",
      "11:player1.03206494375         11:kane:1.55710785505          11:southampton:1.05293894297\n",
      "12:mirror1.00618476828         12:season:1.47828829315        12:payet:0.997056504779\n",
      "13:messi0.999416734377         13:twittertostreamable:1.45492116128 13:cup:0.969341403784\n",
      "14:madrid0.884477832752        14:subject=:1.45492116128      14:fanni:0.956823364947\n",
      "15:vs0.874498984691            15:to=erody:1.45492116128      15:goals:0.935197725442\n",
      "16:cup0.842029214376           16:___:1.45022908355           16:villarreal:0.915044473153\n",
      "17:penalty0.825710426116       17:feyenoord:1.42363907976     17:zlatan:0.882500662563\n",
      "18:players0.821672633487       18:vitesse:1.41568150447       18:ajax:0.877159111885\n",
      "19:fuck0.788546887725          19:creator:1.37349827264       19:depay:0.865391531889\n",
      "20:ball0.782876180797          20:vs:1.35649446844            20:kick-ins:0.86276922752\n",
      "21:football0.744980144092      21:team:1.32592873621          21:ahead:0.859974043244\n",
      "22:bot0.730914986622           22:suggestions:1.26439627135   22:birsa:0.857859365468\n",
      "23:suggestions0.728190233898   23:football:1.20564938943      23:perisic:0.810958333109\n",
      "24:___0.722047727755           24:arnhem:1.20349815507        24:throw-ins:0.801624807405\n",
      "25:creator0.722047727755       25:ajax:1.17222902466          25:granada:0.798613565154\n",
      "26:twittertostreamable0.722047727755 26:heracles:1.16821643106      26:metz:0.776993223146\n",
      "27:subject=0.722047727755      27:game:1.1658245153           27:wolfsburg:0.76840858963\n",
      "28:to=erody0.722047727755      28:roma:1.14424920476          28:kazuyoshi:0.755178525005\n",
      "29:pm0.722047727755            29:southampton:1.14260333306   29:leagues:0.747126142373\n",
      "30:fans0.707679514527          30:psv:1.14203723691           30:miura:0.732090387335\n",
      "\n",
      "****************************************design****************************************\n",
      "         [tf]                         [tf-idf]                     [trend_score]\n",
      "1:logo0.0625                   1:sacrifice:0.16569301912      1:sacrifice:0.16569301912\n",
      "2:sacrifice0.0625              2:logo:0.148709230397          2:wrestling:0.120103432227\n",
      "3:planning0.0422535211268      3:wrestling:0.120103432227     3:logo:0.108023367481\n",
      "4:design0.0422535211268        4:planning:0.107450456026      4:planning:0.107450456026\n",
      "5:hospital0.0422535211268      5:charity:0.0985190054565      5:charity:0.0985190054565\n",
      "6:wrestling0.0416666666667     6:pro:0.0985190054565          6:pro:0.0985190054565\n",
      "7:company0.0416666666667       7:hospital:0.0968728241633     7:hospital:0.0968728241633\n",
      "8:charity0.0416666666667       8:child/old:0.0898717945334    8:child/old:0.0898717945334\n",
      "9:pro0.0416666666667           9:saluting:0.0898717945334     9:saluting:0.0898717945334\n",
      "10:pieces0.0416666666667       10:pinkie:0.0898717945334      10:pinkie:0.0898717945334\n",
      "11:something0.0416666666667    11:pootis:0.0836003362904      11:pootis:0.0836003362904\n",
      "12:ideas0.0416666666667        12:design:0.0817028829249      12:hatchet:0.0799317683934\n",
      "13:community0.0281690140845    13:hatchet:0.0799317683934     13:crucifix:0.0799317683934\n",
      "14:buildings0.0281690140845    14:crucifix:0.0799317683934    14:silhouette:0.0736603101504\n",
      "15:show0.0208333333333         15:pieces:0.076290523083       15:buildings:0.0686981654707\n",
      "16:focus0.0208333333333        16:silhouette:0.0736603101504  16:ect:0.0673888519074\n",
      "17:thanks0.0208333333333       17:buildings:0.0686981654707   17:wrestler:0.065369893303\n",
      "18:dark0.0208333333333         18:ect:0.0673888519074         18:pieces:0.0649796202195\n",
      "19:child/old0.0208333333333    19:wrestler:0.065369893303     19:eagle:0.0619046553496\n",
      "20:creation0.0208333333333     20:company:0.0641691538592     20:fittings:0.0607583963042\n",
      "21:lion0.0208333333333         21:eagle:0.0619046553496       21:/r/architecture:0.0607583963042\n",
      "22:religion0.0208333333333     22:fittings:0.0607583963042    22:/r/health:0.0607583963042\n",
      "23:pootis0.0208333333333       23:/r/architecture:0.0607583963042 23:extensive:0.0540383786321\n",
      "24:saluting0.0208333333333     24:/r/health:0.0607583963042   24:sicker:0.0540383786321\n",
      "25:contrast0.0208333333333     25:extensive:0.0540383786321   25:consultation:0.0522786781165\n",
      "26:eagle0.0208333333333        26:/r/design:0.0540383786321   26:architects:0.0509137483558\n",
      "27:ect0.0208333333333          27:sicker:0.0540383786321      27:fighter:0.0508080182127\n",
      "28:hatchet0.0208333333333      28:consultation:0.0522786781165 28:lion:0.050224086471\n",
      "29:crucifix0.0208333333333     29:contrast:0.0514322520331    29:soldier:0.0500006726087\n",
      "30:image0.0208333333333        30:architects:0.0509137483558  30:contrast:0.0486951455008\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in sss:\n",
    "        print \"*\"*40 + i + \"*\"*40\n",
    "        print (\" \"*9 + \"[\" + \"tf\" + \"]\").ljust(30),\n",
    "        print (\" \"*7 + \"[\" + \"tf-idf\" + \"]\").ljust(30),\n",
    "        print (\" \"*5 + \"[\" + \"trend_score\" + \"]\")\n",
    "        for j in range(30):\n",
    "            print (str(j+1) + \":\" + sorted_tf[i][j][0] + str(sorted_tf[i][j][1])).ljust(30),\n",
    "            print (str(j+1) + \":\" + sorted_tf_idf[i][j][0] + \":\" + str(sorted_tf_idf[i][j][1])).ljust(30),\n",
    "            print str(j+1) + \":\" + sorted_trend[i][j][0] + \":\" + str(sorted_trend[i][j][1])\n",
    "        print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}