manual_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Models: ['data/materials/mini.ftz']\n"
     ],
     "output_type": "stream"
    },
    {
     "name": "stderr",
     "text": [
      "\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "from model import FastTextInferenceModel\n",
    "from performance_evaluator import PerformanceEvaluator\n",
    "\n",
    "fasttext_model_path = 'data/materials/mini.ftz'\n",
    "fasttext_model = FastTextInferenceModel(fasttext_model_path)\n",
    "\n",
    "vec_emb_path = 'data/materials/zh.300.vec.gz'\n",
    "performance_evaluator = PerformanceEvaluator(vec_emb_path, defence_model=fasttext_model)"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "raw_text = '我⽇你妈⼤⾎逼螺旋升天'\n",
    "out_lines = [\n",
    "    '我⽇你妈⼤⾎逼螺旋升天',\n",
    "    '我⽇你ma⼤⾎逼螺旋升天',\n",
    "    '我⽇你Ma⼤⾎逼螺旋升天',\n",
    "]\n",
    "refs = [raw_text] * len(out_lines)\n",
    "\n",
    "soft_scores, hard_scores,details = performance_evaluator.calc_final_score(refs, out_lines, show_details=True)\n",
    "for out,score,detail in zip(out_lines,soft_scores,details):\n",
    "    print(score,out)\n",
    "    print(detail)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   },
   "execution_count": 4,
   "outputs": [
    {
     "name": "stderr",
     "text": [
      "Building prefix dict from C:\\Users\\zsf\\Anaconda3\\lib\\site-packages\\jieba\\dict.txt ...\n",
      "Loading model from cache C:\\Users\\zsf\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 1.058 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ],
     "output_type": "stream"
    },
    {
     "name": "stdout",
     "text": [
      "0.011986136436462402 我⽇你妈⼤⾎逼螺旋升天\n",
      "{'normalized_levenshtein': 1.0, 'jaccard_word': 1.0, 'jaccard_char': 1.0, 'embedding_cosine': 1.0000000000000002, 'sim': 1.0, 'prob': 0.98801386, 'final_score': 0.011986136436462402}\n",
      "0.10737893160945419 我⽇你ma⼤⾎逼螺旋升天\n",
      "{'normalized_levenshtein': 0.75, 'jaccard_word': 0.7142857142857143, 'jaccard_char': 0.7, 'embedding_cosine': 0.957654560735385, 'sim': 0.8231784852131242, 'prob': 0.8695557, 'final_score': 0.10737893160945419}\n",
      "0.09611386251454525 我⽇你Ma⼤⾎逼螺旋升天\n",
      "{'normalized_levenshtein': 0.75, 'jaccard_word': 0.7142857142857143, 'jaccard_char': 0.7, 'embedding_cosine': 0.9160900992918837, 'sim': 0.8053651445944807, 'prob': 0.88065803, 'final_score': 0.09611386251454525}\n"
     ],
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "0.14897114038467407 你这⼟狗只能看着⽼⼦⽆情说出你家的悲惨身世不敢讲话，只能在家默默吃shi\n",
      "{'normalized_levenshtein': 1.0, 'jaccard_word': 1.0, 'jaccard_char': 1.0, 'embedding_cosine': 1.0000000000000002, 'sim': 1.0, 'prob': 0.85102886, 'final_score': 0.14897114038467407}\n",
      "0.6242033769116726 伱zhegou只hehg看着情说出苨家de悲惨身世敢讲话只能在家默默吃shi\n",
      "{'normalized_levenshtein': 0.5, 'jaccard_word': 0.4230769230769231, 'jaccard_char': 0.6, 'embedding_cosine': 0.9602863489326914, 'sim': 0.707705138553571, 'prob': 0.11798948, 'final_score': 0.6242033769116726}\n",
      "0.5968516024924118 伱zheg0u只hehg看着情说出苨家de悲惨身世敢讲话只能在家默默吃shi\n",
      "{'normalized_levenshtein': 0.5, 'jaccard_word': 0.4230769230769231, 'jaccard_char': 0.6, 'embedding_cosine': 0.9602863489326914, 'sim': 0.707705138553571, 'prob': 0.15663803, 'final_score': 0.5968516024924118}\n",
      "0.49380605740915284 伱zhgeo只uhghe情说看着家出苨悲de惨世身敢话讲在只能默吃家默hsi\n",
      "{'normalized_levenshtein': 0.15625, 'jaccard_word': 0.05714285714285714, 'jaccard_char': 0.6, 'embedding_cosine': 0.8566680387403334, 'sim': 0.5373602819091224, 'prob': 0.081052184, 'final_score': 0.49380605740915284}\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "raw_text = '你这⼟狗只能看着⽼⼦⽆情说出你家的悲惨身世不敢讲话，只能在家默默吃shi'\n",
    "out_lines = [\n",
    "    '你这⼟狗只能看着⽼⼦⽆情说出你家的悲惨身世不敢讲话，只能在家默默吃shi',\n",
    "    '伱zhegou只hehg看着情说出苨家de悲惨身世敢讲话只能在家默默吃shi',\n",
    "    '伱zheg0u只hehg看着情说出苨家de悲惨身世敢讲话只能在家默默吃shi',\n",
    "    '伱zhgeo只uhghe情说看着家出苨悲de惨世身敢话讲在只能默吃家默hsi'\n",
    "]\n",
    "refs = [raw_text] * len(out_lines)\n",
    "\n",
    "soft_scores, hard_scores,details = performance_evaluator.calc_final_score(refs, out_lines, show_details=True)\n",
    "for out,score,detail in zip(out_lines,soft_scores,details):\n",
    "    print(score,out)\n",
    "    print(detail)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}