te8.lua

require 'mobdebug'.start()

require 'nn'
require 'nngraph'
require 'optim'
require 'image'
local model_utils=require 'model_utils'
local mnist = require 'mnist'
require 'table_utils'

nngraph.setDebug(true)


words2omit = {["!"] = 177, ["#"] = 2930, ["$"] = 314, ["%"] = 139, ["["] = 312, ["]"] = 316, ["&"] = 482, ["'"] = 92, ["'an"] = 9189, ["'d"] = 1069, ["'i"] = 5286, ["'ll"] = 566, ["'m"] = 808, ["'re"] = 358, ["'s"] = 26, ["'t"] = 85, ["'ve"] = 555, [">"] = 730, ["<"] = 2691, ["""] = 12, ["("] = 35, [")"] = 33, ["*"] = 339, ["+"] = 795, [","]
= 3, ["-"] = 54, ["--"] = 716, ["."] = 2, [".."] = 2947, ["..."] = 247, ["...."] = 4014, [".net"] = 3428, ["/"] = 75, ["0"] = 2316, ["000"] = 4396, ["00"] = 2005, ["0.5"] = 7859, ["01"] = 6862, ["1"] = 222, ["1,000"] = 4242, ["1."] = 1300, ["1.0"] = 7528, ["1.1"] = 6694, ["1.2"] = 6832, ["1.3"] = 8018, ["1.4"] = 9711, ["1.5"] = 4156, ["1.6"] = 8774, ["1.8"] = 9963, ["1st"] = 3930, ["2"] = 238, ["2,000"]
= 6466, ["2-3"] = 6955, ["2-minute"] = 9010, ["2."] = 1229, ["2.0"] = 4407, ["2.1"] = 9735, ["2.2"] = 7247, ["2.5"] = 5633, ["2.8"] = 7673, ["2nd"] = 6812, ["3"] = 356, ["3,000"] = 7919, ["3-4"] = 8690, ["3."] = 1400, ["3.0"] = 5553, ["3.1"] = 9867, ["3.2"] = 9107, ["3.5"] = 5586, ["3.6"] = 9320, ["3d"] = 3708, ["3g"] = 9218, ["3rd"] = 6331, ["4"] = 474, ["4,000"] = 8857, ["4."] = 1881, ["4.0"] =
6230, ["4.2"] = 8779, ["4.3"] = 9825, ["4.5"] = 7668, ["4th"] = 6710, ["5"] = 400, ["5-6"] = 9930, ["5-minute"] = 5058, ["5."] = 2616, ["5.5"] = 8908, ["5th"] = 6887, ["6"] = 779, ["6."] = 2808, ["6.0"] = 8951, ["6th"] = 6605, ["7"] = 811, ["07"] = 7037, ["7,000"] = 9182, ["7."] = 3306, ["7th"] = 8414, ["8"] = 940, ["08"] = 8865, ["8."] = 4133, ["09"] = 9400, ["9"] = 1211, ["9."] = 4395, ["9th"] =
9808, ["10"] = 311, ["10,000"] = 4589, ["10-minute"] = 4643, ["10."] = 4621, ["10th"] = 7888, ["11"] = 1051, ["11."] = 6319, ["11th"] = 7332, ["12"] = 718, ["12."] = 6740, ["12th"] = 8203, ["13"] = 1338, ["13."] = 7729, ["14"] = 1368, ["14."] = 6705, ["15"] = 617, ["15,000"] = 8262, ["15-minute"] = 6130, ["15."] = 8488, ["15th"] = 7537, ["16"] = 1286, ["16."] = 7187, ["16th"] = 7028, ["17"] = 1458,
["17."] = 8341, ["17th"] = 6554, ["18"] = 1194, ["18."] = 7736, ["18th"] = 7469, ["19"] = 1847, ["19th"] = 4889, ["20"] = 519, ["20,000"] = 7848, ["20-minute"] = 8861, ["20."] = 9216, ["20th"] = 4055, ["21"] = 1777, ["21."] = 9414, ["21st"] = 6058, ["22"] = 2032, ["22."] = 9936, ["23"] = 2321, ["24"] = 1472, ["24-hour"] = 5209, ["25"] = 1254, ["26"] = 2305, ["27"] = 2377, ["28"] = 2332, ["29"] = 2638,
["30"] = 582, ["30th"] = 9768, ["31"] = 2987, ["32"] = 3178, ["32-bit"] = 7819, ["33"] = 4799, ["34"] = 5494, ["35"] = 2845, ["36"] = 5288, ["37"] = 4662, ["38"] = 5690, ["39"] = 5438, ["40"] = 1175, ["41"] = 6558, ["42"] = 4145, ["42."] = 9697, ["43"] = 8376, ["44"] = 5540, ["45"] = 3855, ["46"] = 7054, ["47"] = 5900, ["48"] = 4714, ["49"] = 5490, ["50"] = 996, ["50,000"] = 7222, ["51"] = 7446, ["52"]
= 6519, ["53"] = 7133, ["54"] = 9291, ["55"] = 5993, ["56"] = 5833, ["57"] = 6381, ["58"] = 6779, ["59"] = 8356, ["60"] = 1822, ["60th"] = 9997, ["61"] = 9440, ["62"] = 6781, ["63"] = 9032, ["64"] = 5984, ["64-bit"] = 7596, ["65"] = 5614, ["66"] = 7568, ["68"] = 6129, ["69"] = 9519, ["70"] = 2542, ["72"] = 8836, ["75"] = 4078, ["76"] = 8804, ["78"] = 7972, ["80"] = 1894, ["83"] = 7869, ["85"] = 6894,
["90"] = 2527, ["92"] = 9050, ["94"] = 8565, ["95"] = 4990, ["96"] = 8489, ["98"] = 5281, ["99"] = 8778, ["100"] = 757, ["100,000"] = 6335, ["110"] = 7532, ["120"] = 4256, ["130"] = 9678, ["140"] = 6998, ["150"] = 3456, ["160"] = 8775, ["180"] = 8084, ["200"] = 1953, ["250"] = 4376, ["300"] = 2809, ["360"] = 8207, ["365"] = 9283, ["400"] = 3071, ["450"] = 7880, ["500"] = 1989, ["500,000"] = 9346,
["600"] = 4852, ["700"] = 5512, ["800"] = 4509, ["900"] = 7214, ["1000"] = 3969, ["1200"] = 9407, ["1500"] = 8003, ["1900"] = 9699, ["1917"] = 8069, ["1918"] = 8430, ["1919"] = 9981, ["1920"] = 7796, ["1922"] = 9756, ["1925"] = 8751, ["1929"] = 8835, ["1930s"] = 8611, ["1937"] = 8904, ["1940"] = 7676, ["1941"] = 8034, ["1943"] = 8274, ["1944"] = 6502, ["1945"] = 6637, ["1946"] = 9096, ["1948"] = 6881,
["1949"] = 8843, ["1950"] = 6991, ["1950s"] = 8124, ["1952"] = 9774, ["1953"] = 8193, ["1954"] = 9574, ["1955"] = 8319, ["1956"] = 8504, ["1957"] = 8701, ["1958"] = 8024, ["1959"] = 8628, ["1960"] = 7095, ["1960s"] = 7775, ["1964"] = 9215, ["1965"] = 9921, ["1967"] = 7714, ["1968"] = 7883, ["1969"] = 9575, ["1970"] = 6572, ["1970s"] = 5587, ["1971"] = 8164, ["1972"] = 6125, ["1973"] = 5938, ["1974"]
= 8507, ["1975"] = 7589, ["1976"] = 7760, ["1977"] = 7376, ["1978"] = 6032, ["1979"] = 5246, ["1980"] = 6387, ["1980s"] = 5228, ["1981"] = 8072, ["1982"] = 7650, ["1983"] = 8897, ["1984"] = 6612, ["1985"] = 5899, ["1986"] = 5889, ["1987"] = 7971, ["1988"] = 4274, ["1989"] = 4890, ["1990"] = 3550, ["1990s"] = 3792, ["1991"] = 2971, ["1992"] = 3037, ["1993"] = 3648, ["1994"] = 3106, ["1995"] = 2850,
["1996"] = 2459, ["1997"] = 2793, ["1998"] = 2204, ["1999"] = 1714, ["2000"] = 978, ["2001"] = 1475, ["2002"] = 1406, ["2003"] = 698, ["2004"] = 1020, ["2005"] = 761, ["2006"] = 688, ["2007"] = 398, ["2008"] = 411, ["2009"] = 585, ["2010"] = 686, ["2011"] = 1501, ["2012"] = 1643, ["2013"] = 3186, ["2014"] = 8081, ["2015"] = 7465, ["2020"] = 8777, ["3000"] = 7536, ["5000"] = 9966, [":"] = 36, [";"]
= 90, ["="] = 3533, ["?"] = 65, ["@"] = 4342, ["\\"] = 1674, ["^"] = 9971, _ = 512, a = 8, ["a."] = 2369, ["a.m."] = 6769}
function word2omit(word)
  for key, _ in pairs(words2omit) do
    if word == key then 
      return true
    end
    
  end
end


word2omit('!')