generate_peerread_small_semi_synthetic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "from warnings import simplefilter\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "simplefilter(\"ignore\", category=ConvergenceWarning)\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation as LDA\n",
    "from sklearn.decomposition import NMF, PCA\n",
    "from sklearn.linear_model import Ridge, LogisticRegression, Lasso\n",
    "from sklearn.metrics import mean_squared_error as mse, roc_auc_score as roc, accuracy_score as acc, log_loss\n",
    "from sklearn.neural_network import MLPClassifier, MLPRegressor\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from data.dataset import BaseDataset\n",
    "from scipy.sparse import csr_matrix\n",
    "from importlib import reload\n",
    "import itertools as it\n",
    "import torch\n",
    "from model.models import SparseVAESpikeSlab, VAE\n",
    "import scipy\n",
    "from torch.nn import functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import colors\n",
    "from matplotlib.ticker import PercentFormatter\n",
    "import seaborn as sns\n",
    "from scipy.special import expit\n",
    "import os\n",
    "\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def negative_log_likelihood(logits, count):\n",
    "    prob = logits/logits.sum(axis=1)[:,np.newaxis]\n",
    "    return -(np.log(prob + 1e-7) * count).sum(axis=1).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def col_normalize(x):\n",
    "    return x/x.sum(axis=1)[:,np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_correlation_study(theta_sim, beta, true_expected, true_data, doc_n):\n",
    "    p_words = theta_sim.dot(beta)\n",
    "    expected_counts = doc_n[:,np.newaxis] * p_words\n",
    "    expected_counts = np.around(expected_counts)\n",
    "    \n",
    "    valid = np.where(expected_counts.sum(axis=1)!=0)[0]\n",
    "    expected_counts = expected_counts[valid,:]\n",
    "\n",
    "    K=beta.shape[0]\n",
    "    m = NMF(n_components=K)\n",
    "    theta_hat = m.fit_transform(expected_counts)\n",
    "    beta_hat = m.components_\n",
    "    pred_tr = theta_hat.dot(beta_hat)\n",
    "\n",
    "    npmi = get_normalized_pmi(beta_hat, true_data)\n",
    "    \n",
    "    theta_new = m.transform(true_expected)\n",
    "    pred_new = theta_new.dot(beta_hat)\n",
    "    \n",
    "    tr_mse = mse(pred_tr, expected_counts)\n",
    "    te_mse = mse(pred_new, true_expected)\n",
    "    \n",
    "    return npmi, tr_mse, te_mse, expected_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_expected_counts(theta_sim, beta, doc_n):\n",
    "    p_words = theta_sim.dot(beta)\n",
    "    expected_counts = doc_n[:,np.newaxis] * p_words\n",
    "    expected_counts = np.around(expected_counts)\n",
    "    \n",
    "    valid = np.where(expected_counts.sum(axis=1)!=0)[0]\n",
    "    expected_counts = expected_counts[valid,:]\n",
    "\n",
    "    return expected_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_normalized_pmi(topics, counts, num_words=10):\n",
    "    num_topics = topics.shape[0]\n",
    "    per_topic_npmi = np.zeros(num_topics)\n",
    "    tf = csr_matrix(counts)\n",
    "    cooccurence = tf.T.dot(tf)\n",
    "    cooccurence = cooccurence.toarray()\n",
    "    \n",
    "    count = counts.sum(axis=0)\n",
    "    prob = count/count.sum()\n",
    "    cooccurence_prob = cooccurence/cooccurence.sum()\n",
    "\n",
    "    for k in range(num_topics):\n",
    "        npmi_total = 0\n",
    "        beta = topics[k,:]\n",
    "        top_words = (-beta).argsort()[:num_words]\n",
    "        n = 0 \n",
    "        for (w1, w2) in it.combinations(top_words, 2):\n",
    "            log_joint_prob = np.log(cooccurence_prob[w1][w2]+1e-7)\n",
    "            log_prob_w1 = np.log(prob[w1]+1e-7)\n",
    "            log_prob_w2 = np.log(prob[w2]+1e-7)\n",
    "            pmi = log_joint_prob - (log_prob_w1+log_prob_w2)\n",
    "            normalizer = -log_joint_prob\n",
    "            npmi_total += pmi/normalizer\n",
    "            n+=1\n",
    "        per_topic_npmi[k] = npmi_total / n\n",
    "    return per_topic_npmi.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_simulated_theta(sigma, theta):\n",
    "    n_col = theta.shape[1]//2\n",
    "    log_theta = np.log(theta)    ##logits\n",
    "    log_theta_real = log_theta[:,:n_col]\n",
    "    log_theta_simulated = np.zeros((log_theta.shape[0], n_col))\n",
    "    for k in range(n_col):\n",
    "        noise = (sigma*np.random.randn(log_theta.shape[0]))\n",
    "        log_theta_simulated[:,k] = log_theta[:,k] + noise\n",
    "    \n",
    "    log_theta_corr = np.hstack([log_theta_real, log_theta_simulated])\n",
    "    theta_corr = col_normalize(np.exp(log_theta_corr))\n",
    "    \n",
    "    violations = 0\n",
    "    cov = np.cov(theta_corr.T)\n",
    "    diag = np.diag(cov)\n",
    "    for k in range(diag.shape[0]):\n",
    "        max_cov = np.max(cov[k, :])\n",
    "        if max_cov != diag[k]:\n",
    "            violations+=1\n",
    "    print(\"No. violations of cov condition:\", violations)\n",
    "    \n",
    "    return theta_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((11777, 500), (500,))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proc_file = '../../dat/proc/peerread_small_proc.npz'\n",
    "arr = np.load(proc_file)\n",
    "data = arr['data']\n",
    "vocab = arr['metadata']\n",
    "terms_total = data.sum(axis=1)\n",
    "data.shape, vocab.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((11777, 20), (20, 500))"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrained_file = '../../dat/proc/peerread_small_pretraining.npz'\n",
    "arr = np.load(pretrained_file)\n",
    "theta = arr['theta']\n",
    "beta = arr['beta']\n",
    "theta.shape, beta.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 0: ['user', 'feature', 'task', 'online', 'item', 'group', 'method', 'based', 'approach', 'propose', 'detection', 'selection', 'signal', 'system', 'proposed', 'information', 'preference', 'problem', 'paper', 'model']\n",
      "Topic 1: ['system', 'event', 'process', 'paper', 'information', 'ontology', 'application', 'tool', 'web', 'present', 'knowledge', 'approach', 'software', 'research', 'used', 'ha', 'development', 'semantic', 'based', 'domain']\n",
      "Topic 2: ['graph', 'constraint', 'program', 'programming', 'set', 'problem', 'solver', 'node', 'map', 'variable', 'structure', 'algorithm', 'show', 'model', 'instance', 'technique', 'paper', 'approach', 'solving', 'present']\n",
      "Topic 3: ['method', 'function', 'algorithm', 'problem', 'matrix', 'data', 'learning', 'kernel', 'optimization', 'linear', 'proposed', 'sparse', 'approach', 'propose', 'show', 'regression', 'paper', 'point', 'result', 'loss']\n",
      "Topic 4: ['model', 'learning', 'data', 'machine', 'topic', 'approach', 'latent', 'framework', 'application', 'modeling', 'unsupervised', 'ha', 'generative', 'supervised', 'algorithm', 'technique', 'structure', 'paper', 'demonstrate', 'learn']\n",
      "Topic 5: ['problem', 'algorithm', 'solution', 'search', 'optimal', 'time', 'approach', 'planning', 'decision', 'heuristic', 'value', 'paper', 'cost', 'solving', 'show', 'result', 'ha', 'optimization', 'number', 'strategy']\n",
      "Topic 6: ['domain', 'target', 'metric', 'search', 'distance', 'source', 'data', 'similarity', 'learning', 'space', 'different', 'method', 'measure', 'function', 'propose', 'result', 'procedure', 'new', 'two', 'using']\n",
      "Topic 7: ['classification', 'feature', 'classifier', 'method', 'data', 'clustering', 'image', 'cluster', 'set', 'using', 'class', 'result', 'accuracy', 'approach', 'proposed', 'algorithm', 'object', 'time', 'based', 'paper']\n",
      "Topic 8: ['representation', 'document', 'vector', 'query', 'word', 'semantic', 'method', 'embedding', 'object', 'space', 'model', 'similarity', 'information', 'feature', 'task', 'term', 'text', 'show', 'database', 'result']\n",
      "Topic 9: ['algorithm', 'bound', 'problem', 'learning', 'show', 'regret', 'stochastic', 'n', 'setting', 'result', 'online', 'rate', 'sample', 'function', 'convergence', 'gradient', 'complexity', 'number', 'lower', 'k']\n",
      "Topic 10: ['network', 'neural', 'model', 'deep', 'layer', 'architecture', 'convolutional', 'recurrent', 'input', 'task', 'performance', 'show', 'memory', 'result', 'using', 'image', 'trained', 'feature', 'propose', 'proposed']\n",
      "Topic 11: ['language', 'translation', 'model', 'machine', 'corpus', 'natural', 'system', 'task', 'text', 'english', 'sentence', 'method', 'using', 'human', 'processing', 'show', 'evaluation', 'result', 'annotation', 'paper']\n",
      "Topic 12: ['model', 'inference', 'network', 'distribution', 'probability', 'bayesian', 'variable', 'probabilistic', 'method', 'belief', 'conditional', 'algorithm', 'structure', 'show', 'parameter', 'paper', 'markov', 'uncertainty', 'approximate', 'state']\n",
      "Topic 13: ['image', 'text', 'relation', 'entity', 'model', 'task', 'visual', 'knowledge', 'semantic', 'video', 'approach', 'dataset', 'extraction', 'method', 'information', 'description', 'propose', 'concept', 'datasets', 'novel']\n",
      "Topic 14: ['data', 'analysis', 'social', 'information', 'sentiment', 'technique', 'different', 'method', 'study', 'set', 'mining', 'paper', 'review', 'approach', 'using', 'ha', 'attribute', 'used', 'pattern', 'research']\n",
      "Topic 15: ['word', 'model', 'speech', 'sequence', 'sentence', 'task', 'embeddings', 'recognition', 'approach', 'context', 'language', 'system', 'using', 'paper', 'two', 'based', 'result', 'performance', 'show', 'feature']\n",
      "Topic 16: ['rule', 'tree', 'decision', 'fuzzy', 'model', 'based', 'system', 'proposed', 'structure', 'algorithm', 'used', 'method', 'paper', 'result', 'using', 'hierarchical', 'dependency', 'approach', 'performance', 'order']\n",
      "Topic 17: ['logic', 'question', 'knowledge', 'theory', 'reasoning', 'set', 'answer', 'paper', 'property', 'semantics', 'new', 'notion', 'base', 'one', 'framework', 'result', 'belief', 'also', 'problem', 'measure']\n",
      "Topic 18: ['agent', 'learning', 'action', 'game', 'policy', 'environment', 'reinforcement', 'human', 'task', 'state', 'robot', 'control', 'behavior', 'approach', 'system', 'reward', 'model', 'learn', 'using', 'simulation']\n",
      "Topic 19: ['training', 'learning', 'method', 'deep', 'network', 'label', 'performance', 'large', 'parameter', 'show', 'accuracy', 'gradient', 'approach', 'number', 'algorithm', 'time', 'example', 'problem', 'size', 'neural']\n",
      "NPMI: 0.036797887316803536\n"
     ]
    }
   ],
   "source": [
    "for k in range(beta.shape[0]):\n",
    "    top_words = (-beta[k]).argsort()[:20]\n",
    "    topic_words = [vocab[t] for t in top_words]\n",
    "    print('Topic {}: {}'.format(k, topic_words))\n",
    "\n",
    "print(\"NPMI:\", get_normalized_pmi(beta, data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta = col_normalize(theta)\n",
    "true_p_words = theta.dot(beta)\n",
    "\n",
    "true_expected = terms_total[:,np.newaxis] * true_p_words\n",
    "true_expected = np.around(true_expected)\n",
    "valid = np.where(true_expected.sum(axis=1)!=0)[0]\n",
    "true_expected = true_expected[valid, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on experiment 0\n",
      "Working on sigma 0.5 ...\n",
      "No. violations of cov condition: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/gem/opt/anaconda3/envs/sparse-vae/lib/python3.7/site-packages/sklearn/decomposition/_nmf.py:1091: ConvergenceWarning: Maximum number of iterations 200 reached. Increase it to improve convergence.\n",
      "  \" improve convergence.\" % max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NPMI, Training loss, intervened data loss: 0.015518007274422232 0.005455749828197064 0.011101625423968116\n",
      "Working on sigma 1.0 ...\n",
      "No. violations of cov condition: 0\n",
      "NPMI, Training loss, intervened data loss: 0.022688744095815037 0.006191162399490585 0.0079658154641498\n",
      "Working on sigma 2.0 ...\n",
      "No. violations of cov condition: 0\n",
      "NPMI, Training loss, intervened data loss: 0.02921255305518599 0.006422207465129802 0.007144053227221853\n"
     ]
    }
   ],
   "source": [
    "for i in range(1):\n",
    "    print(\"Working on experiment\", i)\n",
    "    sigmas = [0.5, 1.0, 2.0]\n",
    "    for sigma in sigmas:\n",
    "        print(\"Working on sigma\", sigma, \"...\")\n",
    "        theta_sim = make_simulated_theta(sigma, theta)\n",
    "\n",
    "        npmi, tr_mse, te_mse, sim_data = run_correlation_study(theta_sim, beta, true_expected, data, terms_total)\n",
    "        \n",
    "        outdir = '../dat/intervened_n=5/' + str(i) + '/'\n",
    "        os.makedirs(outdir, exist_ok=True)\n",
    "        fname = outdir + '/sigma=' + str(round(sigma,2))\n",
    "        np.savez_compressed(fname, theta_sim=theta_sim, sim_obs=sim_data, orig_obs=true_expected, features=vocab)\n",
    "\n",
    "        print(\"NPMI, Training loss, intervened data loss:\", npmi, tr_mse, te_mse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on experiment 0\n",
      "Working on sigma 0.5 ...\n",
      "No. violations of cov condition: 0\n",
      "No. violations of cov condition: 0\n",
      "Working on sigma 1.0 ...\n",
      "No. violations of cov condition: 0\n",
      "No. violations of cov condition: 0\n",
      "Working on sigma 3.0 ...\n",
      "No. violations of cov condition: 0\n",
      "No. violations of cov condition: 0\n"
     ]
    }
   ],
   "source": [
    "for i in range(1):\n",
    "    print(\"Working on experiment\", i)\n",
    "    sigmas = [0.5, 1.0, 3.0]\n",
    "    for sigma in sigmas:\n",
    "        print(\"Working on sigma\", sigma, \"...\")\n",
    "        tr_theta_sim = make_simulated_theta(sigma, theta)\n",
    "        te_theta_sim = make_simulated_theta(sigma, theta)\n",
    "        \n",
    "        tr_data = make_expected_counts(tr_theta_sim, beta, terms_total)\n",
    "        te_data = make_expected_counts(te_theta_sim, beta, terms_total)\n",
    "        \n",
    "        outdir = '../../dat/intervened_holdout' \n",
    "        os.makedirs(outdir, exist_ok=True)\n",
    "        fname = outdir + '/sigma=' + str(round(sigma,2))\n",
    "        np.savez_compressed(fname, theta_sim=te_theta_sim, sim_obs=tr_data, orig_obs=te_data, features=vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}