Step1_get_matched_sentences.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [Get data and top words](#data)\n",
    "- Get data from file and construct DataSet object\n",
    "- Get top words, placebo words\n",
    "\n",
    "#### [Edit sentences and get embeddings for edited sentences](#sentence_edit)\n",
    "- Edit sentences by removing top / placebo / empty words\n",
    "- Sentence embedding by concatinating last 4 layers of BERT embeddings\n",
    "\n",
    "#### [Matching sentences and calculate ITE:](ite_match)\n",
    "- Treatment match\n",
    "- Placebo match\n",
    "- Control match\n",
    "- E.g., \"This is a good movie\" matched with \"This is a bad movie\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import io, time\n",
    "from io import BytesIO\n",
    "from IPython.display import display\n",
    "import pickle, tarfile, random, re, requests\n",
    "from collections import Counter, defaultdict\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('max_colwidth', -1)\n",
    "\n",
    "import sklearn\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import torch\n",
    "from transformers import * # here introduces bert\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# from allennlp.predictors.predictor import Predictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_structure import Dataset, SentenceEdit, get_IMDB, get_kindle, get_toxic_comment, get_toxic_tw"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get data and top words <a id='data'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_vectorize(df):\n",
    "    \"\"\"\n",
    "    Vectorize text\n",
    "    \"\"\"\n",
    "    vec = CountVectorizer(min_df=5, binary=True, max_df=.8)\n",
    "    X = vec.fit_transform(df.text)\n",
    "    print(X.shape)\n",
    "    y = df.label.values\n",
    "    feats = np.array(vec.get_feature_names())\n",
    "    \n",
    "    return X, y, vec, feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_coef(clf, feats, n=10, pattern=None):\n",
    "    \"\"\"\n",
    "    sort and print words by coef stregth (abs(coef))\n",
    "    \"\"\"\n",
    "    if len(clf.classes_) == 2:\n",
    "        coefs = [-1*clf.coef_[0], clf.coef_[0]] # change the coef relation corresponding with each class\n",
    "    else:\n",
    "        coefs = clf.coef_\n",
    "\n",
    "    for label, coef in zip(clf.classes_, coefs):\n",
    "        print(\"\\nTop features for class %s\" % str(label))\n",
    "        if pattern:\n",
    "            # restrict to features matching pattern\n",
    "            coef = coef.copy()\n",
    "            coef[[i for i,s in enumerate(feats) if len(re.findall(pattern, s)) == 0]] = 0\n",
    "        \n",
    "        topi = coef.argsort()[::-1][:n]\n",
    "        s = ' '.join('%s/%.2f' % (f,c) for f, c in zip(feats[topi], coef[topi]))\n",
    "        print(s)\n",
    "        \n",
    "def get_top_terms(dataset, coef_thresh=.5, placebo_thresh=.5, C=1):\n",
    "    \"\"\"\n",
    "    Fit classifier, print top-n terms;\n",
    "    Top features (features have high coef): abs(coef) >= thresh\n",
    "    Placebos (features have low coef): abs(coef) <= thresh\n",
    "    \"\"\"\n",
    "    clf = LogisticRegression(class_weight='auto', C=C, solver='lbfgs', max_iter=1000)\n",
    "    clf.fit(dataset.X, dataset.y)\n",
    "    \n",
    "    print_coef(clf, dataset.feats, n=100)\n",
    "    #print('dummy coef= %.3f' % clf.coef_[0][dataset.vec.vocabulary_[DUMMY_TERM]])\n",
    "    \n",
    "    top_feature_idx = np.where(abs(clf.coef_[0]) >= coef_thresh)[0]\n",
    "    placebo_feature_idx = np.where(abs(clf.coef_[0]) <= placebo_thresh)[0]\n",
    "    feature_coef = np.array([float(\"%.3f\" % c) for c in clf.coef_[0]]) \n",
    "    \n",
    "    return top_feature_idx, placebo_feature_idx, feature_coef"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Edit sentences (remove top words from sentences) <a id='sentence_edit'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_wd_context(sentence, word, window=5):\n",
    "    \"\"\"\n",
    "    Remove word from sentence and return context (at most 5 tokens before and after the word)\n",
    "    E.g.,\n",
    "        word: delicious \n",
    "        sentence: Love this book full of delicious foods\n",
    "        context: Love this book full of ... foods (at most 5 (window size) left and right words)\n",
    "        \n",
    "    \"\"\"\n",
    "    word = word.lower()\n",
    "    toks = sentence.split()\n",
    "    \n",
    "    for i, t in enumerate(toks):\n",
    "        if re.search(r'(?i)\\b%s\\b' % word, t): # if find the word, then take left 5 and right 5 words\n",
    "            s = ' '.join(toks[max(0, i-window):min(i+window+1, len(toks)-1)])\n",
    "            return re.sub(r'(?i)\\b%s\\b' % word, ' ', s, re.IGNORECASE)\n",
    "        \n",
    "def remove_wd_from_sentences(X, sentences, labels, vec, remove_wd_list, exclude_sent_idx, window=5):\n",
    "    \"\"\"\n",
    "    for each word to be removed\n",
    "      for each treatment sentence (sentence containing this word)\n",
    "        make a copy of the sentence with this word removed\n",
    "        \n",
    "    remove_wd_list:\n",
    "        - top_words: abs(coef) > thresh\n",
    "        - placebo_words: abs(coef) < thresh\n",
    "    exclude_sent_idx:\n",
    "        - sentences to exclude (sentences containing placebo_words should not contain top_words)\n",
    "    \n",
    "    X: feature matrix;\n",
    "    sentences: a list of sentences containing keywords;\n",
    "    labels: sentence labels;\n",
    "    all_words: words to be removed;\n",
    "    vec: CountVectorizer;\n",
    "    window: number of tokens to keep before and after the removing word\n",
    "    \n",
    "    unk sentence: sentence with a specific word removed;\n",
    "        E.g., \"Love this book full of delicious foods\"\n",
    "              \"Love this book full of ... foods\"\n",
    "                \n",
    "    returns:\n",
    "       unk_sentences: a list of SentenceEdit objects\n",
    "       word2sentences: map from word to a list of SentenceEdit objects related with this word\n",
    "       sentence_idx: list of processed sentence ids \n",
    "       \n",
    "    \"\"\"\n",
    "    word2sentenceEdit = defaultdict(list)\n",
    "    sentenceEdit_objs = []\n",
    "    sentence_idx = []\n",
    "    \n",
    "    for word in remove_wd_list:\n",
    "        wi = vec.vocabulary_[word]\n",
    "        for sent_id in X[:,wi].nonzero()[0]: # sentences containing current word\n",
    "            if(sent_id not in exclude_sent_idx): # not contain any top words\n",
    "                wd_context = get_wd_context(sentences[sent_id], word, window) # context within window=5\n",
    "                sEdit_obj = SentenceEdit(wd_context, sent_id, word, labels[sent_id])\n",
    "                word2sentenceEdit[word].append(sEdit_obj)\n",
    "                sentenceEdit_objs.append(sEdit_obj)\n",
    "                sentence_idx.append(sent_id)\n",
    "                \n",
    "#     print('%d sentences with placebo terms\\n' % len(unk_sentences))\n",
    "\n",
    "    return sentenceEdit_objs, word2sentenceEdit, list(set(sentence_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_original_sentences(df):\n",
    "    \"\"\"\n",
    "    Original sentences without any word removed;\n",
    "    Used when control case is not removing any word;\n",
    "    \"\"\"\n",
    "    df['i_th'] = range(df.shape[0])\n",
    "    original_sentEdit = []\n",
    "    for ri, row in df.iterrows():\n",
    "        sEdit_obj = SentenceEdit(row['text'], row['i_th'], '', row['label'])\n",
    "        original_sentEdit.append(sEdit_obj)\n",
    "        \n",
    "    return original_sentEdit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### BERT embeddings for edited sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embed_one_sentence(sentence, sentence_model, tokenizer):\n",
    "    \"\"\"\n",
    "    # bert_tokenizer.vocab_size\n",
    "    # bert_tokenizer.tokenize(sentence)\n",
    "    # bert_tokenizer.convert_tokens_to_ids('on')\n",
    "    # bert_tokenizer.convert_ids_to_tokens(102)\n",
    "    # each sentence is encoded as a 3072 vec: 768 * 4 (concat last four layers)\n",
    "    \"\"\"\n",
    "    with torch.no_grad():\n",
    "        # sentence_model returns (logit output layer, pooler_output, hidden states, attentions)\n",
    "        hidden_states = sentence_model(torch.tensor([tokenizer.encode(sentence, add_special_tokens=True)]))[2]\n",
    "        #last_four_layers = [hidden_states[i] for i in (-1, -2, -3, -4)]\n",
    "        last_four_layers = [hidden_states[i] for i in (0,1,2,3)] # list of 4 element, each element is [1,16,768]\n",
    "        # cast layers to a tuple and concatenate over the last dimension\n",
    "        cat_hidden_states = torch.cat(tuple(last_four_layers), dim=-1) # [1,16,3072]\n",
    "        cat_sentence_embedding = torch.mean(cat_hidden_states, dim=1).squeeze() # 3027\n",
    "        #return(torch.mean(hidden_states[-1], dim=1).squeeze()) # average word embeddings in last layer\n",
    "        return cat_sentence_embedding.numpy()                         # average last 4 layers\n",
    "        \n",
    "        \n",
    "def embed_all_sentences(sentences, bert_tokenizer=None, sentence_model=None):\n",
    "    \"\"\"\n",
    "    sentences: a list of SentenceEdit objects;\n",
    "    Each sentence is a SentenceEdit object;\n",
    "    Add embedding attribute for SentenceEdit object;\n",
    "    \"\"\"\n",
    "    if not bert_tokenizer:\n",
    "        bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "        sentence_model = BertModel.from_pretrained('bert-base-uncased',output_hidden_states=True, output_attentions=True)\n",
    "    \n",
    "    for s in tqdm(sentences):\n",
    "        s.embedding = embed_one_sentence(s.context, sentence_model, bert_tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run experiment for: get data, edit sentences, get embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset_embeddings():\n",
    "    \"\"\"\n",
    "    1. Get data from file and construct DataSet object\n",
    "    2. Get top words, placebo words\n",
    "    3. Edit sentences and get embeddings for edited sentences:\n",
    "        Remove top words;\n",
    "        Remove placebo words;\n",
    "        Original sentences without edit;\n",
    "    \"\"\"\n",
    "    random.seed(42)\n",
    "    datasets = []\n",
    "    for get_data_df, moniker, coef_thresh, placebo_thresh in [\n",
    "            (get_IMDB, 'imdb', 1.0, 0.1),\n",
    "            (get_kindle, 'kindle', 0.9, 0.2),\n",
    "            (get_toxic_comment, 'toxic', 1.0, 0.05), \n",
    "            (get_toxic_tw, 'toxic_tw', 0.7, 0.2),\n",
    "        ]: \n",
    "        \n",
    "        # Get data and show basic information\n",
    "        df = get_data_df()\n",
    "        X, y, vec, feats = simple_vectorize(df) # vectorize text\n",
    "        ds = Dataset(X, y, vec, df, moniker) # construct dataset object\n",
    "        \n",
    "        print('%s dataset, %d instances' % (moniker,len(df)))\n",
    "        print('Label distribution: %s' % str(Counter(y).items()))\n",
    "        print('Feature matrix: %s' % str(X.shape))\n",
    "        \n",
    "        # Get top features \n",
    "        ds.top_feature_idx, ds.placebo_feature_idx, ds.coef = get_top_terms(ds, coef_thresh=coef_thresh, placebo_thresh=placebo_thresh, C=1)\n",
    "        ds.top_features = feats[ds.top_feature_idx]\n",
    "        ds.placebo_features = feats[ds.placebo_feature_idx]\n",
    "        print('\\n%d top terms: %d pos, %d neg\\n' % (len(ds.top_features), len(np.where(ds.coef[ds.top_feature_idx]>0)[0]), len(np.where(ds.coef[ds.top_feature_idx]<0)[0])))\n",
    "        print('\\n%d placebo terms: %d pos, %d neg\\n' % (len(ds.placebo_features), len(np.where(ds.coef[ds.placebo_feature_idx]>0)[0]), len(np.where(ds.coef[ds.placebo_feature_idx]<0)[0])))\n",
    "        \n",
    "        # Edit sentences (remove top features, placebo features, keep as it is) and get embedding representations\n",
    "        \n",
    "#         print('Edit sentences containing top words')                                                                   \n",
    "        ds.topwd_sentObj_list, ds.topwd_sentObj_dict, topwd_sent_idx = remove_wd_from_sentences(X,df.text,df.label,vec,ds.top_features,exclude_sent_idx=[])\n",
    "        print('Edit %d sentences for top terms\\n' % len(ds.topwd_sentObj_list))\n",
    "        embed_all_sentences(ds.topwd_sentObj_list) \n",
    "\n",
    "#         print('Edit sentences containing placebo words')     \n",
    "        ds.placebowd_sentObj_list, ds.placebowd_sentObj_dict, placebowd_sent_idx = remove_wd_from_sentences(X,df.text,df.label,vec,ds.placebo_features,topwd_sent_idx)\n",
    "        print('Edit %d sentences for placebo terms\\n' % len(ds.placebowd_sentObj_list))\n",
    "        embed_all_sentences(ds.placebowd_sentObj_list) \n",
    "        \n",
    "#         print('Original sentences convert to sentence edit object')\n",
    "        ds.original_sentObj_list = get_original_sentences(df)\n",
    "        print('%d original sentences')\n",
    "        embed_all_sentences(ds.original_sentObj_list)    \n",
    "        \n",
    "        datasets.append(ds)\n",
    "        \n",
    "    return datasets\n",
    "\n",
    "# pickle.dump(datasets, open('/data/zwang/2020_S/Toxic/Concat_last4_emb/data_with_placebo_match/datasets_emb.pickle','wb'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10662, 4574)\n",
      "new dataset with 10662 records\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1</td>\n",
       "      <td>simplistic , silly and tedious .</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                              text\n",
       "0 -1      simplistic , silly and tedious ."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "imdb dataset, 10662 instances\n",
      "Label distribution: dict_items([(-1, 5331), (1, 5331)])\n",
      "Feature matrix: (10662, 4574)\n",
      "\n",
      "Top features for class -1\n",
      "boring/2.15 bore/2.11 dull/2.06 supposed/2.00 fails/1.96 badly/1.89 unless/1.79 routine/1.77 waste/1.77 mindless/1.76 pie/1.75 junk/1.74 plodding/1.74 unfunny/1.71 neither/1.68 flat/1.68 worst/1.68 generic/1.66 stupid/1.62 ill/1.62 too/1.62 suffers/1.58 disappointment/1.58 incoherent/1.58 intentions/1.58 wasn/1.54 superficial/1.54 mediocre/1.53 devoid/1.50 disguise/1.49 exhausting/1.48 uninspired/1.48 stunt/1.48 product/1.47 stale/1.46 propaganda/1.45 animal/1.44 tv/1.43 schneider/1.43 mess/1.43 lacking/1.43 benigni/1.43 artificial/1.42 sadly/1.41 tedious/1.41 inept/1.38 uneasy/1.38 lack/1.36 pretentious/1.33 hasn/1.33 sheridan/1.33 seagal/1.32 already/1.32 pointless/1.30 poorly/1.30 bland/1.30 barely/1.30 unfortunately/1.29 conceived/1.29 lifeless/1.29 choppy/1.28 god/1.28 unintentionally/1.28 college/1.27 pathetic/1.27 bits/1.26 none/1.26 banal/1.26 thin/1.25 lousy/1.25 overwrought/1.25 named/1.25 insights/1.25 demme/1.24 frankly/1.24 dreary/1.24 shoot/1.24 plain/1.24 apparent/1.23 thinks/1.23 laughable/1.22 heavy/1.22 save/1.22 lacks/1.21 mildly/1.21 soggy/1.21 six/1.20 requires/1.20 wilder/1.20 problem/1.20 pulp/1.19 lower/1.18 jokes/1.18 51/1.18 eager/1.18 dramatically/1.17 joyless/1.17 scattered/1.17 embarrassment/1.17 failure/1.17\n",
      "\n",
      "Top features for class 1\n",
      "unexpected/2.03 engrossing/2.02 powerful/1.92 count/1.87 entertain/1.84 enjoyable/1.82 provides/1.81 wonderful/1.80 smarter/1.76 skin/1.74 refreshing/1.67 remarkable/1.63 imax/1.60 glorious/1.59 unique/1.56 gem/1.53 entertaining/1.52 treat/1.51 works/1.49 solid/1.48 delightful/1.48 spider/1.48 unflinching/1.47 hilarious/1.47 secretary/1.46 ages/1.46 cinema/1.43 warm/1.43 resist/1.43 grown/1.40 freedom/1.40 captures/1.39 refreshingly/1.39 sexual/1.38 thornberrys/1.38 riveting/1.37 breathtaking/1.37 animated/1.37 ingenious/1.37 touching/1.36 polished/1.35 jealousy/1.35 affection/1.34 masterpiece/1.34 bride/1.33 smart/1.33 pulls/1.32 frailty/1.32 delicate/1.32 unpretentious/1.30 culture/1.30 winning/1.30 delivers/1.30 respect/1.28 history/1.28 chilling/1.27 sly/1.27 debate/1.26 witty/1.26 inventive/1.25 strength/1.25 smith/1.25 help/1.25 soulful/1.25 heartwarming/1.24 rare/1.23 deliciously/1.23 son/1.22 pray/1.21 flaws/1.21 savvy/1.20 exhilarating/1.19 evocative/1.19 still/1.18 conduct/1.18 brilliant/1.18 mesmerizing/1.17 insomnia/1.17 heart/1.17 effective/1.16 wonderfully/1.16 marvel/1.16 pacino/1.16 dots/1.15 bringing/1.15 terrific/1.15 loved/1.15 denying/1.14 thanks/1.14 subversive/1.14 constructed/1.14 strange/1.14 rhythm/1.14 beauty/1.13 literate/1.13 department/1.13 stylistic/1.13 assured/1.13 tasty/1.12 comfortable/1.12\n",
      "\n",
      "366 top terms: 173 pos, 193 neg\n",
      "\n",
      "\n",
      "626 placebo terms: 306 pos, 317 neg\n",
      "\n",
      "Edit 8882 sentences for top terms\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9cf2df1e302452ca318ce1e85e1c168",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8882.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Edit 12996 sentences for placebo terms\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17cfd098d2944f708eacea12db7eb076",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=12996.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "%d original sentences\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f378810be82408f833469b044cbdcb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10662.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(12000, 3390)\n",
      "new dataset with 12000 records\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>this book jumped around so much, it was confusing at certain parts, did not make since, I skimmed through it, did not enjoy</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                          text  \\\n",
       "0  this book jumped around so much, it was confusing at certain parts, did not make since, I skimmed through it, did not enjoy   \n",
       "\n",
       "   rating  label  \n",
       "0  2      -1      "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kindle dataset, 12000 instances\n",
      "Label distribution: dict_items([(-1, 6000), (1, 6000)])\n",
      "Feature matrix: (12000, 3390)\n",
      "\n",
      "Top features for class -1\n",
      "waste/2.18 disappointing/2.10 poorly/1.96 deleted/1.91 flat/1.90 weird/1.85 lacks/1.84 stupid/1.78 bored/1.78 religious/1.77 boring/1.70 unless/1.62 sorry/1.60 unrealistic/1.57 not/1.57 silly/1.57 disappointment/1.49 unfortunately/1.48 lost/1.45 wasted/1.43 title/1.43 shallow/1.40 finish/1.40 99/1.40 okay/1.39 worst/1.38 nothing/1.38 mildly/1.36 didnt/1.35 didn/1.33 skip/1.33 free/1.32 dialog/1.32 call/1.31 weak/1.31 chicken/1.29 standard/1.27 lacking/1.27 looked/1.26 basically/1.26 blah/1.25 lacked/1.24 rush/1.24 paid/1.24 lack/1.23 horrible/1.22 english/1.21 star/1.21 lame/1.21 hopes/1.20 print/1.19 desired/1.19 15/1.19 stock/1.19 cinderella/1.18 pat/1.17 sick/1.17 unbelievable/1.16 annoying/1.14 poor/1.13 impressed/1.13 quit/1.13 potential/1.13 disgusting/1.13 grammar/1.11 decent/1.11 chapters/1.10 captivating/1.10 chapter/1.09 luck/1.09 tried/1.09 girl/1.09 selfish/1.08 incomplete/1.08 talking/1.08 tired/1.07 tells/1.07 catherine/1.06 sounded/1.06 fleshed/1.06 online/1.06 childish/1.06 teenage/1.06 seemed/1.06 gross/1.05 ugh/1.05 marriage/1.05 rushed/1.04 maybe/1.04 terrible/1.04 don/1.04 pages/1.04 working/1.04 premise/1.03 garbage/1.02 cheated/1.02 let/1.02 ok/1.02 none/1.01 then/1.01\n",
      "\n",
      "Top features for class 1\n",
      "amazing/2.50 loved/2.14 enjoyed/2.07 wait/2.07 great/1.95 keeps/1.86 disappoint/1.75 fantastic/1.72 awesome/1.62 enjoyable/1.58 hot/1.58 wedding/1.57 fun/1.55 informative/1.52 wow/1.52 liked/1.50 refreshing/1.50 excellent/1.49 love/1.48 highly/1.48 loves/1.39 brings/1.38 glad/1.37 wonderful/1.37 forward/1.37 favorite/1.36 useful/1.34 turner/1.34 hooked/1.32 delightful/1.31 easy/1.30 us/1.28 guide/1.28 twists/1.28 seat/1.27 cry/1.26 super/1.25 awaiting/1.24 damn/1.20 beautiful/1.20 installment/1.19 engaging/1.18 nicely/1.18 must/1.17 worked/1.16 indeed/1.16 good/1.16 fascinating/1.16 stop/1.14 hilarious/1.13 lucas/1.13 intrigue/1.12 lol/1.12 night/1.11 required/1.11 live/1.10 penny/1.10 perfect/1.10 picked/1.08 complaint/1.08 fabulous/1.07 unusual/1.06 heroines/1.06 bedtime/1.06 right/1.06 tough/1.06 down/1.06 heard/1.05 full/1.05 anxiously/1.05 perfectly/1.05 improve/1.05 house/1.04 benefits/1.04 surprised/1.04 sister/1.04 humorous/1.04 stands/1.03 sweet/1.03 helped/1.02 stopping/1.02 caught/1.02 questions/1.02 humans/1.02 includes/1.01 joy/1.01 funny/1.01 kept/1.01 rounded/1.00 solid/1.00 fairly/1.00 tho/1.00 asap/1.00 lovely/0.98 considering/0.97 superb/0.97 strategies/0.97 available/0.97 regardless/0.97 mouse/0.97\n",
      "\n",
      "272 top terms: 129 pos, 143 neg\n",
      "\n",
      "\n",
      "1157 placebo terms: 580 pos, 576 neg\n",
      "\n",
      "Edit 32734 sentences for top terms\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45f16c19b7f84467b7e743c587d4c7ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=32734.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Edit 5243 sentences for placebo terms\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dcbb6b6736694bdb97094f2502d40d54",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5243.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "%d original sentences\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1d15ed7494446eab827e83fc93f5f2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=12000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(20872, 13025)\n",
      "new dataset with 20872 records\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6048366</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>Zuma is nothing if we compare him with ramaphosa and rupert, black people struggling till now in their own own country and rupert and wmc are busy in looting from our country.</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id    target  \\\n",
       "0  6048366  0.428571   \n",
       "\n",
       "                                                                                                                                                                              text  \\\n",
       "0  Zuma is nothing if we compare him with ramaphosa and rupert, black people struggling till now in their own own country and rupert and wmc are busy in looting from our country.   \n",
       "\n",
       "   label  \n",
       "0 -1      "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "toxic dataset, 20872 instances\n",
      "Label distribution: dict_items([(-1, 10436), (1, 10436)])\n",
      "Feature matrix: (20872, 13025)\n",
      "\n",
      "Top features for class -1\n",
      "meaningful/1.57 card/1.55 irrational/1.44 recognize/1.41 ndp/1.40 quarters/1.38 campus/1.34 gain/1.32 superior/1.32 severe/1.30 exceptions/1.30 honored/1.30 adding/1.27 figures/1.26 46/1.26 market/1.26 india/1.25 items/1.25 103/1.24 editorial/1.24 spin/1.22 hunting/1.21 reduced/1.20 helicopters/1.20 confused/1.19 censorship/1.18 evident/1.18 niqab/1.16 july/1.14 diminish/1.14 filed/1.14 mps/1.14 theirs/1.13 planes/1.12 reject/1.12 priorities/1.11 trusting/1.11 kettle/1.11 denominations/1.10 opposing/1.10 virtue/1.09 accurately/1.09 despite/1.09 robert/1.09 prosecuted/1.09 physically/1.09 internal/1.09 911/1.08 reform/1.08 shariah/1.07 fuel/1.07 plight/1.06 william/1.05 supporter/1.05 rail/1.05 colorado/1.05 prevalent/1.05 holocaust/1.05 understanding/1.05 happening/1.05 helpless/1.04 bravery/1.04 blackberry/1.04 totalitarian/1.04 ben/1.03 clever/1.03 1984/1.02 casting/1.02 stronger/1.02 discussions/1.02 closer/1.02 boston/1.02 courses/1.01 profiling/1.01 trend/1.01 describes/1.01 denouncing/1.01 paris/1.00 reported/1.00 needlessly/1.00 conflict/1.00 speed/0.99 trade/0.99 performance/0.99 stores/0.99 practiced/0.99 eating/0.99 struggling/0.98 rhyner/0.98 maximum/0.98 loose/0.98 supply/0.98 transgenderism/0.98 sullivan/0.98 blessing/0.97 histories/0.97 slowing/0.96 anonymous/0.96 stolen/0.96 contradictions/0.96\n",
      "\n",
      "Top features for class 1\n",
      "stupid/5.93 idiot/5.60 idiots/5.09 pathetic/4.64 stupidity/4.52 crap/4.49 idiotic/4.06 hypocrite/3.94 damn/3.94 ignorant/3.82 dumb/3.81 fool/3.80 fools/3.69 jerk/3.61 penis/3.58 moron/3.55 shit/3.46 ridiculous/3.44 trash/3.41 garbage/3.32 hypocrites/3.32 foolish/3.30 pig/3.16 hypocritical/3.04 silly/3.03 loser/2.88 morons/2.88 clown/2.84 ass/2.70 bitch/2.65 buffoon/2.64 clowns/2.63 troll/2.58 losers/2.55 jerks/2.55 pussy/2.50 scum/2.45 disgusting/2.43 rubbish/2.33 darn/2.30 bullshit/2.27 hypocrisy/2.26 asshole/2.25 liar/2.25 genitals/2.20 fuck/2.18 nasty/2.10 scumbag/2.09 pigs/2.05 suck/2.03 raping/2.03 idiocy/2.01 gay/2.00 imbecile/1.99 filthy/1.98 blacks/1.92 vagina/1.91 whore/1.90 lunatic/1.90 fucking/1.88 twit/1.86 useless/1.82 fag/1.80 imbeciles/1.74 balls/1.73 hell/1.72 pedophile/1.71 lunatics/1.69 homosexuals/1.68 cretin/1.66 kill/1.65 rapist/1.64 sex/1.63 ugly/1.62 niggers/1.62 liars/1.61 gays/1.61 ludicrous/1.61 screwing/1.60 dumbest/1.60 traitor/1.60 trolls/1.57 testicles/1.57 crook/1.57 ignorance/1.56 dick/1.56 insane/1.56 prostitutes/1.55 genocidal/1.54 crazy/1.54 arse/1.52 pile/1.50 scumbags/1.48 genitalia/1.47 raped/1.47 rape/1.46 nuts/1.46 asinine/1.46 stupidest/1.44 sick/1.44\n",
      "\n",
      "318 top terms: 238 pos, 80 neg\n",
      "\n",
      "\n",
      "1479 placebo terms: 701 pos, 768 neg\n",
      "\n",
      "Edit 29014 sentences for top terms\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6490a664066146328464f2eb3bb00c37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=29014.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Edit 43542 sentences for placebo terms\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2aebe23e7214c13adbc58a06ee2cd02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=43542.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "%d original sentences\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1de2c354e26b4012a84090bcdcd5aaac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=20872.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(6774, 1866)\n",
      "new dataset with 6774 records\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1105487920813821952</td>\n",
       "      <td>@FlyGuyCree Nigga whatever one you gave me 🤦🏻‍♀️</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id                                              text  \\\n",
       "0  1105487920813821952  @FlyGuyCree Nigga whatever one you gave me 🤦🏻‍♀️   \n",
       "\n",
       "   label  \n",
       "0 -1      "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "toxic_tw dataset, 6774 instances\n",
      "Label distribution: dict_items([(-1, 3186), (1, 3588)])\n",
      "Feature matrix: (6774, 1866)\n",
      "\n",
      "Top features for class -1\n",
      "season/1.70 sleep/1.58 tears/1.47 used/1.44 omg/1.44 tryna/1.40 love/1.35 voice/1.27 meet/1.25 ily/1.23 week/1.22 bruh/1.21 holy/1.20 miss/1.19 cute/1.18 2nd/1.15 reasons/1.14 happen/1.14 calling/1.14 bigger/1.13 process/1.11 draw/1.11 above/1.10 forever/1.10 sunday/1.10 jihad/1.09 summer/1.09 rn/1.09 country/1.08 ago/1.07 today/1.04 skills/1.03 mad/1.03 myself/1.03 fly/1.02 owner/1.01 cry/1.00 month/0.99 james/0.99 starting/0.97 dying/0.97 terms/0.97 killed/0.97 within/0.96 given/0.96 events/0.96 lock/0.95 adult/0.95 perfect/0.95 appreciate/0.95 chicken/0.95 children/0.95 gone/0.94 ye/0.94 birthday/0.93 thru/0.93 level/0.93 jumped/0.93 compare/0.92 bus/0.92 basketball/0.91 find/0.91 together/0.90 honestly/0.90 racism/0.90 basically/0.90 karma/0.90 reach/0.89 kinda/0.89 phone/0.88 walk/0.87 naruto/0.87 classy/0.87 test/0.87 pain/0.86 defense/0.86 wish/0.86 lmaooo/0.86 move/0.85 marriage/0.85 genius/0.85 crowd/0.85 current/0.85 asf/0.84 next/0.84 mah/0.84 plain/0.84 easier/0.83 queer/0.83 record/0.83 lives/0.83 females/0.83 ugh/0.83 loved/0.82 thank/0.82 isn/0.82 12/0.82 thing/0.82 video/0.82 ig/0.82\n",
      "\n",
      "Top features for class 1\n",
      "twat/3.13 cunt/3.02 retard/2.73 realdonaldtrump/2.42 faggot/2.18 pussy/2.07 slut/1.85 mouth/1.81 shut/1.79 stfu/1.76 hypocrite/1.69 three/1.68 scum/1.68 dumbass/1.68 penis/1.68 donaldjtrumpjr/1.66 fag/1.56 anal/1.52 idiot/1.47 lady/1.46 jihadi/1.45 handle/1.44 cum/1.44 house/1.43 hole/1.41 showing/1.41 gop/1.38 typical/1.35 ass/1.34 blocked/1.33 ugly/1.31 sucks/1.31 dick/1.30 america/1.30 stupid/1.28 fans/1.28 deleted/1.27 trash/1.27 la/1.26 lying/1.26 goofy/1.25 rat/1.25 arianagrande/1.25 corny/1.24 learn/1.23 profile/1.23 useless/1.23 citizens/1.22 fire/1.22 thinks/1.21 cock/1.20 plz/1.20 sexy/1.19 fuck/1.18 dirty/1.16 spot/1.15 realize/1.15 disgusting/1.15 wat/1.14 attractive/1.13 everyday/1.11 hoe/1.11 suck/1.10 article/1.09 nobody/1.08 indians/1.08 hahahaha/1.07 cnn/1.07 debt/1.07 car/1.06 image/1.06 face/1.05 bitch/1.04 michael/1.04 dumb/1.03 retarded/1.02 pure/1.02 off/1.02 fake/1.02 violence/1.02 text/1.00 bye/1.00 nigger/0.99 smelly/0.99 piece/0.99 yesterday/0.99 neither/0.99 light/0.98 bastard/0.98 grown/0.98 african/0.98 sarri/0.96 fortnitegame/0.96 stirring/0.95 except/0.95 big/0.95 piss/0.95 movies/0.95 ho/0.94 sleeping/0.94\n",
      "\n",
      "341 top terms: 198 pos, 143 neg\n",
      "\n",
      "\n",
      "574 placebo terms: 289 pos, 285 neg\n",
      "\n",
      "Edit 9224 sentences for top terms\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7305a844a30f4a7f8346758b09a6d508",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=9224.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Edit 5457 sentences for placebo terms\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c8ba24b61674d67aa2384e661a4cc89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5457.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "%d original sentences\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "083da339abf34887b9dcde76098ea82e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6774.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "103.93969039122264\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "datasets = get_dataset_embeddings()\n",
    "end = time.time()\n",
    "print((end-start)/60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(datasets, open('/data/zwang/2020_S/EMNLP/V_7_rerun/datasets_emb.pickle','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matching sentences and calculate ITE <a id='ite_match'></a>\n",
    "\n",
    "- Treatment match:  context_A + top_termA = context_B + top_term_B <br>\n",
    "\n",
    "- Placebo match:  context_A + top_termA = context_B + non_top_term_B <br>\n",
    "\n",
    "- Control match:  context_A + top_termA = sentence_B (not contain top_termA) <br>\n",
    "\n",
    "- Takes 6 hours to run one matching strategy for all datasets (one time run and save for future use)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_matched_sentence(t_sentObj, c_sentObj_list, min_sim=.7):\n",
    "    \"\"\"\n",
    "    For each treatment sentence (sentence contain a top word), find a matched sentence (sentences with similar contexts);\n",
    "        \n",
    "    Find the sentence with closest contexts:\n",
    "    context_A = sentence_A - word_A\n",
    "    context_B = sentence_B - word_B\n",
    "    \n",
    "    for context_A:\n",
    "        sort similarity score of (context_A, all other contexts) in descending order\n",
    "        if((sentence_A != sentence_B) and (word_A != word_B) and (cos(context_A,context_B)>0.7)):\n",
    "            context_B is a match for context_A\n",
    "    \n",
    "    diff: difference between sentence_A.label - sentence_B.label\n",
    "    \"\"\"\n",
    "    \n",
    "    # similarity between current treatment context with all other contexts\n",
    "    sims = cosine_similarity([t_sentObj.embedding],[c_sent.embedding for c_sent in c_sentObj_list])[0]\n",
    "    \n",
    "    match = None\n",
    "    match_sim = 0\n",
    "    for c_sentObj, sim in sorted(zip(c_sentObj_list, sims), key=lambda x: -x[1]): # find the first most similar match\n",
    "        if((sim >= min_sim) and (c_sentObj.sentence_idx != t_sentObj.sentence_idx) and (c_sentObj.remove_wd != t_sentObj.remove_wd) and (not re.search(r'(?i)\\b%s\\b' % t_sentObj.remove_wd, c_sentObj.context))):\n",
    "            match = c_sentObj\n",
    "            match_sim = sim        \n",
    "            break\n",
    "            \n",
    "    if match:\n",
    "        diff = (t_sentObj.label - match.label)\n",
    "    else:\n",
    "        print('no match')\n",
    "        diff = 0\n",
    "#         diff = np.nan\n",
    "\n",
    "    return diff, match_sim, match # [(match_sim, target_sentence)], [(match_sim,match)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_ites_from_matched_sentences(ds_data, matchby='treat', min_sim=0.01):\n",
    "    \"\"\"\n",
    "    For each treatment sentence (sentence contain a top word), find a matched sentence (sentences with similar contexts);\n",
    "    \n",
    "    t_sentObj_list: a list of SentenceEdit objects for sentences with top words removed\n",
    "    c_sentObj_list: a list of SentenceEdit objects for sentences with top / placebo / '' words removed\n",
    "    matchby = treat / control / placebo\n",
    "    \"\"\"\n",
    "    t_sentObj_list = ds_data.topwd_sentObj_list\n",
    "    \n",
    "    if(matchby == 'treat'):\n",
    "        c_sentObj_list = ds_data.topwd_sentObj_list\n",
    "    elif(matchby == 'placebo'):\n",
    "        c_sentObj_list = ds_data.placebowd_sentObj_list\n",
    "    elif(matchby == 'control'):\n",
    "        c_sentObj_list = ds_data.original_sentObj_list\n",
    "    \n",
    "    matched_pairs = []    \n",
    "    for t_sentObj in tqdm(t_sentObj_list):\n",
    "        ite_bylabel, sim, control_obj = find_matched_sentence(t_sentObj, c_sentObj_list, min_sim=min_sim)\n",
    "            \n",
    "        matched_pairs.append(\n",
    "            {\n",
    "                'term': t_sentObj.remove_wd,\n",
    "                'sentence_id': t_sentObj.sentence_idx,\n",
    "                'treat_obj': t_sentObj,\n",
    "                'control_obj': control_obj, # this is an object             \n",
    "                'similarity': sim,\n",
    "                'difference': t_sentObj.embedding - control_obj.embedding, \n",
    "                'ite': ite_bylabel,\n",
    "            }\n",
    "        )\n",
    "    return pd.DataFrame(matched_pairs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiments using treatment match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0504df5ff12a48ce80099a4d07ba55e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8882.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fa6fa0eed7749d7aa6a503c0bfc726c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=32734.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11e22b16d5d444a1bc4ba83bb49c2ed4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=29014.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92a88de30078494fb65f963440b25dd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=9224.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1293.4846917788188\n"
     ]
    }
   ],
   "source": [
    "# ds_imdb, ds_kindle, ds_toxic, ds_toxic_tw = datasets\n",
    "ds_imdb, ds_kindle, ds_toxic, ds_toxic_tw = pickle.load(open('/data/zwang/2020_S/EMNLP/V_7_rerun//datasets_emb.pickle','rb'))\n",
    "datasets = [ds_imdb, ds_kindle, ds_toxic, ds_toxic_tw]\n",
    "\n",
    " \n",
    "start = time.time()\n",
    "data_ites_byTreat = []\n",
    "for ds_data in datasets:\n",
    "    ds_data.ites = calculate_ites_from_matched_sentences(ds_data, matchby='treat', min_sim=0.01)\n",
    "    data_ites_byTreat.append(ds_data)\n",
    "    \n",
    "end = time.time()\n",
    "print((end-start)/60)\n",
    "pickle.dump(data_ites_byTreat, open('/data/zwang/2020_S/EMNLP/V_7_rerun/datasets_treat_match.pickle', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiments using placebo match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ds_imdb, ds_kindle, ds_toxic, ds_toxic_tw = pickle.load(open('/data/zwang/2020_S/Toxic/Concat_last4_emb/V_1_treat/datasets_emb_mindf5.pickle','rb'))\n",
    "# datasets = [ds_imdb, ds_kindle, ds_toxic, ds_toxic_tw]\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "data_ites_byPlacebo = []\n",
    "for ds_data in datasets:\n",
    "    ds_data.ites = calculate_ites_from_matched_sentences(ds_data, matchby='placebo', min_sim=0.01)\n",
    "    data_ites_byPlacebo.append(ds_data)\n",
    "    \n",
    "end = time.time()\n",
    "print((end-start)/60)\n",
    "pickle.dump(data_ites_byPlacebo, open('/data/zwang/2020_S/Toxic/Concat_last4_emb/V_3_placebo/datasets_placebo_match.pickle', 'wb'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiments using control match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ds_imdb, ds_kindle, ds_toxic, ds_toxic_tw = pickle.load(open('/data/zwang/2020_S/Toxic/Concat_last4_emb/V_1_treat/datasets_emb_mindf5.pickle','rb'))\n",
    "# datasets = [ds_imdb, ds_kindle, ds_toxic, ds_toxic_tw]\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "data_ites_byControl = []\n",
    "for ds_data in datasets:\n",
    "    ds_data.ites = calculate_ites_from_matched_sentences(ds_data, matchby='control', min_sim=0.01)\n",
    "    data_ites_byControl.append(ds_data)\n",
    "\n",
    "end = time.time()\n",
    "print((end-start)/60)\n",
    "pickle.dump(data_ites_byControl, open('/data/zwang/2020_S/Toxic/Concat_last4_emb/V_2_control/datasets_control_match.pickle', 'wb'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Manually check label correctness for Kindle\n",
    "- Sentence labels are inherited from document labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20233, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_kindle = get_kindle()\n",
    "df_kindle.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>This was a very fun story</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Not fast moving but a very well managed pace</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The story line is an interesting take on zombie mythology and is a great journey</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The story was good, but I was getting very irritated at all the grammatical and spelling errors</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>series is always a good read</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                              text  \\\n",
       "0  This was a very fun story                                                                         \n",
       "1  Not fast moving but a very well managed pace                                                      \n",
       "2  The story line is an interesting take on zombie mythology and is a great journey                  \n",
       "3  The story was good, but I was getting very irritated at all the grammatical and spelling errors   \n",
       "4  series is always a good read                                                                      \n",
       "\n",
       "   rating  label  \n",
       "0  5       1      \n",
       "1  5       1      \n",
       "2  5       1      \n",
       "3  2      -1      \n",
       "4  5       1      "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_kindle.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"This was a teaser, I can't wait for part 2\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_kindle.iloc[3648].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3648</th>\n",
       "      <td>So I do not want to sway them either way</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>I thought this was a sweet story</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9012</th>\n",
       "      <td>my favorite book is Deed of  Paksenarrion, which takes me a day or two to read</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8024</th>\n",
       "      <td>I think the author had fun writing this book, but it's the sort of food I throw together without the benefit of a recipe</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7314</th>\n",
       "      <td>You are supposed to laugh here if you loved this</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7216</th>\n",
       "      <td>I did finish it but would not reccomend it</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>However it was just too confusing</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2326</th>\n",
       "      <td>Funny, sexy and a whole lot of fun from start to finish</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1929</th>\n",
       "      <td>figured why not give the story a try</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7501</th>\n",
       "      <td>this book was way too short, plus the title gave you the thought that it would be an awesome book but it wasn't</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                          text  \\\n",
       "3648  So I do not want to sway them either way                                                                                   \n",
       "819   I thought this was a sweet story                                                                                           \n",
       "9012  my favorite book is Deed of  Paksenarrion, which takes me a day or two to read                                             \n",
       "8024  I think the author had fun writing this book, but it's the sort of food I throw together without the benefit of a recipe   \n",
       "7314  You are supposed to laugh here if you loved this                                                                           \n",
       "...                                                ...                                                                           \n",
       "7216  I did finish it but would not reccomend it                                                                                 \n",
       "235   However it was just too confusing                                                                                          \n",
       "2326  Funny, sexy and a whole lot of fun from start to finish                                                                    \n",
       "1929  figured why not give the story a try                                                                                       \n",
       "7501  this book was way too short, plus the title gave you the thought that it would be an awesome book but it wasn't            \n",
       "\n",
       "      rating  label  \n",
       "3648  1      -1      \n",
       "819   4       1      \n",
       "9012  1      -1      \n",
       "8024  2      -1      \n",
       "7314  1      -1      \n",
       "...  ..      ..      \n",
       "7216  2      -1      \n",
       "235   2      -1      \n",
       "2326  5       1      \n",
       "1929  2      -1      \n",
       "7501  2      -1      \n",
       "\n",
       "[200 rows x 3 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rand_idx = random.sample(list(df_kindle.index),200)\n",
    "df_rand = df_kindle.iloc[rand_idx]\n",
    "df_rand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_rand.to_csv('/data/zwang/2020_S/EMNLP/kindle_random_samples.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17911</th>\n",
       "      <td>Apart from that, there isn't much to the plot</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12430</th>\n",
       "      <td>A hot steamy love affair with secrets  felonies and hotties</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1319</th>\n",
       "      <td>Corrupt cop, porn girlfriend, porn producer = predictable crap triangle</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16636</th>\n",
       "      <td>The book started to get interesting and then it ended and is over</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4264</th>\n",
       "      <td>Not much depth to it</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                          text  \\\n",
       "17911  Apart from that, there isn't much to the plot                             \n",
       "12430  A hot steamy love affair with secrets  felonies and hotties               \n",
       "1319   Corrupt cop, porn girlfriend, porn producer = predictable crap triangle   \n",
       "16636  The book started to get interesting and then it ended and is over         \n",
       "4264   Not much depth to it                                                      \n",
       "\n",
       "       rating  label  \n",
       "17911  4       1      \n",
       "12430  5       1      \n",
       "1319   2      -1      \n",
       "16636  2      -1      \n",
       "4264   2      -1      "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rand.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Zhao</th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17911</td>\n",
       "      <td>-1</td>\n",
       "      <td>Apart from that, there isn't much to the plot</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12430</td>\n",
       "      <td>1</td>\n",
       "      <td>A hot steamy love affair with secrets  felonies and hotties</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1319</td>\n",
       "      <td>-1</td>\n",
       "      <td>Corrupt cop, porn girlfriend, porn producer = predictable crap triangle</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16636</td>\n",
       "      <td>-1</td>\n",
       "      <td>The book started to get interesting and then it ended and is over</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4264</td>\n",
       "      <td>-1</td>\n",
       "      <td>Not much depth to it</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Zhao  \\\n",
       "0  17911      -1      \n",
       "1  12430       1      \n",
       "2  1319       -1      \n",
       "3  16636      -1      \n",
       "4  4264       -1      \n",
       "\n",
       "                                                                      text  \\\n",
       "0  Apart from that, there isn't much to the plot                             \n",
       "1  A hot steamy love affair with secrets  felonies and hotties               \n",
       "2  Corrupt cop, porn girlfriend, porn producer = predictable crap triangle   \n",
       "3  The book started to get interesting and then it ended and is over         \n",
       "4  Not much depth to it                                                      \n",
       "\n",
       "   rating  label  \n",
       "0  4       1      \n",
       "1  5       1      \n",
       "2  2      -1      \n",
       "3  2      -1      \n",
       "4  2      -1      "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rand_labeled = pd.read_csv('/data/zwang/2020_S/EMNLP/kindle_random_samples.csv')\n",
    "df_rand_labeled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([17911, 12430,  1319, 16636,  4264])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rand_labeled['Unnamed: 0'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(484, 5)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rand_labeled[df_rand_labeled['Zhao'] == df_rand_labeled['label']].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.965, 0.032, 0.968)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "193/200, 16/500, 484/500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rand_idx_2 = random.sample(list(set(df_kindle.index) - set(df_rand_labeled['Unnamed: 0'].values)),300)\n",
    "len(rand_idx_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(rand_idx_2).intersection(set(df_rand_labeled['Unnamed: 0'].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_rand_2 = df_kindle.iloc[rand_idx_2]\n",
    "df_rand_2.to_csv('/data/zwang/2020_S/EMNLP/kindle_random_samples_2.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Out-of-vocabulary words in each train / test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = pickle.load(open('/data/zwang/2020_S/EMNLP/V_7_rerun/datasets_emb.pickle','rb'))\n",
    "ds_imdb = ds[0]\n",
    "ds_kindle = ds[1]\n",
    "ds_toxic = ds[2]\n",
    "ds_tw = ds[3]\n",
    "ds_kindle_short = pickle.load(open('/data/zwang/2020_S/EMNLP/V_6_shortSents/kindle_emb.pickle','rb'))\n",
    "ds_toxic_short = pickle.load(open('/data/zwang/2020_S/EMNLP/V_6_shortSents/toxic_emb.pickle','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10662, 3), 366)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_imdb.df.shape, len(ds_imdb.top_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((20233, 4), 270)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_kindle_short.df.shape, len(ds_kindle_short.top_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(ds_imdb.top_features).intersection(set(ds_kindle_short.top_features)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.12568306010928962"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "46/366"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((15216, 5), 329)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_toxic_short.df.shape, len(ds_toxic_short.top_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6774, 4), 341)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_tw.df.shape, len(ds_tw.top_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(ds_toxic_short.top_features).intersection(set(ds_tw.top_features)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08504398826979472"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "29/341"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python tf_bert",
   "language": "python",
   "name": "tf_bert"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}