LF_MATRIX.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import types\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lfs.helper import *\n",
    "from lfs.lf_categorical import *\n",
    "from lfs.lf_extration import *\n",
    "from lfs.lf_numerical import *\n",
    "from lfs.lf_unusable import *\n",
    "from lfs.lf_mixed_class import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading files data_for_labelling4.xlsx\n",
      "reading files data_for_labelling8.xlsx\n",
      "reading files data_for_labelling9.xlsx\n",
      "reading files data_for_labelling5.xlsx\n",
      "reading files data_for_labelling3.xlsx\n",
      "reading files data_for_labelling2.csv.xlsx\n",
      "reading files data_for_labelling6 (1) (1).xlsx\n",
      "reading files data_for_labelling1.csv.xlsx\n",
      "reading files data_for_labelling0.csv.xlsx\n",
      "reading files data_for_labelling7.xlsx\n"
     ]
    }
   ],
   "source": [
    "number_of_files = 1000\n",
    "base_dir = 'data/labels/'\n",
    "ext = '.xlsx'\n",
    "df = pd.DataFrame()\n",
    "for file_name in os.listdir(base_dir):\n",
    "    if number_of_files < 0: break\n",
    "    if not file_name.endswith(ext): continue\n",
    "    number_of_files -= 1\n",
    "    print('reading files', file_name)\n",
    "    df = df.append(pd.read_excel(base_dir + file_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['% nans', 'Attribute_name', 'Check', 'Num of nans', 'Reason', 'Record_id', 'Total_val', 'Unnamed: 2', 'Unnamed: 9', 'check', 'max_val', 'mean', 'min_val', 'num of dist_val', 'reason', 'sample_1', 'sample_2', 'sample_3', 'sample_4', 'sample_5', 'std_dev', 'y_Arun', 'y_act', 'y_pred']\n"
     ]
    }
   ],
   "source": [
    "cols = df.columns.tolist()\n",
    "print(cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['% nans'] = df['Num of nans'] / df['Total_val']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['y_act','Reason','Attribute_name','Total_val','num of dist_val','% nans'] + cols[10:13] + ['std_dev'] + cols[15:-4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_act</th>\n",
       "      <th>Reason</th>\n",
       "      <th>Attribute_name</th>\n",
       "      <th>Total_val</th>\n",
       "      <th>num of dist_val</th>\n",
       "      <th>% nans</th>\n",
       "      <th>max_val</th>\n",
       "      <th>mean</th>\n",
       "      <th>min_val</th>\n",
       "      <th>std_dev</th>\n",
       "      <th>sample_1</th>\n",
       "      <th>sample_2</th>\n",
       "      <th>sample_3</th>\n",
       "      <th>sample_4</th>\n",
       "      <th>sample_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unusable</td>\n",
       "      <td>l</td>\n",
       "      <td>StratificationCategory2</td>\n",
       "      <td>403984</td>\n",
       "      <td>1</td>\n",
       "      <td>0.803648</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      y_act Reason           Attribute_name  Total_val  num of dist_val  \\\n",
       "0  Unusable      l  StratificationCategory2     403984                1   \n",
       "\n",
       "     % nans  max_val  mean  min_val  std_dev sample_1 sample_2 sample_3  \\\n",
       "0  0.803648      0.0   0.0      0.0      0.0      NaN      NaN      NaN   \n",
       "\n",
       "  sample_4 sample_5  \n",
       "0      NaN      NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.reindex(columns=cols)\n",
    "df[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['y_act', 'Reason', 'Attribute_name', 'Total_val', 'num of dist_val', '% nans', 'max_val', 'mean', 'min_val', 'std_dev', 'sample_1', 'sample_2', 'sample_3', 'sample_4', 'sample_5']\n"
     ]
    }
   ],
   "source": [
    "print(df.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df.columns.tolist()[-5:]] = df[df.columns.tolist()[-5:]].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Unusable', 'Usable with extraction',\n",
       "       'Usable directly categorical', 'Context_specific',\n",
       "       'Usable directly numeric', 'Usable Directly Numeric',\n",
       "       'Usable Directly Categorical', 'Usable with Extraction',\n",
       "       'Usable Directly Categorical ', 'Usable with Extration',\n",
       "       'Usable With Extraction', 'Usable with Extraction ', 'unusable',\n",
       "       'Usable Directly numeric', 'Usable directly Categorical',\n",
       "       'Usable Directly categorical'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.y_act.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['y_act'] = df.apply(lambda row: label_mapper(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = {'Usable Directly Categorical':    1, \n",
    "              'Usable Directly Numeric':        2, \n",
    "              'Usable With Extraction':         3, \n",
    "              'Unusable':                       4, \n",
    "              'Context_Specific':               5, \n",
    "              'None':                           0\n",
    "              }\n",
    "categories_size = {'Context_Specific':              2050,\n",
    "                   'Unusable':                      891,\n",
    "                   'Usable Directly Categorical':   2087,\n",
    "                   'Usable Directly Numeric':       5063,\n",
    "                   'Usable With Extraction':        650\n",
    "                   }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.y_act != 'Context_Specific']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Unusable', 'Usable With Extraction',\n",
       "       'Usable Directly Categorical', 'Usable Directly Numeric'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.y_act.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LABELING FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 10, 6, 3, 1)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_lfs = [f for f in globals().values() if type(f) == types.FunctionType]\n",
    "lf_extra_list = [f for f in all_lfs if f.__module__ == 'lfs.lf_extration']\n",
    "lf_cate_list = [f for f in all_lfs if f.__module__ == 'lfs.lf_categorical']\n",
    "lf_num_list = [f for f in all_lfs if f.__module__ == 'lfs.lf_numerical']\n",
    "lf_unuse_list = [f for f in all_lfs if f.__module__ == 'lfs.lf_unusable']\n",
    "lf_mixed_list = [f for f in all_lfs if f.__module__ == 'lfs.lf_mixed_class']\n",
    "len(lf_extra_list),len(lf_cate_list),len(lf_num_list),len(lf_unuse_list),len(lf_mixed_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_lfs = lf_extra_list + lf_cate_list + lf_num_list + lf_unuse_list + lf_mixed_list\n",
    "len(total_lfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8691, 30)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n, m = len(df), len(total_lfs)\n",
    "n, m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ls = np.zeros((n,m), dtype=int)\n",
    "j = -1\n",
    "for lf in total_lfs:\n",
    "    j += 1\n",
    "    i = -1\n",
    "    for index, row in df.iterrows():\n",
    "        i += 1\n",
    "        Ls[i][j] = lf(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('lf_outs.txt', Ls, fmt='%i')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from collections import defaultdict\n",
    "def take_mode(row):\n",
    "    occ_map = Counter(row)\n",
    "    occ_map.pop(0,None)\n",
    "    rev_map = defaultdict(list)\n",
    "    for k, v in occ_map.items():\n",
    "        rev_map[v].append(k)\n",
    "    rev_tuple = [(k, v) for k, v in rev_map.items()]\n",
    "    rev_tuple.sort(key=lambda x:-x[0])\n",
    "    if len(rev_tuple):\n",
    "        return np.random.choice(rev_tuple[0][1])\n",
    "    else:\n",
    "        return np.random.randint(1,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "modes = np.apply_along_axis(take_mode, 1, Ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3, 4])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(modes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('modes.txt', modes, fmt='%i')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hierarchy LFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "LFs_1 = lfs_unusable\n",
    "LFs_2 = lfs_ext + lfs_cate + lfs_num + lfs_mixed\n",
    "LFs_3 = []\n",
    "LFs = LFs_1+LFs_2+LFs_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_lfs = len(LFs_1)+len(LFs_2)+len(LFs_3)\n",
    "total_lfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8691, 20)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n, m = len(df), total_lfs\n",
    "n, m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "Ls = np.zeros((n,m), dtype=int)\n",
    "j = -1\n",
    "flag_1 = [False] * n\n",
    "cnt = 0\n",
    "for lf in LFs_1:\n",
    "    i = -1\n",
    "    j += 1\n",
    "    for index, row in df.iterrows():\n",
    "        i += 1\n",
    "        Ls[i][j] = lf(row)\n",
    "        if Ls[i][j] != 0:\n",
    "            flag_1[i] = True\n",
    "\n",
    "flag_2 = flag_1.copy()\n",
    "for lf in LFs_2:\n",
    "    i = -1\n",
    "    j += 1\n",
    "    for index, row in df.iterrows():\n",
    "        i += 1\n",
    "        if not flag_1[i]:\n",
    "            Ls[i][j] = lf(row)\n",
    "        if Ls[i][j] != 0:\n",
    "            flag_2[i] = True\n",
    "\n",
    "flag_3 = flag_2.copy()\n",
    "for lf in LFs_3:\n",
    "    i = -1\n",
    "    j += 1\n",
    "    for index, row in df.iterrows():\n",
    "        i += 1\n",
    "        if not flag_2[i]:\n",
    "            Ls[i][j] = lf(row)\n",
    "        if Ls[i][j] != 0:\n",
    "            flag_3[i] = True\n",
    "            \n",
    "# j+=1\n",
    "# for i in np.where(np.array(flag_3)==False)[0]:\n",
    "#     Ls[i][j] = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ls = np.array(Ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4, 4, 0, ..., 0, 0, 0],\n",
       "       [4, 4, 0, ..., 0, 0, 0],\n",
       "       [4, 4, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 2, 0],\n",
       "       [0, 0, 0, ..., 0, 2, 0],\n",
       "       [0, 0, 0, ..., 0, 2, 0]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('lf_outs.txt', Ls, fmt='%i')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from collections import defaultdict\n",
    "def take_mode(row):\n",
    "    occ_map = Counter(row)\n",
    "    occ_map.pop(0,None)\n",
    "    rev_map = defaultdict(list)\n",
    "    for k, v in occ_map.items():\n",
    "        rev_map[v].append(k)\n",
    "    rev_tuple = [(k, v) for k, v in rev_map.items()]\n",
    "    rev_tuple.sort(key=lambda x:-x[0])\n",
    "    if len(rev_tuple):\n",
    "        return np.random.choice(rev_tuple[0][1])\n",
    "    else:\n",
    "        return np.random.randint(1,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "modes = np.apply_along_axis(take_mode, 1, Ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3, 4])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(modes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('modes.txt', modes, fmt='%i')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.],\n",
       "       [0.],\n",
       "       [0.],\n",
       "       ...,\n",
       "       [0.],\n",
       "       [0.],\n",
       "       [0.]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import stats\n",
    "stats.mode(Ls,axis=1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0., 1., 2., 3., 4.]), array([160941,   4240,   6081,   1213,   1345]))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(Ls, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0,\n",
       "        0, 0, 0, 0, 4, 4, 4, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0,\n",
       "        0, 0, 0, 0, 4, 4, 4, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0,\n",
       "        0, 0, 0, 0, 4, 4, 4, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0,\n",
       "        0, 0, 0, 0, 4, 4, 4, 0],\n",
       "       [3, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Ls[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Snorkel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from snorkel.annotations import LabelAnnotator\n",
    "from scipy.sparse import csr_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "from snorkel.learning import GenerativeModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<8691x30 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 32988 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L_train = csr_matrix(np.array(Ls, dtype=int))\n",
    "L_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inferred cardinality: 4\n"
     ]
    }
   ],
   "source": [
    "gen_model = GenerativeModel()\n",
    "gen_model.train(L_train, epochs=100, decay=0.95, step_size=0.1 / L_train.shape[0], reg_param=1e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_marginals = gen_model.marginals(L_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.83161171e-02, 4.33624676e-01, 1.85913966e-02, 4.89467810e-01],\n",
       "       [1.93604865e-02, 4.51562886e-01, 1.93604865e-02, 5.09716141e-01],\n",
       "       [5.83161171e-02, 4.33624676e-01, 1.85913966e-02, 4.89467810e-01],\n",
       "       ...,\n",
       "       [4.05642510e-05, 9.99878307e-01, 4.05642510e-05, 4.05642510e-05],\n",
       "       [4.05642510e-05, 9.99878307e-01, 4.05642510e-05, 4.05642510e-05],\n",
       "       [4.05642510e-05, 9.99878307e-01, 4.05642510e-05, 4.05642510e-05]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_marginals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One immediate santity check you can peform using the generative model is to visually examine the distribution of predicted training marginals. Ideally, there should get a bimodal distribution with large seperation between each peaks, as shown below by the far right image. The corresponds to good signal for true and positive class labels. For your first Snorkel application, you'll probably see marginals closer to the far left or middle images. With all mass centered around p=0.5, you probably need to write more LFs got get more overall coverage. In the middle image, you have good negative coverage, but not enough positive LFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAFBlJREFUeJzt3W+MXNd93vHvE9KSVSeypWpDEKRUsgFr618lm1uFbYzAieCKtotQBQKBSWsRhiq2kBI4QIGGyosmRUFUfVMkQisVguOKQlILrGNHbBK5YJm4bpHQ9CqRTVMUq41lRWQpcaM0ZusACij/+mKO5dGK250ld2e4Ot8PMJhzzz3n7jnc5Txz/8ydVBWSpD5936QHIEmaHENAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1LG1kx7AYq655pratGnTpIchSavK008//adVNbVYu0s+BDZt2sTMzMykhyFJq0qSF0dp5+EgSerYoiGQ5L1Jnhl6nE3yc0muTnIwyfPt+aqhPg8kmU1yIskdQ/Vbkxxt6x5KkpWamCRpcYuGQFWdqKpbq+pWYCvwF8DngT3AoaraAhxqyyS5AdgJ3AhsBx5OsqZt7hHgXmBLe2xf3ulIkpZiqYeDbgf+uKpeBHYA+1r9PuDOVt4BPFFVr1XVC8AscFuS9cCVVXW4BvevfnyojyRpApYaAjuBz7Tyuqo63covA+taeQPw0lCfk61uQyvPr5ckTcjIIZDkMuAngP80f117Z79s306TZHeSmSQzc3Nzy7VZSdI8S9kT+Ajwh1X1Slt+pR3ioT2fafWngGuH+m1sdadaeX79W1TVo1U1XVXTU1OLXuYqSbpASwmBn+J7h4IADgC7WnkX8ORQ/c4klyfZzOAE8JF26Ohskm3tqqC7h/pIkiZgpA+LJXkX8GHgHw9VPwjsT3IP8CJwF0BVHUuyH3gWOAfcX1Wvtz73AY8BVwBPtYckaUJyqX/R/PT0dC33J4Zv3nfzG+Wju44u67Yl6VKQ5Omqml6snZ8YlqSOXfL3DroYm/b89hvlbz74sQmORJIuTW/rEBjF8fdd/0b5+ueOT3AkkjR+Hg6SpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHWs+y+VkaSx+6V3D5W/Nblx4J6AJHVtpBBI8p4kn03yXJLjSf52kquTHEzyfHu+aqj9A0lmk5xIcsdQ/dYkR9u6h5JkJSYlSRrNqHsCvwJ8oareB9wCHAf2AIeqagtwqC2T5AZgJ3AjsB14OMmatp1HgHuBLe2xfZnmIUm6AIuGQJJ3Az8K/CpAVf1lVf05sAPY15rtA+5s5R3AE1X1WlW9AMwCtyVZD1xZVYerqoDHh/pIkiZglD2BzcAc8B+S/FGSTyV5F7Cuqk63Ni8D61p5A/DSUP+TrW5DK8+vlyRNyCghsBb4APBIVb0f+Dbt0M93tXf2tVyDSrI7yUySmbm5ueXarCRpnlFC4CRwsqq+3JY/yyAUXmmHeGjPZ9r6U8C1Q/03trpTrTy//i2q6tGqmq6q6ampqVHnIklaokVDoKpeBl5K8t5WdTvwLHAA2NXqdgFPtvIBYGeSy5NsZnAC+Eg7dHQ2ybZ2VdDdQ30kSRMw6ofFfhb49SSXAd8APsEgQPYnuQd4EbgLoKqOJdnPICjOAfdX1ettO/cBjwFXAE+1hyRpQkYKgap6Bpg+z6rbF2i/F9h7nvoZ4KalDFCStHL8xLAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSx0YKgSTfTHI0yTNJZlrd1UkOJnm+PV811P6BJLNJTiS5Y6h+a9vObJKHkmT5pyRJGtVS9gR+rKpurarptrwHOFRVW4BDbZkkNwA7gRuB7cDDSda0Po8A9wJb2mP7xU9BknShLuZw0A5gXyvvA+4cqn+iql6rqheAWeC2JOuBK6vqcFUV8PhQH0nSBIwaAgX81yRPJ9nd6tZV1elWfhlY18obgJeG+p5sdRtaeX69JGlC1o7Y7oNVdSrJDwIHkzw3vLKqKkkt16Ba0OwGuO6665Zrs5KkeUbaE6iqU+35DPB54DbglXaIh/Z8pjU/BVw71H1jqzvVyvPrz/fzHq2q6aqanpqaGn02kqQlWTQEkrwryQ98twz8XeDrwAFgV2u2C3iylQ8AO5NcnmQzgxPAR9qho7NJtrWrgu4e6iNJmoBRDgetAz7fruZcC/zHqvpCkq8A+5PcA7wI3AVQVceS7AeeBc4B91fV621b9wGPAVcAT7WHJGlCFg2BqvoGcMt56l8Fbl+gz15g73nqZ4Cblj5MSdJK8BPDktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpYyOHQJI1Sf4oyW+15auTHEzyfHu+aqjtA0lmk5xIcsdQ/dYkR9u6h5JkeacjSVqKpewJfBI4PrS8BzhUVVuAQ22ZJDcAO4Ebge3Aw0nWtD6PAPcCW9pj+0WNXpJ0UUYKgSQbgY8Bnxqq3gHsa+V9wJ1D9U9U1WtV9QIwC9yWZD1wZVUdrqoCHh/qI0magFH3BH4Z+GfAd4bq1lXV6VZ+GVjXyhuAl4banWx1G1p5fv1bJNmdZCbJzNzc3IhDlCQt1aIhkOTvAWeq6umF2rR39rVcg6qqR6tquqqmp6amlmuzkqR51o7Q5keAn0jyUeCdwJVJfg14Jcn6qjrdDvWcae1PAdcO9d/Y6k618vx6SdKELLonUFUPVNXGqtrE4ITv71bVPwQOALtas13Ak618ANiZ5PIkmxmcAD7SDh2dTbKtXRV091AfSdIEjLInsJAHgf1J7gFeBO4CqKpjSfYDzwLngPur6vXW5z7gMeAK4Kn2kCRNyJJCoKq+CHyxlV8Fbl+g3V5g73nqZ4CbljpISdLK8BPDktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR1bNASSvDPJkSRfTXIsyb9o9VcnOZjk+fZ81VCfB5LMJjmR5I6h+q1JjrZ1DyXJykxLkjSKUfYEXgN+vKpuAW4FtifZBuwBDlXVFuBQWybJDcBO4EZgO/BwkjVtW48A9wJb2mP7Ms5FkrREi4ZADfzftviO9ihgB7Cv1e8D7mzlHcATVfVaVb0AzAK3JVkPXFlVh6uqgMeH+kiSJmCkcwJJ1iR5BjgDHKyqLwPrqup0a/IysK6VNwAvDXU/2eo2tPL8+vP9vN1JZpLMzM3NjTwZSdLSjBQCVfV6Vd0KbGTwrv6meeuLwd7BsqiqR6tquqqmp6amlmuzkqR5lnR1UFX9OfB7DI7lv9IO8dCez7Rmp4Brh7ptbHWnWnl+vSRpQka5OmgqyXta+Qrgw8BzwAFgV2u2C3iylQ8AO5NcnmQzgxPAR9qho7NJtrWrgu4e6iNJmoC1I7RZD+xrV/h8H7C/qn4ryR8A+5PcA7wI3AVQVceS7AeeBc4B91fV621b9wGPAVcAT7WHJGlCFg2Bqvoa8P7z1L8K3L5An73A3vPUzwA3vbWHJGkS/MSwJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscWDYEk1yb5vSTPJjmW5JOt/uokB5M8356vGurzQJLZJCeS3DFUvzXJ0bbuoSRZmWlJkkYxyp7AOeCfVtUNwDbg/iQ3AHuAQ1W1BTjUlmnrdgI3AtuBh5Osadt6BLgX2NIe25dxLpKkJVo0BKrqdFX9YSv/H+A4sAHYAexrzfYBd7byDuCJqnqtql4AZoHbkqwHrqyqw1VVwONDfSRJE7CkcwJJNgHvB74MrKuq023Vy8C6Vt4AvDTU7WSr29DK8+slSRMycggk+X7gN4Cfq6qzw+vaO/tarkEl2Z1kJsnM3Nzccm1WkjTPSCGQ5B0MAuDXq+pzrfqVdoiH9nym1Z8Crh3qvrHVnWrl+fVvUVWPVtV0VU1PTU2NOhdJ0hKNcnVQgF8FjlfVvxladQDY1cq7gCeH6ncmuTzJZgYngI+0Q0dnk2xr27x7qI8kaQLWjtDmR4CPA0eTPNPqfgF4ENif5B7gReAugKo6lmQ/8CyDK4vur6rXW7/7gMeAK4Cn2kOSNCGLhkBV/Q9goev5b1+gz15g73nqZ4CbljJASdLK8RPDktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR1bNASSfDrJmSRfH6q7OsnBJM+356uG1j2QZDbJiSR3DNVvTXK0rXsoSZZ/OpKkpRhlT+AxYPu8uj3AoaraAhxqyyS5AdgJ3Nj6PJxkTevzCHAvsKU95m9TkjRmi4ZAVX0J+LN51TuAfa28D7hzqP6Jqnqtql4AZoHbkqwHrqyqw1VVwONDfSRJE3Kh5wTWVdXpVn4ZWNfKG4CXhtqdbHUbWnl+vSRpgi76xHB7Z1/LMJY3JNmdZCbJzNzc3HJuWpI05EJD4JV2iIf2fKbVnwKuHWq3sdWdauX59edVVY9W1XRVTU9NTV3gECVJi7nQEDgA7GrlXcCTQ/U7k1yeZDODE8BH2qGjs0m2tauC7h7qI0makLWLNUjyGeBDwDVJTgK/CDwI7E9yD/AicBdAVR1Lsh94FjgH3F9Vr7dN3cfgSqMrgKfaQ5I0QYuGQFX91AKrbl+g/V5g73nqZ4CbljQ6SdKKWjQEBPzSu4fK35rcOCRpmXnbCEnqmCEgSR0zBCSpY4aAJHXME8MX6fj7rn+jfP1zxyc4EklaOvcEJKlj/ewJDF/mufm6C97MzftuftPy/gvekiRNnnsCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI7182GxJdq057ffKH/znRMcyAK8XYWk5WAIXOr8QhtJK8gQWEWGb1mx0O0q3rQH8+DHFtyWexLj5b/39wz/jcL//+9UK88QGIcF3s2P+4VhlBCZpIUCbHjcR3cdXdlBXMCe16jBK12KDAFJSw4/g+/tY+whkGQ78CvAGuBTVfXguMdwqbvUT0rr7eFS/Dsb616fgDGHQJI1wL8DPgycBL6S5EBVPTvOcahPHn7TxbokDlkus3HvCdwGzFbVNwCSPAHsAAyBS8Sb3x3+9Bvlm4e+g2H/vzr3RvlSOcm5Eocn3vQiPjRnuHTmPYqlht+SX9BW6xVsYx73pbjnBeMPgQ3AS0PLJ4EfHvMYxmKhX7jvDke0wJcAjfSCNtT3TeG1wI+6ZP5zLuNx+Un9nS0UnCsWmku86GKU/5fjfic/6b2IVNX4fljyk8D2qvpHbfnjwA9X1c/Ma7cb2N0W3wucWOKPugb404sc7mrU47x7nDP0Oe8e5wwXPu+/VlVTizUa957AKeDaoeWNre5NqupR4NEL/SFJZqpq+kL7r1Y9zrvHOUOf8+5xzrDy8x73vYO+AmxJsjnJZcBO4MCYxyBJasa6J1BV55L8DPBfGFwi+umqOjbOMUiSvmfsnxOoqt8BfmeFf8wFH0pa5Xqcd49zhj7n3eOcYYXnPdYTw5KkS4vfJyBJHVvVIZBke5ITSWaT7DnP+iR5qK3/WpIPTGKcy2mEOf+DNtejSX4/yS2TGOdyW2zeQ+3+VpJz7XLkVW2UOSf5UJJnkhxL8t/GPcaVMMLf+LuT/OckX23z/sQkxrmcknw6yZkkX19g/cq9llXVqnwwOLH8x8BfBy4DvgrcMK/NR4GngADbgC9PetxjmPPfAa5q5Y+s9jmPOu+hdr/L4JzTT0563GP4Xb+Hwaftr2vLPzjpcY9p3r8A/OtWngL+DLhs0mO/yHn/KPAB4OsLrF+x17LVvCfwxi0oquovge/egmLYDuDxGjgMvCfJ+nEPdBktOueq+v2q+t9t8TCDz2KsdqP8rgF+FvgN4Mw4B7dCRpnzTwOfq6o/AaiqXuZdwA8kCfD9DELgHKtYVX2JwTwWsmKvZas5BM53C4oNF9BmNVnqfO5h8O5htVt03kk2AH8feGSM41pJo/yu/wZwVZIvJnk6yd1jG93KGWXe/xa4HvhfwFHgk1X1nfEMb2JW7LXM7xN4m0ryYwxC4IOTHsuY/DLw81X1ncEbxC6sBbYCtwNXAH+Q5HBV/c/JDmvF3QE8A/w48EPAwST/varOTnZYq9NqDoFRbkEx0m0qVpGR5pPkbwKfAj5SVa+OaWwraZR5TwNPtAC4BvhoknNV9ZvjGeKyG2XOJ4FXq+rbwLeTfAm4BVjNITDKvD8BPFiDg+WzSV4A3gccGc8QJ2LFXstW8+GgUW5BcQC4u51Z3wZ8q6pOj3ugy2jROSe5Dvgc8PG30TvCReddVZuralNVbQI+C9y3igMARvv7fhL4YJK1Sf4Kgzvyrp57XJ/fKPP+EwZ7PyRZx+Amk98Y6yjHb8Vey1btnkAtcAuKJP+krf/3DK4S+SgwC/wFg3cQq9aIc/7nwF8FHm7vis/VKr/p1ojzflsZZc5VdTzJF4CvAd9h8E19573EcLUY8Xf9L4HHkhxlcLXMz1fVqr67aJLPAB8CrklyEvhF4B2w8q9lfmJYkjq2mg8HSZIukiEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLH/h/B8zo8sTSWhQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1232c5d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(train_marginals, bins=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Empirical Acc.</th>\n",
       "      <th>Learned Acc.</th>\n",
       "      <th>Learned Cov.</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>lf_extractable_list</th>\n",
       "      <td>0.571060</td>\n",
       "      <td>0.511863</td>\n",
       "      <td>0.77550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_extractable_units</th>\n",
       "      <td>0.571015</td>\n",
       "      <td>0.510946</td>\n",
       "      <td>0.77200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_extractable_pattern</th>\n",
       "      <td>0.578039</td>\n",
       "      <td>0.512982</td>\n",
       "      <td>0.77800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_extractable_sample_length</th>\n",
       "      <td>0.572303</td>\n",
       "      <td>0.512346</td>\n",
       "      <td>0.77350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_extractable_number_sci</th>\n",
       "      <td>0.570534</td>\n",
       "      <td>0.506033</td>\n",
       "      <td>0.77495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_extractable_name</th>\n",
       "      <td>0.564594</td>\n",
       "      <td>0.503484</td>\n",
       "      <td>0.77490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_extractable_email_url</th>\n",
       "      <td>0.571668</td>\n",
       "      <td>0.504279</td>\n",
       "      <td>0.77705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_date_extraction_name</th>\n",
       "      <td>0.576089</td>\n",
       "      <td>0.514078</td>\n",
       "      <td>0.77780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_same_len_string</th>\n",
       "      <td>0.576813</td>\n",
       "      <td>0.507580</td>\n",
       "      <td>0.78165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_date_extraction_samples</th>\n",
       "      <td>0.585866</td>\n",
       "      <td>0.516828</td>\n",
       "      <td>0.77550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_max_eq_dist</th>\n",
       "      <td>0.537933</td>\n",
       "      <td>0.488287</td>\n",
       "      <td>0.77690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_dist_num_percentage</th>\n",
       "      <td>0.482293</td>\n",
       "      <td>0.465403</td>\n",
       "      <td>0.77535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_dist_string_percentage</th>\n",
       "      <td>0.583389</td>\n",
       "      <td>0.523366</td>\n",
       "      <td>0.77355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_binary_category</th>\n",
       "      <td>0.495347</td>\n",
       "      <td>0.470814</td>\n",
       "      <td>0.78035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_finite_set_name</th>\n",
       "      <td>0.559995</td>\n",
       "      <td>0.503189</td>\n",
       "      <td>0.77615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_name_category</th>\n",
       "      <td>0.571590</td>\n",
       "      <td>0.511252</td>\n",
       "      <td>0.77545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_same_len_string_lt</th>\n",
       "      <td>0.563862</td>\n",
       "      <td>0.503058</td>\n",
       "      <td>0.77665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_coded_abbreviation</th>\n",
       "      <td>0.577020</td>\n",
       "      <td>0.515265</td>\n",
       "      <td>0.77465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_same_len_num_lt</th>\n",
       "      <td>0.464756</td>\n",
       "      <td>0.455537</td>\n",
       "      <td>0.77480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_finite_set_sample</th>\n",
       "      <td>0.587831</td>\n",
       "      <td>0.521063</td>\n",
       "      <td>0.78100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_cast_to_numbers</th>\n",
       "      <td>1.574740</td>\n",
       "      <td>0.883365</td>\n",
       "      <td>0.84280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_has_mean_std</th>\n",
       "      <td>1.577269</td>\n",
       "      <td>0.882686</td>\n",
       "      <td>0.84730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_sample_cast_to_float</th>\n",
       "      <td>0.842544</td>\n",
       "      <td>0.647819</td>\n",
       "      <td>0.78170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_std_gt_mean</th>\n",
       "      <td>1.182005</td>\n",
       "      <td>0.776798</td>\n",
       "      <td>0.80510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_max_min_difference</th>\n",
       "      <td>0.722237</td>\n",
       "      <td>0.581974</td>\n",
       "      <td>0.78165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_has_mean_dev_min</th>\n",
       "      <td>1.015049</td>\n",
       "      <td>0.719932</td>\n",
       "      <td>0.79570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_unusable_dist</th>\n",
       "      <td>0.532625</td>\n",
       "      <td>0.491816</td>\n",
       "      <td>0.77590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_unusable</th>\n",
       "      <td>0.528599</td>\n",
       "      <td>0.489732</td>\n",
       "      <td>0.78155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_unusable_nans</th>\n",
       "      <td>0.574086</td>\n",
       "      <td>0.518676</td>\n",
       "      <td>0.77775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lf_key_word_year</th>\n",
       "      <td>0.552511</td>\n",
       "      <td>0.496734</td>\n",
       "      <td>0.78070</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Empirical Acc.  Learned Acc.  Learned Cov.\n",
       "lf_extractable_list                 0.571060      0.511863       0.77550\n",
       "lf_extractable_units                0.571015      0.510946       0.77200\n",
       "lf_extractable_pattern              0.578039      0.512982       0.77800\n",
       "lf_extractable_sample_length        0.572303      0.512346       0.77350\n",
       "lf_extractable_number_sci           0.570534      0.506033       0.77495\n",
       "lf_extractable_name                 0.564594      0.503484       0.77490\n",
       "lf_extractable_email_url            0.571668      0.504279       0.77705\n",
       "lf_date_extraction_name             0.576089      0.514078       0.77780\n",
       "lf_same_len_string                  0.576813      0.507580       0.78165\n",
       "lf_date_extraction_samples          0.585866      0.516828       0.77550\n",
       "lf_max_eq_dist                      0.537933      0.488287       0.77690\n",
       "lf_dist_num_percentage              0.482293      0.465403       0.77535\n",
       "lf_dist_string_percentage           0.583389      0.523366       0.77355\n",
       "lf_binary_category                  0.495347      0.470814       0.78035\n",
       "lf_finite_set_name                  0.559995      0.503189       0.77615\n",
       "lf_name_category                    0.571590      0.511252       0.77545\n",
       "lf_same_len_string_lt               0.563862      0.503058       0.77665\n",
       "lf_coded_abbreviation               0.577020      0.515265       0.77465\n",
       "lf_same_len_num_lt                  0.464756      0.455537       0.77480\n",
       "lf_finite_set_sample                0.587831      0.521063       0.78100\n",
       "lf_cast_to_numbers                  1.574740      0.883365       0.84280\n",
       "lf_has_mean_std                     1.577269      0.882686       0.84730\n",
       "lf_sample_cast_to_float             0.842544      0.647819       0.78170\n",
       "lf_std_gt_mean                      1.182005      0.776798       0.80510\n",
       "lf_max_min_difference               0.722237      0.581974       0.78165\n",
       "lf_has_mean_dev_min                 1.015049      0.719932       0.79570\n",
       "lf_unusable_dist                    0.532625      0.491816       0.77590\n",
       "lf_unusable                         0.528599      0.489732       0.78155\n",
       "lf_unusable_nans                    0.574086      0.518676       0.77775\n",
       "lf_key_word_year                    0.552511      0.496734       0.78070"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learned_table = gen_model.learned_lf_stats()\n",
    "learned_cov = learned_table.values[:, 1]\n",
    "learned_acc = learned_table.values[:,0]\n",
    "empirical_acc = gen_model.weights.lf_accuracy\n",
    "compared_stats = pd.DataFrame(np.stack((empirical_acc, learned_acc, learned_cov)).T,\n",
    "                             index = [lf.__name__ for lf in total_lfs],\n",
    "                             columns=['Empirical Acc.','Learned Acc.','Learned Cov.'])\n",
    "compared_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('train_marginals_4.csv', train_marginals, delimiter=',',)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('train_raw_4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 3, 3, 3, 2, 3, 1, 0, 0, 0, 0, 0, 3, 3, 3, 3, 2, 0, 0, 0, 2, 0,\n",
       "       0, 0, 0, 2, 2, 2, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 2, 0, 1,\n",
       "       0, 0, 0, 0, 0, 1])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "denoised_labels = np.argmax(train_marginals, axis=1)\n",
    "denoised_labels[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrects = 0\n",
    "cm = np.zeros(5)\n",
    "i = -1\n",
    "for index, row in df.iterrows():\n",
    "    i += 1\n",
    "    if denoised_labels[i]+1 == categories[row.y_act]:\n",
    "        corrects += 1\n",
    "    else:\n",
    "        cm[categories[row.y_act]-1] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7839143942008975"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrects / len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6813"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1188.,    7.,  201.,  482.,    0.])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = [categories[v] for v in df.y_act.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8147508917270739"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(Y, modes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BACKUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "from snorkel.annotations import LabelAnnotator\n",
    "labeler = LabelAnnotator(lfs=LFs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<10741x17 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 11999 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L_train = csr_matrix(np.array(Ls, dtype=int))\n",
    "L_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inferred cardinality: 5\n"
     ]
    }
   ],
   "source": [
    "from snorkel.learning import GenerativeModel\n",
    "\n",
    "gen_model = GenerativeModel()\n",
    "gen_model.train(L_train, epochs=100, decay=0.95, step_size=0.1 / L_train.shape[0], reg_param=1e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.71233108, 0.70567372, 0.70536448, 0.70789609, 0.70711461,\n",
       "       0.70982874, 0.70698972, 0.70794799, 0.70350743, 0.72981279,\n",
       "       0.72691405, 0.71446542, 0.72066986, 0.74939244, 0.72188569,\n",
       "       0.81458312, 0.70733583])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen_model.weights.lf_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_marginals = gen_model.marginals(L_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAD6tJREFUeJzt3X+s3XV9x/Hna60g6kAYdw0ruNstnfzqjNJhp8Y42ULVZWWJIc2mNIZBCOjcsmQW/5gmSxOWLIuSAYbgRsmMTYNkdEPcSJlziwO8KFpLx+hEoJUf1W2yuARXeO+P80GOl3b3XHrvOb39PB/Jyfmcz/fz/d73J/f2vM73x/k2VYUkqU8/MekCJEmTYwhIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOrZ80gXM5dRTT63p6elJlyFJS8r999//3aqammvcUR8C09PTzMzMTLoMSVpSkjw6yjgPB0lSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMgTFas3UNa7aumXQZkvQjhsCEXXfF3Vx3xd2TLkNSpwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCYgD1nnsWeM8+adBmSZAhIUs8MAUnqmCEgSR0zBCSpY4aAJHVspBBI8vtJdif5ZpLPJnllklOS3JXk4fZ88tD4q5PsTfJQkguH+s9LsqstuzZJFmNSR4PpzXcwvfmOSZchSf+vOUMgyUrgd4G1VXUusAzYCGwGdlbVamBne02Ss9vyc4D1wPVJlrXN3QBcBqxuj/ULOhtJ0ryMejhoOXBCkuXAq4DvABuArW35VuCi1t4AbKuqZ6vqEWAvcH6S04ATq+qeqirglqF1JEkTMGcIVNV+4E+Bx4AngO9X1d8DK6rqiTbsSWBFa68EHh/axL7Wt7K1Z/e/RJLLk8wkmTlw4MA8piNJmo9RDgedzODT/SrgZ4BXJ3nf8Jj2yb4WqqiqurGq1lbV2qmpqYXarCRpllEOB/0q8EhVHaiq/wVuA94CPNUO8dCen27j9wNnDK1/euvb39qz+yVJEzJKCDwGrEvyqnY1zwXAHmAHsKmN2QTc3to7gI1Jjk+yisEJ4PvaoaNnkqxr27lkaB1J0gQsn2tAVd2b5Fbgq8BB4GvAjcBrgO1JLgUeBS5u43cn2Q482MZfVVXPtc1dCdwMnADc2R6SpAmZMwQAqupjwMdmdT/LYK/gUOO3AFsO0T8DnDvPGiVJi8RvDEtSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0bKQSSvDbJrUn+NcmeJL+c5JQkdyV5uD2fPDT+6iR7kzyU5MKh/vOS7GrLrk2SxZiUJGk0o+4JfBL4QlWdCbwB2ANsBnZW1WpgZ3tNkrOBjcA5wHrg+iTL2nZuAC4DVrfH+gWah7S0ffykFx/SGM0ZAklOAt4OfBqgqn5YVf8FbAC2tmFbgYtaewOwraqerapHgL3A+UlOA06sqnuqqoBbhtaRJE3AKHsCq4ADwF8m+VqSm5K8GlhRVU+0MU8CK1p7JfD40Pr7Wt/K1p7d/xJJLk8yk2TmwIEDo89GkjQvo4TAcuBNwA1V9UbgB7RDPy9on+xroYqqqhuram1VrZ2amlqozUqSZhklBPYB+6rq3vb6Vgah8FQ7xEN7frot3w+cMbT+6a1vf2vP7pckTcicIVBVTwKPJ3l967oAeBDYAWxqfZuA21t7B7AxyfFJVjE4AXxfO3T0TJJ17aqgS4bWkSRNwPIRx30I+EyS44BvAR9gECDbk1wKPApcDFBVu5NsZxAUB4Grquq5tp0rgZuBE4A720OSNCEjhUBVPQCsPcSiCw4zfguw5RD9M8C58ylQkrR4/MawJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI6NHAJJliX5WpK/ba9PSXJXkofb88lDY69OsjfJQ0kuHOo/L8mutuzaJFnY6UiS5mM+ewIfBvYMvd4M7Kyq1cDO9pokZwMbgXOA9cD1SZa1dW4ALgNWt8f6I6peknRERgqBJKcD7wFuGureAGxt7a3ARUP926rq2ap6BNgLnJ/kNODEqrqnqgq4ZWgdSdIEjLon8AngD4Hnh/pWVNUTrf0ksKK1VwKPD43b1/pWtvbsfknShMwZAkl+HXi6qu4/3Jj2yb4WqqgklyeZSTJz4MCBhdqsJGmWUfYE3gr8RpJvA9uAdyb5K+CpdoiH9vx0G78fOGNo/dNb3/7Wnt3/ElV1Y1Wtraq1U1NT85iOJGk+5gyBqrq6qk6vqmkGJ3zvrqr3ATuATW3YJuD21t4BbExyfJJVDE4A39cOHT2TZF27KuiSoXUkSROw/AjWvQbYnuRS4FHgYoCq2p1kO/AgcBC4qqqea+tcCdwMnADc2R6SpAmZVwhU1ReBL7b294ALDjNuC7DlEP0zwLnzLVKStDj8xrAkdcwQkKSOHck5AY3i4ye92F71usnVIUmH4J6AJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWNzhkCSM5L8Q5IHk+xO8uHWf0qSu5I83J5PHlrn6iR7kzyU5MKh/vOS7GrLrk2SxZmWJGkUo+wJHAT+oKrOBtYBVyU5G9gM7Kyq1cDO9pq2bCNwDrAeuD7JsratG4DLgNXtsX4B5yJJmqc5Q6Cqnqiqr7b2fwN7gJXABmBrG7YVuKi1NwDbqurZqnoE2Aucn+Q04MSquqeqCrhlaB1J0gTM65xAkmngjcC9wIqqeqItehJY0dorgceHVtvX+la29ux+SdKEjBwCSV4DfA74vap6ZnhZ+2RfC1VUksuTzCSZOXDgwEJtVpI0y0ghkOQVDALgM1V1W+t+qh3ioT0/3fr3A2cMrX5669vf2rP7X6KqbqyqtVW1dmpqatS5SJLmaZSrgwJ8GthTVX82tGgHsKm1NwG3D/VvTHJ8klUMTgDf1w4dPZNkXdvmJUPrSJImYPkIY94KvB/YleSB1vdR4Bpge5JLgUeBiwGqaneS7cCDDK4suqqqnmvrXQncDJwA3NkekqQJmTMEquqfgcNdz3/BYdbZAmw5RP8McO58CpQkLR6/MSxJHTMEJKljhoAkdcwQkI7Q9OY7mN58x6TLkF4WQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQFtGeM89iz5lnTboM6bAMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdWz7pAsbi4ycNtb8P8GNf5b/7HdcBcNWn3jnWsiRp0twTkKSOGQKS1DFDQJI61sc5gSFrtq4BYPuE65Cko4F7ApLUMUNAkjpmCEhSxwwBSUvK9OY7mN58x6TLOGYYApLUse6uDpIm4bor7v5R22+m62hiCEiLwEuRtVR4OEiSOmYISFLHxn44KMl64JPAMuCmqrpm3DXM2wt3IW13IIUX70L6wh1IwWO9R+qFKz6+fc17gBcPqQDs2rRrIjVJx7qxhkCSZcB1wK8B+4CvJNlRVQ8uxs/70ZvKKxdj65I0utkfco4W4z4cdD6wt6q+VVU/BLYBG8Zcg6RjzJqta35sz1GjG/fhoJXA40Ov9wFvHnMNR71DfWJ44Q/cwyI6Ggx/WcvDdy/f0fDvOlU1vh+WvBdYX1W/016/H3hzVX1w1rjLgcvby9cDD83zR50KfPcIy12KnHd/ep27857bz1bV1FyDxr0nsB84Y+j16a3vx1TVjcCNL/eHJJmpqrUvd/2lynn3p9e5O++FM+5zAl8BVidZleQ4YCOwY8w1SJKase4JVNXBJB8E/o7BJaJ/UVW7x1mDJOlFY/+eQFV9Hvj8Iv+Yl30oaYlz3v3pde7Oe4GM9cSwJOno4m0jJKljSzYEkqxP8lCSvUk2H2J5klzbln8jyZsmUedCG2Hev93muyvJl5O8YRJ1Loa55j407peSHGyXJC95o8w7yTuSPJBkd5J/HHeNi2WEv/eTkvxNkq+3uX9gEnUupCR/keTpJN88zPKFfW+rqiX3YHBS+d+BnwOOA74OnD1rzLuBO4EA64B7J133mOb9FuDk1n7XsTDvUec+NO5uBued3jvpusf0O38t8CDwuvb6pydd9xjn/lHgT1p7CvgP4LhJ136E83478Cbgm4dZvqDvbUt1T2CU209sAG6pgXuA1yY5bdyFLrA5511VX66q/2wv72HwXYxjwai3HPkQ8Dng6XEWt4hGmfdvAbdV1WMAVdXT3Av4ySQBXsMgBA6Ot8yFVVVfYjCPw1nQ97alGgKHuv3EypcxZqmZ75wuZfCJ4Vgw59yTrAR+E7hhjHUttlF+578AnJzki0nuT3LJ2KpbXKPM/c+Bs4DvALuAD1fV8+Mpb2IW9L3N/1nsGJXkVxiEwNsmXcsYfQL4SFU9P/hg2I3lwHnABcAJwL8kuaeq/m2yZY3FhcADwDuBnwfuSvJPVfXMZMtaOpZqCIxy+4mRblGxxIw0pyS/CNwEvKuqvjem2hbbKHNfC2xrAXAq8O4kB6vqr8dT4qIYZd77gO9V1Q+AHyT5EvAGYKmHwChz/wBwTQ0Olu9N8ghwJnDfeEqciAV9b1uqh4NGuf3EDuCSdiZ9HfD9qnpi3IUusDnnneR1wG3A+4+xT4Jzzr2qVlXVdFVNA7cCVy7xAIDR/tZvB96WZHmSVzG4M++eMde5GEaZ+2MM9oBIsoLBDSe/NdYqx29B39uW5J5AHeb2E0muaMs/xeDqkHcDe4H/YfCJYUkbcd5/BPwUcH37RHywjoEbbY0492POKPOuqj1JvgB8A3iewf/Yd8jLC5eSEX/nfwzcnGQXg6tlPlJVS/ruokk+C7wDODXJPuBjwCtgcd7b/MawJHVsqR4OkiQtAENAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSO/R8leVNxfA1jLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x120a13710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.hist(train_marginals, bins=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Coverage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.510354</td>\n",
       "      <td>0.79968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.503390</td>\n",
       "      <td>0.79652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.504969</td>\n",
       "      <td>0.79688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.503832</td>\n",
       "      <td>0.79852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.507562</td>\n",
       "      <td>0.79612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.508077</td>\n",
       "      <td>0.79736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.510088</td>\n",
       "      <td>0.79696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.505249</td>\n",
       "      <td>0.80016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.498974</td>\n",
       "      <td>0.79932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.519279</td>\n",
       "      <td>0.79880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.519301</td>\n",
       "      <td>0.79892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.508961</td>\n",
       "      <td>0.79676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.520306</td>\n",
       "      <td>0.79976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.527420</td>\n",
       "      <td>0.79504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.510533</td>\n",
       "      <td>0.79940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.560296</td>\n",
       "      <td>0.80636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.502914</td>\n",
       "      <td>0.79608</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Accuracy  Coverage\n",
       "0   0.510354   0.79968\n",
       "1   0.503390   0.79652\n",
       "2   0.504969   0.79688\n",
       "3   0.503832   0.79852\n",
       "4   0.507562   0.79612\n",
       "5   0.508077   0.79736\n",
       "6   0.510088   0.79696\n",
       "7   0.505249   0.80016\n",
       "8   0.498974   0.79932\n",
       "9   0.519279   0.79880\n",
       "10  0.519301   0.79892\n",
       "11  0.508961   0.79676\n",
       "12  0.520306   0.79976\n",
       "13  0.527420   0.79504\n",
       "14  0.510533   0.79940\n",
       "15  0.560296   0.80636\n",
       "16  0.502914   0.79608"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen_model.learned_lf_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.12260231, 0.12260231, 0.12260231, 0.50959077, 0.12260231],\n",
       "       [0.12260231, 0.12260231, 0.12260231, 0.50959077, 0.12260231],\n",
       "       [0.12260231, 0.12260231, 0.12260231, 0.50959077, 0.12260231],\n",
       "       ...,\n",
       "       [0.10989469, 0.56042125, 0.10989469, 0.10989469, 0.10989469],\n",
       "       [0.10989469, 0.56042125, 0.10989469, 0.10989469, 0.10989469],\n",
       "       [0.10989469, 0.56042125, 0.10989469, 0.10989469, 0.10989469]])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_marginals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 3, 3, 3, 2, 3, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 2, 0, 0, 0, 0, 2,\n",
       "       0, 2, 0, 4, 0, 0, 2, 2, 2, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1, 0, 0, 0,\n",
       "       4, 2, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "denoised_labels = np.argmax(train_marginals, axis=1)\n",
    "denoised_labels[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrects = 0\n",
    "cm = np.zeros(5)\n",
    "i = -1\n",
    "for index, row in df.iterrows():\n",
    "    i += 1\n",
    "    if denoised_labels[i]+1 == categories[row.y_act]:\n",
    "        corrects += 1\n",
    "    else:\n",
    "        cm[categories[row.y_act]-1] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6813145889581975"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrects / len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7318"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 530.,  535.,  317.,  105., 1936.])"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3.5",
   "language": "python",
   "name": "python3.5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}