NER model Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1bb50b14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d77266e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('ner_dataset.csv', low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8d50bc3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_idx</th>\n",
       "      <th>word</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>of</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>have</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>marched</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  sentence_idx           word tag\n",
       "0            1      Thousands   O\n",
       "1            1             of   O\n",
       "2            1  demonstrators   O\n",
       "3            1           have   O\n",
       "4            1        marched   O"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a5d2dc02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_idx</th>\n",
       "      <th>word</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>of</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>have</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>marched</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1048557</th>\n",
       "      <td>47850</td>\n",
       "      <td>to</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1048558</th>\n",
       "      <td>47850</td>\n",
       "      <td>deploy</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1048572</th>\n",
       "      <td>47851</td>\n",
       "      <td>Last</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1048573</th>\n",
       "      <td>47851</td>\n",
       "      <td>month</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1048574</th>\n",
       "      <td>47851</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>175296 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        sentence_idx           word tag\n",
       "0                  1      Thousands   O\n",
       "1                  1             of   O\n",
       "2                  1  demonstrators   O\n",
       "3                  1           have   O\n",
       "4                  1        marched   O\n",
       "...              ...            ...  ..\n",
       "1048557        47850             to   O\n",
       "1048558        47850         deploy   O\n",
       "1048572        47851           Last   O\n",
       "1048573        47851          month   O\n",
       "1048574        47851              ,   O\n",
       "\n",
       "[175296 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.groupby(\"sentence_idx\").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "50564eb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "O                  888072\n",
       "B-geo               37444\n",
       "B-tim               20159\n",
       "B-org               20134\n",
       "I-per               17346\n",
       "B-per               16975\n",
       "I-org               16498\n",
       "B-gpe               16368\n",
       "I-geo                7396\n",
       "I-tim                6291\n",
       "B-art                 434\n",
       "B-eve                 348\n",
       "I-eve                 297\n",
       "I-art                 280\n",
       "I-gpe                 229\n",
       "B-nat                 226\n",
       "I-nat                  76\n",
       "prev-prev-lemma         1\n",
       "Name: tag, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['tag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c45c7ff7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABDAAAAFgCAYAAABNIolGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAsdElEQVR4nO3dfbhddXnn//eHBAEfQJTgYIJCMVMLjEDJDxn1alWspFYEFWocH5jKNEixtp3Or4W2jjqVTplrlNZasDiIgVIhBRlQQaXgw6g8GCjyKGNGUFIYEgGBWkUT7vljrwM7h5MQSdZZK3u9X9e1rrP3vffa576vvc7e+9z7u77fVBWSJEmSJEl9tk3XCUiSJEmSJD0RGxiSJEmSJKn3bGBIkiRJkqTes4EhSZIkSZJ6zwaGJEmSJEnqvbldJzDbdtlll9pjjz26TkOSJEmSJM3g2muv/X5VzZseH1wDY4899mDFihVdpyFJkiRJkmaQ5LszxT2FRJIkSZIk9Z4NDEmSJEmS1Hs2MCRJkiRJUu/ZwJAkSZIkSb1nA0OSJEmSJPWeDQxJkiRJktR7NjAkSZIkSVLv2cCQJEmSJEm9ZwNDkiRJkiT13tyuE+iLNaf9bdcpPGnzjntr1ylIkiRJktQqR2BIkiRJkqTes4EhSZIkSZJ6zwaGJEmSJEnqPRsYkiRJkiSp92xgSJIkSZKk3rOBIUmSJEmSes8GhiRJkiRJ6j0bGJIkSZIkqfdsYEiSJEmSpN6zgSFJkiRJknrPBoYkSZIkSeq91hoYSbZPck2Sbya5Ocn7m/j7kvxTkuub7TVj+5yYZGWS25IcOhY/MMmNzW0fTpImvl2S85r41Un2aKseSZIkSZLUnTZHYDwMvLKq9gP2BxYnObi57ZSq2r/ZLgFIsjewBNgHWAycmmROc//TgKXAwmZb3MSPAe6vqhcApwAnt1iPJEmSJEnqSGsNjBr55+bqts1WG9nlcODcqnq4qm4HVgIHJdkN2LGqrqyqAs4CjhjbZ1lz+XzgkKnRGZIkSZIkaXK0OgdGkjlJrgdWA5dV1dXNTe9KckOSjyfZuYnNB+4c231VE5vfXJ4eX2+fqloLPAA8e4Y8liZZkWTFmjVrtkxxkiRJkiRp1rTawKiqdVW1P7CA0WiKfRmdDrIXo9NK7gY+2Nx9ppETtZH4xvaZnsfpVbWoqhbNmzfvZ6pBkiRJkiR1b1ZWIamqHwBfAhZX1T1NY+MR4GPAQc3dVgG7j+22ALiriS+YIb7ePknmAjsB97VThSRJkiRJ6kqbq5DMS/LM5vIOwKuAbzVzWkx5PXBTc/liYEmzssiejCbrvKaq7gYeSnJwM7/F24GLxvY5url8JHBFM0+GJEmSJEmaIHNbfOzdgGXNSiLbAMur6jNJzk6yP6NTPe4AjgWoqpuTLAduAdYCx1fVuuaxjgM+AewAXNpsAGcAZydZyWjkxZIW65EkSZIkSR1prYFRVTcAB8wQf9tG9jkJOGmG+Apg3xniPwaO2rxMJUmSJElS383KHBiSJEmSJEmbwwaGJEmSJEnqPRsYkiRJkiSp92xgSJIkSZKk3rOBIUmSJEmSes8GhiRJkiRJ6j0bGJIkSZIkqfdsYEiSJEmSpN6zgSFJkiRJknrPBoYkSZIkSeo9GxiSJEmSJKn3bGBIkiRJkqTes4EhSZIkSZJ6zwaGJEmSJEnqPRsYkiRJkiSp92xgSJIkSZKk3rOBIUmSJEmSes8GhiRJkiRJ6j0bGJIkSZIkqfdsYEiSJEmSpN5rrYGRZPsk1yT5ZpKbk7y/iT8ryWVJvt383HlsnxOTrExyW5JDx+IHJrmxue3DSdLEt0tyXhO/OskebdUjSZIkSZK60+YIjIeBV1bVfsD+wOIkBwMnAJdX1ULg8uY6SfYGlgD7AIuBU5PMaR7rNGApsLDZFjfxY4D7q+oFwCnAyS3WI0mSJEmSOtJaA6NG/rm5um2zFXA4sKyJLwOOaC4fDpxbVQ9X1e3ASuCgJLsBO1bVlVVVwFnT9pl6rPOBQ6ZGZ0iSJEmSpMnR6hwYSeYkuR5YDVxWVVcDz6mquwGan7s2d58P3Dm2+6omNr+5PD2+3j5VtRZ4AHj2DHksTbIiyYo1a9ZsoeokSZIkSdJsabWBUVXrqmp/YAGj0RT7buTuM42cqI3EN7bP9DxOr6pFVbVo3rx5T5C1JEmSJEnqm1lZhaSqfgB8idHcFfc0p4XQ/Fzd3G0VsPvYbguAu5r4ghni6+2TZC6wE3BfGzVIkiRJkqTutLkKybwkz2wu7wC8CvgWcDFwdHO3o4GLmssXA0ualUX2ZDRZ5zXNaSYPJTm4md/i7dP2mXqsI4ErmnkyJEmSJEnSBJnb4mPvBixrVhLZBlheVZ9JciWwPMkxwPeAowCq6uYky4FbgLXA8VW1rnms44BPADsAlzYbwBnA2UlWMhp5saTFeiRJkiRJUkdaa2BU1Q3AATPE7wUO2cA+JwEnzRBfATxu/oyq+jFNA0SSJEmSJE2uWZkDQ5IkSZIkaXPYwJAkSZIkSb1nA0OSJEmSJPWeDQxJkiRJktR7NjAkSZIkSVLv2cCQJEmSJEm9ZwNDkiRJkiT1ng0MSZIkSZLUezYwJEmSJElS79nAkCRJkiRJvWcDQ5IkSZIk9Z4NDEmSJEmS1Hs2MCRJkiRJUu/ZwJAkSZIkSb1nA0OSJEmSJPWeDQxJkiRJktR7NjAkSZIkSVLv2cCQJEmSJEm9ZwNDkiRJkiT1ng0MSZIkSZLUezYwJEmSJElS77XWwEiye5IvJrk1yc1JfqeJvy/JPyW5vtleM7bPiUlWJrktyaFj8QOT3Njc9uEkaeLbJTmviV+dZI+26pEkSZIkSd1pcwTGWuD3q+oXgIOB45Ps3dx2SlXt32yXADS3LQH2ARYDpyaZ09z/NGApsLDZFjfxY4D7q+oFwCnAyS3WI0mSJEmSOtJaA6Oq7q6q65rLDwG3AvM3ssvhwLlV9XBV3Q6sBA5KshuwY1VdWVUFnAUcMbbPsuby+cAhU6MzJEmSJEnS5JiVOTCaUzsOAK5uQu9KckOSjyfZuYnNB+4c221VE5vfXJ4eX2+fqloLPAA8e4bfvzTJiiQr1qxZs2WKkiRJkiRJs6b1BkaSpwMXAL9bVQ8yOh1kL2B/4G7gg1N3nWH32kh8Y/usH6g6vaoWVdWiefPm/WwFSJIkSZKkzrXawEiyLaPmxTlV9SmAqrqnqtZV1SPAx4CDmruvAnYf230BcFcTXzBDfL19kswFdgLua6caSZIkSZLUlTZXIQlwBnBrVX1oLL7b2N1eD9zUXL4YWNKsLLIno8k6r6mqu4GHkhzcPObbgYvG9jm6uXwkcEUzT4YkSZIkSZogc1t87JcCbwNuTHJ9E/sj4M1J9md0qscdwLEAVXVzkuXALYxWMDm+qtY1+x0HfALYAbi02WDUIDk7yUpGIy+WtFiPJEmSJEnqSGsNjKr6KjPPUXHJRvY5CThphvgKYN8Z4j8GjtqMNCVJkiRJ0lZgVlYhkSRJkiRJ2hw2MCRJkiRJUu/ZwJAkSZIkSb1nA0OSJEmSJPWeDQxJkiRJktR7NjAkSZIkSVLv2cCQJEmSJEm9ZwNDkiRJkiT1ng0MSZIkSZLUezYwJEmSJElS79nAkCRJkiRJvWcDQ5IkSZIk9Z4NDEmSJEmS1Hs2MCRJkiRJUu9tUgMjyeWbEpMkSZIkSWrD3I3dmGR74KnALkl2BtLctCPw3JZzkyRJkiRJAp6ggQEcC/wuo2bFtTzWwHgQ+Ov20pIkSZIkSXrMRhsYVfWXwF8m+e2q+qtZykmSJEmSJGk9TzQCA4Cq+qskLwH2GN+nqs5qKS9JkiRJkqRHbVIDI8nZwF7A9cC6JlyADQxJkiRJktS6TWpgAIuAvauqNvWBk+zOqMHxr4BHgNOr6i+TPAs4j9FojjuAX6+q+5t9TgSOYdQkeXdVfb6JHwh8AtgBuAT4naqqJNs1v+NA4F7gTVV1x6bmKEmSJEmStg6btIwqcBOjRsTPYi3w+1X1C8DBwPFJ9gZOAC6vqoXA5c11mtuWAPsAi4FTk8xpHus0YCmwsNkWN/FjgPur6gXAKcDJP2OOkiRJkiRpK7CpIzB2AW5Jcg3w8FSwql63oR2q6m7g7ubyQ0luBeYDhwMvb+62DPgS8IdN/Nyqehi4PclK4KAkdwA7VtWVAEnOAo4ALm32eV/zWOcDH0mSn2WkiCSpP371ojd3ncKTdunhn+w6BUmSpIm2qQ2M923OL0myB3AAcDXwnKa5QVXdnWTX5m7zgavGdlvVxH7aXJ4en9rnzuax1iZ5AHg28P1pv38poxEcPO95z9ucUiT1xMeXvbrrFJ60dxz9ha5TkCRJkrY6m7oKyZef7C9I8nTgAuB3q+rBJBu860y/eiPxje2zfqDqdOB0gEWLFjk6Q5IkSZKkrcwmzYGR5KEkDzbbj5OsS/LgJuy3LaPmxTlV9akmfE+S3ZrbdwNWN/FVwO5juy8A7mriC2aIr7dPkrnATsB9m1KTJEmSJEnaemxSA6OqnlFVOzbb9sAbgY9sbJ+MhlqcAdxaVR8au+li4Ojm8tHARWPxJUm2S7Ino8k6r2lON3koycHNY7592j5Tj3UkcIXzX0iSJEmSNHk2dQ6M9VTV/0xywhPc7aXA24Abk1zfxP4I+HNgeZJjgO8BRzWPeXOS5cAtjFYwOb6q1jX7Hcdjy6he2mwwapCc3Uz4eR+jVUwkSZIkSdKE2aQGRpI3jF3dBljEDHNNjKuqrzLzHBUAh2xgn5OAk2aIrwD2nSH+Y5oGiCRJkiRJmlybOgLjsLHLa4E7GC1hKkmSJEmS1LpNXYXkN9pORJIkSZIkaUM2dRWSBUkuTLI6yT1JLkiy4In3lCRJkiRJ2nyb1MAAzmS04sdzgfnAp5uYJEmSJElS6za1gTGvqs6sqrXN9glgXot5SZIkSZIkPWpTGxjfT/LWJHOa7a3AvW0mJkmSJEmSNGVTGxjvAH4d+L/A3cCRgBN7SpIkSZKkWbGpy6j+KXB0Vd0PkORZwH9n1NiQJEmSJElq1aaOwHjRVPMCoKruAw5oJyVJkiRJkqT1bWoDY5skO09daUZgbOroDUmSJEmSpM2yqU2IDwJfT3I+UIzmwziptawkSZK2Qoedf0HXKWyWTx/5xq5TkCRpgzapgVFVZyVZAbwSCPCGqrql1cwkSZIkSZIam3waSNOwsGmhrco/fvSwrlPYLAe889NdpyBJkiRJvbCpc2BIkiRJkiR1xgaGJEmSJEnqPVcSkaSe+8B5h3adwpP2J2/6fNcpSJIkaULYwBigu0/9465TeNJ2+y0Xv5EkSZKkIfIUEkmSJEmS1Hs2MCRJkiRJUu/ZwJAkSZIkSb1nA0OSJEmSJPVeaw2MJB9PsjrJTWOx9yX5pyTXN9trxm47McnKJLclOXQsfmCSG5vbPpwkTXy7JOc18auT7NFWLZIkSZIkqVttjsD4BLB4hvgpVbV/s10CkGRvYAmwT7PPqUnmNPc/DVgKLGy2qcc8Bri/ql4AnAKc3FYhkiRJkiSpW601MKrqK8B9m3j3w4Fzq+rhqrodWAkclGQ3YMequrKqCjgLOGJsn2XN5fOBQ6ZGZ0iSJEmSpMnSxRwY70pyQ3OKyc5NbD5w59h9VjWx+c3l6fH19qmqtcADwLNn+oVJliZZkWTFmjVrtlwlkiRJkiRpVsx2A+M0YC9gf+Bu4INNfKaRE7WR+Mb2eXyw6vSqWlRVi+bNm/czJSxJkiRJkro3qw2MqrqnqtZV1SPAx4CDmptWAbuP3XUBcFcTXzBDfL19kswFdmLTT1mRJEmSJElbkVltYDRzWkx5PTC1QsnFwJJmZZE9GU3WeU1V3Q08lOTgZn6LtwMXje1zdHP5SOCKZp4MSZIkSZI0Yea29cBJPgm8HNglySrgvcDLk+zP6FSPO4BjAarq5iTLgVuAtcDxVbWueajjGK1osgNwabMBnAGcnWQlo5EXS9qqRZIkSZIkdau1BkZVvXmG8Bkbuf9JwEkzxFcA+84Q/zFw1ObkKEmSJEmStg5drEIiSZIkSZL0M7GBIUmSJEmSes8GhiRJkiRJ6j0bGJIkSZIkqfdsYEiSJEmSpN6zgSFJkiRJknrPBoYkSZIkSeo9GxiSJEmSJKn3bGBIkiRJkqTes4EhSZIkSZJ6zwaGJEmSJEnqPRsYkiRJkiSp92xgSJIkSZKk3rOBIUmSJEmSes8GhiRJkiRJ6j0bGJIkSZIkqfdsYEiSJEmSpN6zgSFJkiRJknrPBoYkSZIkSeo9GxiSJEmSJKn3WmtgJPl4ktVJbhqLPSvJZUm+3fzceey2E5OsTHJbkkPH4gcmubG57cNJ0sS3S3JeE786yR5t1SJJkiRJkrrV5giMTwCLp8VOAC6vqoXA5c11kuwNLAH2afY5NcmcZp/TgKXAwmabesxjgPur6gXAKcDJrVUiSZIkSZI61VoDo6q+Atw3LXw4sKy5vAw4Yix+blU9XFW3AyuBg5LsBuxYVVdWVQFnTdtn6rHOBw6ZGp0hSZIkSZImy2zPgfGcqroboPm5axOfD9w5dr9VTWx+c3l6fL19qmot8ADw7Jl+aZKlSVYkWbFmzZotVIokSZIkSZotfZnEc6aRE7WR+Mb2eXyw6vSqWlRVi+bNm/ckU5QkSZIkSV2Z7QbGPc1pITQ/VzfxVcDuY/dbANzVxBfMEF9vnyRzgZ14/CkrkiRJkiRpAsx2A+Ni4Ojm8tHARWPxJc3KInsymqzzmuY0k4eSHNzMb/H2aftMPdaRwBXNPBmSJEmSJGnCzG3rgZN8Eng5sEuSVcB7gT8Hlic5BvgecBRAVd2cZDlwC7AWOL6q1jUPdRyjFU12AC5tNoAzgLOTrGQ08mJJW7VIkiRJkqRutdbAqKo3b+CmQzZw/5OAk2aIrwD2nSH+Y5oGiCRJkiRJmmx9mcRTkiRJkiRpg2xgSJIkSZKk3rOBIUmSJEmSes8GhiRJkiRJ6j0bGJIkSZIkqfdsYEiSJEmSpN6zgSFJkiRJknrPBoYkSZIkSeo9GxiSJEmSJKn3bGBIkiRJkqTem9t1ApK2jEvOeE3XKWyW1xxzSdcpSJIkSeoxR2BIkiRJkqTes4EhSZIkSZJ6zwaGJEmSJEnqPRsYkiRJkiSp92xgSJIkSZKk3nMVEkmSJOkJvOmC/911Ck/aeW/8112nIElbhCMwJEmSJElS79nAkCRJkiRJvWcDQ5IkSZIk9Z4NDEmSJEmS1HudNDCS3JHkxiTXJ1nRxJ6V5LIk325+7jx2/xOTrExyW5JDx+IHNo+zMsmHk6SLeiRJkiRJUru6HIHxiqrav6oWNddPAC6vqoXA5c11kuwNLAH2ARYDpyaZ0+xzGrAUWNhsi2cxf0mSJEmSNEv6dArJ4cCy5vIy4Iix+LlV9XBV3Q6sBA5KshuwY1VdWVUFnDW2jyRJkiRJmiBdNTAK+EKSa5MsbWLPqaq7AZqfuzbx+cCdY/uuamLzm8vT44+TZGmSFUlWrFmzZguWIUmSJEmSZsPcjn7vS6vqriS7Apcl+dZG7jvTvBa1kfjjg1WnA6cDLFq0aMb7SJIkSZKk/upkBEZV3dX8XA1cCBwE3NOcFkLzc3Vz91XA7mO7LwDuauILZohLkiRJkqQJM+sNjCRPS/KMqcvAq4GbgIuBo5u7HQ1c1Fy+GFiSZLskezKarPOa5jSTh5Ic3Kw+8vaxfSRJkiRJ0gTp4hSS5wAXNiuezgX+rqo+l+QbwPIkxwDfA44CqKqbkywHbgHWAsdX1brmsY4DPgHsAFzabJIkSZIkacLMegOjqr4D7DdD/F7gkA3scxJw0gzxFcC+WzpHSZIkSZLUL31aRlWSJEmSJGlGNjAkSZIkSVLv2cCQJEmSJEm9ZwNDkiRJkiT1ng0MSZIkSZLUezYwJEmSJElS79nAkCRJkiRJvWcDQ5IkSZIk9Z4NDEmSJEmS1Hs2MCRJkiRJUu/ZwJAkSZIkSb1nA0OSJEmSJPWeDQxJkiRJktR7NjAkSZIkSVLv2cCQJEmSJEm9ZwNDkiRJkiT1ng0MSZIkSZLUezYwJEmSJElS79nAkCRJkiRJvWcDQ5IkSZIk9d5W38BIsjjJbUlWJjmh63wkSZIkSdKWN7frBDZHkjnAXwO/AqwCvpHk4qq6pdvMJEnauNdceHLXKTxpl7z+D7tOQZK2iCvOWdN1Ck/aK98yr+sUpFm3VTcwgIOAlVX1HYAk5wKHAzYwJEnqiddecEbXKTxpn3njMV2n0Fuvv+CLXaewWS584yu6TqG3Tv/U6q5TeNKWvmHXrlPorW9/5J6uU3jSFr7rOV2noJ5IVXWdw5OW5EhgcVX9h+b624AXV9W7pt1vKbC0ufrzwG2zmujILsD3O/i9XbDWyWStk2lItcKw6rXWyWStk8laJ9eQ6rXWydRVrc+vqscNM9raR2BkhtjjOjJVdTpwevvpbFiSFVW1qMscZou1TiZrnUxDqhWGVa+1TiZrnUzWOrmGVK+1Tqa+1bq1T+K5Cth97PoC4K6OcpEkSZIkSS3Z2hsY3wAWJtkzyVOAJcDFHeckSZIkSZK2sK36FJKqWpvkXcDngTnAx6vq5o7T2pBOT2GZZdY6max1Mg2pVhhWvdY6max1Mlnr5BpSvdY6mXpV61Y9iackSZIkSRqGrf0UEkmSJEmSNAA2MCRJkiRJUu/ZwJAkSZIkSb1nA0OSJEmSJPWeDYyWJdkvybuabb+u82lLkqcmeU+SjzXXFyZ5bdd5bWlJtklyU9d5zKYBHcPPmmHbtuu82jCkWscl2XG85q7zaUOSozYlNgmGUuvQ3neG8nli3EBem07elNikSfK0rnOYLUOodajHcd/YwGhRkt8BzgF2bba/TfLb3WbVmjOBh4F/21xfBXygu3TaUVWPAN9M8ryuc5kNAzuGrwPWAP8b+HZz+fYk1yU5sNPMtrwh1UqSY5PcA9wAXNtsK7rNqjUnbmJsEgyi1qG97zCQzxMwuNemX5kh9quznsUsSfKSJLcAtzbX90tyasdptWJItTK84/jXkvxBkv88tXWdE8DcrhOYcMcAL66qH8KjHborgb/qNKt27FVVb0ryZoCq+lGSdJ1US3YDbk5yDfDDqWBVva67lFozpGP4c8CFVfV5gCSvBhYDy4FTgRd3mNuWNqRaAf4TsE9Vfb/rRNqS5FeB1wDzk3x47KYdgbXdZNWOIdU6ZkjvO0P6PDGE16bjgN8Cfi7JDWM3PQP4WjdZzYpTgEOBiwGq6ptJfqnblFoz8bUO8ThO8lHgqcArgP8BHAlc02lSDRsY7Qqwbuz6uiY2iX6SZAegAJLsxegblEn0/q4TmEVDOoYXVdU7p65U1ReS/FlV/cck23WZWAuGVCvA/wH+peskWnYXo29uX8foW9wpDwG/10lG7RlSrVOG9L4zpM8TQ3ht+jvgUuC/AieMxR+qqvu6SWl2VNWd03pv6zZ0363dAGod4nH8kqp6UZIbqur9ST4IfKrrpMAGRtvOBK5OcmFz/QjgjO7SadV7GX2ru3uSc4CXAv++04xaUlVfTvJ8YGFV/UOSpwJzus6rJUM6hu9L8ofAuc31NwH3J5kDPNJdWq0YUq0wOq3g60muZuwfoap6d3cpbVnNN143Aa+uqmVd59OmIdU6ZWDvO4P5PMEwXpseAB4A3gyQZFdge+DpSZ5eVd/rMr8W3ZnkJUAleQrwbppTLCbQxNc60OP4R83Pf0nyXOBeYM8O83lUqqrrHCZakl8EXsboW+uvVNU/dpxSa5I8GziYUa1XTeqQyCS/CSwFnlVVeyVZCHy0qg7pOLVWDOUYTrILow/OL2tCXwX+C6M3rOdV1cquctvShlQrQDPs/qvAjYw1aCbxn98knwNeV1U/6TqXtg2s1qG97wzl88SQXpsOAz4EPBdYDTwfuLWq9uk0sZY077N/CbyK0XH8BeB3qureThNrwcBqHcxxnOQ9jE4ZPwT4a0aj4v5HVb2n08RwBMZseCqj4UVnJpmXZM+qur3rpFryy4z+ISpgW+DCjd99q3U8cBBwNUBVfbvpxE6qiT+Gm5EHf1FVb93AXSbmH/oh1TpmbVX9x66TmCXfBb6W5GLWnyvhQ92l1Joh1Tq0952hfJ4Y0mvTBxg1pf6hqg5I8gqab7MnVKrqLV0nMUuGVOtgjuOq+tPm4gVJPgNs34xE6ZwNjBYleS+wCPh5RkPxtwX+ltFwyInSzDb8AuCTTejYJK+qquM7TKstD1fVT6bO9Usyl+Zc3UkzlGO4qtY1zZmnTPq3uUOqdcwXkywFPs36w7Qn8bzVu5ptG0aTi02yIdU6pPedIX2eGNJr00+r6t6MlgXepqq+mMlefvLrSW4HzgMuqKofdJxPm4ZU62CO4+YLr18D9qDpGSTpxZcENjDa9XrgAEZLFlJVdyWZ1A9ZvwzsW805SUmWMRoSOYm+nOSPgB2S/AqjWYk/3XFObRnSMXwHw/k29w6GUyvAv2t+ji+xWcDPdZBLq6pqMJM9DqlWhvW+M6TPE4N5bQJ+kOTpwFeAc5KsZnJXDaKqFiY5CFgC/HFGy4yeW1V/23FqW9yQamVYx/GngR8z7RS3PrCB0a6fVFUlmXoTflrXCbXoNuB5jIb0AuzOaF3zSXQCo+VFbwSOBS5htLzQJBrSMTykb3OHVCtV1YtJp2ZDknnAHwD7MJpgDICqemVnSbVkSLUyrPedwXyeGNJrE3A4o3+Gfg94C7ATo7mXJlZVXQNck+TPGM2bsIzRKNaJM6Bah3QcL6iqF3WdxExsYLRreZK/AZ7ZTMD1DuBjHefUlmcDtzYTUgH8f8CVzTe8E7VWfVU90nwjdDWjb0pum/qmaAIN5hie+ja3GWFSVfXPHafUmqHUmuSVVXVFkjfMdHtV9WI5sC3sHEbDeF8LvBM4GljTaUbtGUytA3vfmfjPE0N8baqqH45dnbhJSqdLsiOjUaxLgL0YzeNyUKdJtWRItQ7sOL40yaur6gtdJzKdq5C0rBnq+WpGs/J+vqou6zilViT55Y3dXlVfnq1c2pbk14CPMlq/PYyWFDq2qi7tNLGWDOgY3hc4G3hWE/o+8Paqurm7rNoxlFqTvL+q3pvkzBlurqp6x6wn1bIk11bVgRmt2/6iJvblqtroa/TWaGC1DuZ9ZwifJwb62vQG4GRgV0bHcBjVumOnibWkmRPifwLLq+rKjtNp1cBqHcxxnOT1jEbRbAP8lB7VagND+hkl+Rbw2qmlJpPsBXy2ql7YbWbaHEm+DvxxVX2xuf5y4M+q6iVd5tWGIdUKMNPKOZO4mg5Akquq6uAknwc+zOhUofOraq+OU9viBlar7zsTaGCvTSuBw6rq1q5zmQ1J0pyC+7Rp39pPnIHVOpjjOMl3gCOAG/s24m+brhOYZEkeSvLgtO3OJBcmmagJmoZUK7B66kNk4zuM1oKeOAN7Xp829Q89QFV9CZjUOT+GVCvABTPEzp/1LGbHB5LsBPw+8J8YzZPwe92m1Joh1er7zmS+7wzptemeIfzTN+bgZjLLWwGS7JfRCjuTaEi1Duk4/jZwU9+aF+AcGG37EKNvhP6O0bCbJcC/YjRB1ceBl3eW2ZY3pFpvTnIJsJzRuchHAd+YOpd1ws5dHdLz+p0k72F0agXAW4GJ+xasMYhak7yQ0QSPO00713xHxiZ9nCRV9Znm4gPAK7rMpW1DqhXfdybqfWeIr03AiiTnMTrVYHzJ2Ek6dsf9BXAoMDV3yzeT/FKnGbXnLxhOrUM6ju8GvpTkUtavtfMV6zyFpEVJrq6qF0+LTQ15/WZV7ddVblvawGqd6ZzVKRN17urAntedgfcDL2P0ofnLwPur6v5OE2vBUGpNcjij4Y+vo/lg1XiI0RJvX+8ir9mS5Lqq+sWu85gNk16r7zuT9b4zxNemIc33AY8dx0n+saoOaGITcfxON7BaB3McJ3nvTPHqwRLmjsBo1yNJfp3HhgMeOXbbpHWOBlNrVf1G1znMoiE9r/cD7+46j9kwlFqr6iLgoiT/dtInFtuAdJ3ALJroWn3fedREvO8M8bVpYMcwwJ1JXgJUkqcwes+d1FMPBlPrkI7jPjQqNsQ5MNr1FuBtjM5TXd1cfmuSHYB3dZlYC4ZU66OSXNd1Di3zeZ1wQ6h1/B+EIdQ75rNdJzCLBlPrAI7hwbzvDPW1aSC1vhM4HpgPrAL2b65PoiHV+qhJP46TLGrmHrouyQ1TW9d5gaeQSJtlfLicJseQntch1QrDqjfJLsC9fZyAS0/ekI7hIRnS8zqkWjW5Jv04TnIb8P8DNwKPTMWr6rudJdVwBMYsmfQu3bgh1Yrf+k2qwTyvDKtWmNB6kxyc5EtJPpXkgCQ3ATcB9yRZ3HV+W9IGVql4cCredX6zYCKP4Zn4vjOxhlTroI7jIdXK5B/Ha6rq4qq6vaq+O7V1nRQ4AmPWTHqXbtzAah3MN5w+r5oEk/rcJlkB/BGwE3A68KtVdVWz2sEnh/K3q8kypPedIZnU1+ENGdJxPKRaJ12SQ4A3A5fTsxVXHIExeya9SzduImsd0jecG+DzupUb2jfXQ3pugblV9YWq+nvg/1bVVQBV9a2O89JmGNrf7Awm9X1nMM/rwF6HN2Qij+MNmMhah/Q3O+Y3GM1nshg4rNle22VCUxyBMUuG1G2e1Fr9hnMy+bxOriE9txlbRjTTlhSdfl2SZsuQXoc3ZFI/F2uyJbmxqv5N13nMxBEYLRhSt3lItTKgbzgH1mkezPM6QEN6bveb+hsFXjT+Nwv08gOING5g7ztDMqTX4UF9LvZvduJdlWTvrpOYydyuE5hQH+GxbvMVTOs2A5/rMrktbEi1PjJ2+UfTbpuornpVPaPrHGbRYJ7XARrMc1tVc7rOQdocA3vfGZLBvA43BvO52L/Zifcy4OgktzOaAyNAVdWLuk3LU0hakeT6qtq/uXxrVf3C2G0TNbnNwGpdB/yQ0R/wDsC/TN0EbF9V23aVm548n9fJ5XMrSd0a2uvwkD4Xa7Ilef5M8T6sROIIjHYMqds8mFr9hnMy+bxOLp9bSerWAF+HB/O5WJOtqr6b5GXAwqo6M8k84Old5wWOwGjFkLrNQ6pVkiRJ2hA/F2tSJHkvsAj4+ar610meC/x9Vb2049QcgdGGIXWbh1SrJEmStCF+LtYEeT1wAHAdQFXdlaQX8564CokkSZIkSZryk2bp3wJI8rSO83mUDQxJkiRJkjRleZK/AZ6Z5DeBfwA+1nFOgHNgSJIkSZKkMUl+BXg1ozlcPl9Vl3WcEmADQ5IkSZIkbQWcxFOSJEmSpIFL8hAzL/kboKpqx1lO6fGJOAJDkiRJkiT1nZN4SpIkSZKkx0mytOscxtnAkCRJkiRJM3ln1wmMs4EhSZIkSZJmkq4TGGcDQ5IkSZIkAZDkgLGrh3WWyAxsYEiSJEmSpCkfSvKtJH8K7NR1MuNsYEiSJEmSJACq6hXAy4E1wOlJbkzyJ91mNeIyqpIkSZIk6XGS/BvgD4A3VdVTus7HERiSJEmSJAmAJL+Q5H1JbgY+AnwdWNBxWoAjMCRJkiRJUiPJVcAngb+vqru6zmecDQxJkiRJkvSoJDsAz6uq27rOZZynkEiSJEmSJACSHAZcD3yuub5/kos7TaphA0OSJEmSJE15H3AQ8AOAqroe2KOzbMbYwJAkSZIkSVPWVtUDXScxk7ldJyBJkiRJknrjpiT/DpiTZCHwbkYrkXTOERiSJEmSJGnKbwP7AA8Dfwc8APxulwlNcRUSSZIkSZJEkjnA56vqVV3nMhNHYEiSJEmSJKpqHfAvSXbqOpeZOAeGJEmSJEma8mPgxiSXAT+cClbVu7tLacQGhiRJkiRJmvLZZusd58CQJEmSJEmPSvIU4IVAAbdV1U86TgmwgSFJkiRJkhpJXgP8DfB/gAB7AsdW1aWdJoYNDEmSJEmS1EjyLeC1VbWyub4X8NmqemG3mbkKiSRJkiRJeszqqeZF4zvA6q6SGecIDEmSJEmSBECS04DnA8sZzYFxFHAb8DWAqvpUZ7nZwJAkSZIkSQBJztzIzVVV75i1ZKaxgSFJkiRJknrPOTAkSZIkSdLjJLmu6xzG2cCQJEmSJEkzSdcJjLOBIUmSJEmSZvLZrhMY5xwYkiRJkiQJgCQXA+cCF1XVD7vOZ5wjMCRJkiRJ0pQPAi8Dbkny90mOTLJ910mBIzAkSZIkSdI0SeYArwR+E1hcVTt2nBJzu05AkiRJkiT1R5IdgMOANwG/CCzrNqMRR2BIkiRJkiQAkpwHvBj4HLAc+FJVPdJtViM2MCRJkiRJEgBJFgOXVdW6rnOZzkk8JUmSJEnSlK8AJyY5HSDJwiSv7TgnwAaGJEmSJEl6zJnAT4CXNNdXAR/oLp3H2MCQJEmSJElT9qqq/wb8FKCqfgSk25RGbGBIkiRJkqQpP2lWISmAJHsBD3eb0ojLqEqSJEmSpCnvZbQCye5JzgFeCvz7TjNquAqJJEmSJEkiyTbAkcDlwMGMTh25qqq+32liDRsYkiRJkiQJgCRfqapf6jqPmdjAkCRJkiRJACR5D/Aj4Dzgh1Pxqrqvs6QaNjAkSZIkSRIASW6nmcBzXFX9XAfprMcGhiRJkiRJAqBZgeS3gJcxamT8L+CjzXKqnbKBIUmSJEmSAEiyHHgQOKcJvRl4ZlX9endZjdjAkCRJkiRJACT5ZlXt90SxLmzTdQKSJEmSJKk3/jHJwVNXkrwY+FqH+TzKERiSJEmSJAmAJLcCPw98rwk9D7gVeASoqnpRZ7nZwJAkSZIkSQBJnr+x26vqu7OVy3Q2MCRJkiRJUu85B4YkSZIkSeo9GxiSJEmSJKn3bGBIkqReS/LMJL/VdR6SJKlbNjAkSVLfPROwgSFJ0sDN7ToBSZKkJ/DnwF5Jrge+CLwI2BnYFviTqroIIMl7gLcAdwLfB66tqv/eScaSJGmLs4EhSZL67gRg36raP8lc4KlV9WCSXYCrklwMHAi8ETiA0eeb64BrO8tYkiRtcTYwJEnS1iTAnyX5JeARYD7wHOBlwEVV9SOAJJ/uLkVJktQGGxiSJGlr8hZgHnBgVf00yR3A9owaG5IkaYI5iackSeq7h4BnNJd3AlY3zYtXAM9v4l8FDkuyfZKnA7/WQZ6SJKlFjsCQJEm9VlX3JvlakpuAbwAvTLICuB74VnOfbzRzYXwT+C6wAnigo5QlSVILUlVd5yBJkrTZkjy9qv45yVOBrwBLq+q6rvOSJElbhiMwJEnSpDg9yd6M5sRYZvNCkqTJ4ggMSZIkSZLUe07iKUmSJEmSes8GhiRJkiRJ6j0bGJIkSZIkqfdsYEiSJEmSpN6zgSFJkiRJknrv/wF9s9sInr1QaQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15, 5))\n",
    "ax = sns.countplot(x='tag', data=data.loc[data['tag'] != 'O'])\n",
    "ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha=\"center\")\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c1956bd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SentenceGetter(object):\n",
    "    \n",
    "    def __init__(self, dataset):\n",
    "        self.n_sent = 1\n",
    "        self.dataset = dataset\n",
    "        self.empty = False\n",
    "        agg_func = lambda s: [(w, t) for w,t in zip(s[\"word\"].values.tolist(),\n",
    "                                                        s[\"tag\"].values.tolist())]\n",
    "        self.grouped = self.dataset.groupby(\"sentence_idx\").apply(agg_func)\n",
    "        self.sentences = [s for s in self.grouped]\n",
    "    \n",
    "    def get_next(self):\n",
    "        try:\n",
    "            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
    "            self.n_sent += 1\n",
    "            return s\n",
    "        except:\n",
    "            return None\n",
    "          \n",
    "getter = SentenceGetter(data)\n",
    "sentences = getter.sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2104394e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Thousands', 'O'), ('of', 'O'), ('demonstrators', 'O'), ('have', 'O'), ('marched', 'O'), ('through', 'O'), ('London', 'B-geo'), ('to', 'O'), ('protest', 'O'), ('the', 'O'), ('war', 'O'), ('in', 'O'), ('Iraq', 'B-geo'), ('and', 'O'), ('demand', 'O'), ('the', 'O'), ('withdrawal', 'O'), ('of', 'O'), ('British', 'B-gpe'), ('troops', 'O'), ('from', 'O'), ('that', 'O'), ('country', 'O'), ('.', 'O'), ('Thousands', 'O'), ('of', 'O'), ('demonstrators', 'O'), ('have', 'O'), ('marched', 'O'), ('through', 'O'), ('London', 'B-geo'), ('to', 'O'), ('protest', 'O'), ('the', 'O'), ('war', 'O'), ('in', 'O'), ('Iraq', 'B-geo'), ('and', 'O'), ('demand', 'O'), ('the', 'O'), ('withdrawal', 'O'), ('of', 'O'), ('British', 'B-gpe'), ('troops', 'O'), ('from', 'O'), ('that', 'O'), ('country', 'O'), ('.', 'O')]\n"
     ]
    }
   ],
   "source": [
    "print(sentences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5a880423",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30128"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = list(set(data[\"word\"].values))\n",
    "n_words = len(words)\n",
    "n_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a54166bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags = []\n",
    "i=0\n",
    "for tag in set(data[\"tag\"].values):\n",
    "    if tag is nan or isinstance(tag, float):\n",
    "        tags.append('unk')\n",
    "    else:\n",
    "        tags.append(tag)\n",
    "n_tags = len(tags)\n",
    "n_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "08620723",
   "metadata": {},
   "outputs": [],
   "source": [
    "from future.utils import iteritems\n",
    "\n",
    "word2idx = {w: i for i, w in enumerate(words)}\n",
    "tag2idx = {t: i for i, t in enumerate(tags)}\n",
    "idx2tag = {v: k for k, v in iteritems(tag2idx)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "53324581",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "maxlen = max([len(s) for s in sentences])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b0d42c64",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [[word2idx[w[0]] for w in s] for s in sentences]\n",
    "X = pad_sequences(maxlen=maxlen, sequences=X, padding=\"post\",value=n_words - 1)\n",
    "\n",
    "y = [[tag2idx[w[1]] for w in s] for s in sentences]\n",
    "y = pad_sequences(maxlen=maxlen, sequences=y, padding=\"post\", value=tag2idx[\"O\"])\n",
    "y = [to_categorical(i, num_classes=n_tags) for i in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "253592e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ec4c654e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 140)]             0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 140, 150)          4519200   \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_1 (Spatial (None, 140, 150)          0         \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 140, 200)          200800    \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 140, 19)           3819      \n",
      "=================================================================\n",
      "Total params: 4,723,819\n",
      "Trainable params: 4,723,819\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model, Input\n",
    "from keras.layers import LSTM, Embedding, Dense,SpatialDropout1D, TimeDistributed, Dropout, Bidirectional\n",
    "import keras\n",
    "import tensorflow as tf\n",
    "def mymodel():\n",
    "    word_embedding_size = 150\n",
    "    input_word = Input(shape=(maxlen,))\n",
    "    model = Embedding(input_dim=n_words,\n",
    "                      output_dim=word_embedding_size,\n",
    "                      input_length=maxlen)(input_word)\n",
    "    model = SpatialDropout1D(0.1)(model)\n",
    "    model = Bidirectional(LSTM(units=100,\n",
    "                               return_sequences=True,\n",
    "                               recurrent_dropout=0.1,\n",
    "                               activation=\"tanh\",\n",
    "                               recurrent_activation=\"sigmoid\"))(model)\n",
    "    out = TimeDistributed(Dense(n_tags,activation='softmax'))(model)\n",
    "    model = Model(input_word,out)\n",
    "\n",
    "    model.compile(optimizer = 'adam',\n",
    "                  loss='categorical_crossentropy',\n",
    "                 metrics=['accuracy'])\n",
    "    return model\n",
    "model = mymodel()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "dbe885bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "99/99 [==============================] - 243s 2s/step - loss: 0.8550 - accuracy: 0.9185 - val_loss: 0.1461 - val_accuracy: 0.9673\n",
      "Epoch 2/20\n",
      "99/99 [==============================] - 267s 3s/step - loss: 0.1290 - accuracy: 0.9677 - val_loss: 0.0909 - val_accuracy: 0.9731\n",
      "Epoch 3/20\n",
      "99/99 [==============================] - 244s 2s/step - loss: 0.0825 - accuracy: 0.9747 - val_loss: 0.0708 - val_accuracy: 0.9786\n",
      "Epoch 4/20\n",
      "99/99 [==============================] - 251s 3s/step - loss: 0.0614 - accuracy: 0.9824 - val_loss: 0.0494 - val_accuracy: 0.9872\n",
      "Epoch 5/20\n",
      "99/99 [==============================] - 288s 3s/step - loss: 0.0397 - accuracy: 0.9897 - val_loss: 0.0384 - val_accuracy: 0.9899\n",
      "Epoch 6/20\n",
      "99/99 [==============================] - 268s 3s/step - loss: 0.0291 - accuracy: 0.9922 - val_loss: 0.0344 - val_accuracy: 0.9906\n",
      "Epoch 7/20\n",
      "99/99 [==============================] - 272s 3s/step - loss: 0.0240 - accuracy: 0.9934 - val_loss: 0.0332 - val_accuracy: 0.9910\n",
      "Epoch 8/20\n",
      "99/99 [==============================] - 274s 3s/step - loss: 0.0215 - accuracy: 0.9940 - val_loss: 0.0328 - val_accuracy: 0.9911\n",
      "Epoch 9/20\n",
      "45/99 [============>.................] - ETA: 2:20 - loss: 0.0191 - accuracy: 0.9946"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-ebb52ff4dd52>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Fit the best model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mhistory\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m256\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1098\u001b[0m                 _r=1):\n\u001b[0;32m   1099\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1100\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1101\u001b[0m               \u001b[1;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1102\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    826\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    827\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 828\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    829\u001b[0m       \u001b[0mcompiler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"xla\"\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;34m\"nonXla\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    830\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    853\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    854\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 855\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    856\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    857\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2940\u001b[0m       (graph_function,\n\u001b[0;32m   2941\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m-> 2942\u001b[1;33m     return graph_function._call_flat(\n\u001b[0m\u001b[0;32m   2943\u001b[0m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0;32m   2944\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1916\u001b[0m         and executing_eagerly):\n\u001b[0;32m   1917\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1918\u001b[1;33m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[0;32m   1919\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0;32m   1920\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    553\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    554\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 555\u001b[1;33m           outputs = execute.execute(\n\u001b[0m\u001b[0;32m    556\u001b[0m               \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    557\u001b[0m               \u001b[0mnum_outputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     57\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 59\u001b[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[0;32m     60\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "early_stopping = EarlyStopping(monitor='val_accuracy',patience=1, verbose=0, mode='max',restore_best_weights= False)\n",
    "callbacks = [early_stopping]\n",
    "\n",
    "# Fit the best model\n",
    "history = model.fit(X_train, \n",
    "                    np.array(y_train), \n",
    "                    batch_size=256, \n",
    "                    epochs=20, \n",
    "                    validation_split=0.1, \n",
    "                    verbose=1, \n",
    "                    callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed84e41b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.evaluate(X_test, np.array(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7711dc0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('Model.h5')"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}