pre_process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3c55a8c8-44a3-4e44-9499-cb41def0d27a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>Two giant cranes holding a bridge collapse int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>@aria_ahrary @TheTawniest The out of control w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>Police investigating after an e-bike collided ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>The Latest: More Homes Razed by Northern Calif...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     Our Deeds are the Reason of this #earthquake M...\n",
       "1                Forest fire near La Ronge Sask. Canada\n",
       "2     All residents asked to 'shelter in place' are ...\n",
       "3     13,000 people receive #wildfires evacuation or...\n",
       "4     Just got sent this photo from Ruby #Alaska as ...\n",
       "...                                                 ...\n",
       "7608  Two giant cranes holding a bridge collapse int...\n",
       "7609  @aria_ahrary @TheTawniest The out of control w...\n",
       "7610  M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...\n",
       "7611  Police investigating after an e-bike collided ...\n",
       "7612  The Latest: More Homes Razed by Northern Calif...\n",
       "\n",
       "[7613 rows x 1 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import re\n",
    "\n",
    "df_train = pd.read_csv(\"dataset/train.csv\")  #changed from \"train.csv\"\n",
    "df_test = pd.read_csv(\"dataset/test.csv\")\n",
    "\n",
    "\n",
    "\n",
    "df_train[['text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e019362e-569c-4b99-8099-fbfa7073c4e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleaned(text):\n",
    "    text = re.sub(r\"\\n\",\"\",text)\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"\\d\",\"\",text)        #Remove digits\n",
    "    text = re.sub(r'[^\\x00-\\x7f]',r' ',text) # remove non-ascii\n",
    "    text = re.sub(r'[^\\w\\s]','',text) #Remove punctuation\n",
    "    text = re.sub(r'http\\S+|www.\\S+', '', text) #Remove http\n",
    "    return text\n",
    "\n",
    "df_train['text'] = df_train['text'].apply(lambda x : cleaned(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ddbdb452-90f7-46a0-b9c3-79bb38c49fbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>our deeds are the reason of this earthquake ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>forest fire near la ronge sask canada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>all residents asked to shelter in place are be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>people receive wildfires evacuation orders in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>just got sent this photo from ruby alaska as s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>two giant cranes holding a bridge collapse int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>aria_ahrary thetawniest the out of control wil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>m  utckm s of volcano hawaii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>police investigating after an ebike collided w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>the latest more homes razed by northern califo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     our deeds are the reason of this earthquake ma...\n",
       "1                 forest fire near la ronge sask canada\n",
       "2     all residents asked to shelter in place are be...\n",
       "3      people receive wildfires evacuation orders in...\n",
       "4     just got sent this photo from ruby alaska as s...\n",
       "...                                                 ...\n",
       "7608  two giant cranes holding a bridge collapse int...\n",
       "7609  aria_ahrary thetawniest the out of control wil...\n",
       "7610                      m  utckm s of volcano hawaii \n",
       "7611  police investigating after an ebike collided w...\n",
       "7612  the latest more homes razed by northern califo...\n",
       "\n",
       "[7613 rows x 1 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[['text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0e759599-41a3-4736-90b7-b4e58cd9f5f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>our deeds are the reason of this earthquake ma...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>forest fire near la ronge sask canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>all residents asked to shelter in place are be...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>people receive wildfires evacuation orders in...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>just got sent this photo from ruby alaska as s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>10869</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>two giant cranes holding a bridge collapse int...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>10870</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>aria_ahrary thetawniest the out of control wil...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>10871</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>m  utckm s of volcano hawaii</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>10872</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>police investigating after an ebike collided w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>10873</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>the latest more homes razed by northern califo...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id keyword location  \\\n",
       "0         1     NaN      NaN   \n",
       "1         4     NaN      NaN   \n",
       "2         5     NaN      NaN   \n",
       "3         6     NaN      NaN   \n",
       "4         7     NaN      NaN   \n",
       "...     ...     ...      ...   \n",
       "7608  10869     NaN      NaN   \n",
       "7609  10870     NaN      NaN   \n",
       "7610  10871     NaN      NaN   \n",
       "7611  10872     NaN      NaN   \n",
       "7612  10873     NaN      NaN   \n",
       "\n",
       "                                                   text  target  \n",
       "0     our deeds are the reason of this earthquake ma...       1  \n",
       "1                 forest fire near la ronge sask canada       1  \n",
       "2     all residents asked to shelter in place are be...       1  \n",
       "3      people receive wildfires evacuation orders in...       1  \n",
       "4     just got sent this photo from ruby alaska as s...       1  \n",
       "...                                                 ...     ...  \n",
       "7608  two giant cranes holding a bridge collapse int...       1  \n",
       "7609  aria_ahrary thetawniest the out of control wil...       1  \n",
       "7610                      m  utckm s of volcano hawaii        1  \n",
       "7611  police investigating after an ebike collided w...       1  \n",
       "7612  the latest more homes razed by northern califo...       1  \n",
       "\n",
       "[7613 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e794b5bc-cf3c-415b-b208-eda6a922d320",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target_ground_T</th>\n",
       "      <th>target</th>\n",
       "      <th>prob_DIS_TRUE</th>\n",
       "      <th>prob_DIS_FALSE</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Communal violence in Bhainsa, Telangana. \"Ston...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.898958</td>\n",
       "      <td>0.118938</td>\n",
       "      <td>Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Telangana: Section 144 has been imposed in Bha...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.809305</td>\n",
       "      <td>0.215982</td>\n",
       "      <td>Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>New York City</td>\n",
       "      <td>Arsonist sets cars ablaze at dealership https:...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.648180</td>\n",
       "      <td>0.335497</td>\n",
       "      <td>Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>Morgantown, WV</td>\n",
       "      <td>Arsonist sets cars ablaze at dealership https:...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.570108</td>\n",
       "      <td>0.406788</td>\n",
       "      <td>Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\"Lord Jesus, your love brings freedom and pard...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.175745</td>\n",
       "      <td>0.791633</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11365</th>\n",
       "      <td>11365</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>Blue State in a red sea</td>\n",
       "      <td>Media should have warned us well in advance. T...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.320826</td>\n",
       "      <td>0.662948</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11366</th>\n",
       "      <td>11366</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>arohaonces</td>\n",
       "      <td>i feel directly attacked 💀 i consider moonbin ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.165259</td>\n",
       "      <td>0.792880</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11367</th>\n",
       "      <td>11367</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>🇵🇭</td>\n",
       "      <td>i feel directly attacked 💀 i consider moonbin ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.174767</td>\n",
       "      <td>0.787528</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11368</th>\n",
       "      <td>11368</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>auroraborealis</td>\n",
       "      <td>ok who remember \"outcast\" nd the \"dora\" au?? T...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.163032</td>\n",
       "      <td>0.793749</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11369</th>\n",
       "      <td>11369</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Jake Corway wrecked while running 14th at IRP.</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.495178</td>\n",
       "      <td>0.496758</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11370 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  keyword                 location  \\\n",
       "0          0   ablaze                      NaN   \n",
       "1          1   ablaze                      NaN   \n",
       "2          2   ablaze            New York City   \n",
       "3          3   ablaze           Morgantown, WV   \n",
       "4          4   ablaze                      NaN   \n",
       "...      ...      ...                      ...   \n",
       "11365  11365  wrecked  Blue State in a red sea   \n",
       "11366  11366  wrecked               arohaonces   \n",
       "11367  11367  wrecked                       🇵🇭   \n",
       "11368  11368  wrecked           auroraborealis   \n",
       "11369  11369  wrecked                      NaN   \n",
       "\n",
       "                                                    text  target_ground_T  \\\n",
       "0      Communal violence in Bhainsa, Telangana. \"Ston...                1   \n",
       "1      Telangana: Section 144 has been imposed in Bha...                1   \n",
       "2      Arsonist sets cars ablaze at dealership https:...                1   \n",
       "3      Arsonist sets cars ablaze at dealership https:...                1   \n",
       "4      \"Lord Jesus, your love brings freedom and pard...                0   \n",
       "...                                                  ...              ...   \n",
       "11365  Media should have warned us well in advance. T...                0   \n",
       "11366  i feel directly attacked 💀 i consider moonbin ...                0   \n",
       "11367  i feel directly attacked 💀 i consider moonbin ...                0   \n",
       "11368  ok who remember \"outcast\" nd the \"dora\" au?? T...                0   \n",
       "11369     Jake Corway wrecked while running 14th at IRP.                1   \n",
       "\n",
       "       target  prob_DIS_TRUE  prob_DIS_FALSE      prediction  \n",
       "0           1       0.898958        0.118938        Disaster  \n",
       "1           1       0.809305        0.215982        Disaster  \n",
       "2           0       0.648180        0.335497        Disaster  \n",
       "3           0       0.570108        0.406788        Disaster  \n",
       "4           0       0.175745        0.791633  NOT a Disaster  \n",
       "...       ...            ...             ...             ...  \n",
       "11365       0       0.320826        0.662948  NOT a Disaster  \n",
       "11366       0       0.165259        0.792880  NOT a Disaster  \n",
       "11367       0       0.174767        0.787528  NOT a Disaster  \n",
       "11368       0       0.163032        0.793749  NOT a Disaster  \n",
       "11369       0       0.495178        0.496758  NOT a Disaster  \n",
       "\n",
       "[11370 rows x 9 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rwo filter\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import re\n",
    "\n",
    "df_new = pd.read_csv(\"results/csv/_23_10_15_22_14_05.csv\")  #changed from \"train.csv\"\n",
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2018c1f3-665c-4053-866c-36f0313abbe9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target_ground_T</th>\n",
       "      <th>target</th>\n",
       "      <th>prob_DIS_TRUE</th>\n",
       "      <th>prob_DIS_FALSE</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Communal violence in Bhainsa, Telangana. \"Ston...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.898958</td>\n",
       "      <td>0.118938</td>\n",
       "      <td>Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Telangana: Section 144 has been imposed in Bha...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.809305</td>\n",
       "      <td>0.215982</td>\n",
       "      <td>Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\"Lord Jesus, your love brings freedom and pard...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.175745</td>\n",
       "      <td>0.791633</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>OC</td>\n",
       "      <td>If this child was Chinese, this tweet would ha...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.248484</td>\n",
       "      <td>0.730252</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>London, England</td>\n",
       "      <td>Several houses have been set ablaze in Ngemsib...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.825068</td>\n",
       "      <td>0.194716</td>\n",
       "      <td>Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11363</th>\n",
       "      <td>11363</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>NaN</td>\n",
       "      <td>He's the oxygen that pumps blood to my living ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.217445</td>\n",
       "      <td>0.770807</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11364</th>\n",
       "      <td>11364</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Had these guys last game n fcked them. Talked ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.170426</td>\n",
       "      <td>0.790354</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11366</th>\n",
       "      <td>11366</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>arohaonces</td>\n",
       "      <td>i feel directly attacked 💀 i consider moonbin ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.165259</td>\n",
       "      <td>0.792880</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11367</th>\n",
       "      <td>11367</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>🇵🇭</td>\n",
       "      <td>i feel directly attacked 💀 i consider moonbin ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.174767</td>\n",
       "      <td>0.787528</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11368</th>\n",
       "      <td>11368</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>auroraborealis</td>\n",
       "      <td>ok who remember \"outcast\" nd the \"dora\" au?? T...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.163032</td>\n",
       "      <td>0.793749</td>\n",
       "      <td>NOT a Disaster</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7406 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  keyword         location  \\\n",
       "0          0   ablaze              NaN   \n",
       "1          1   ablaze              NaN   \n",
       "4          4   ablaze              NaN   \n",
       "5          5   ablaze               OC   \n",
       "6          6   ablaze  London, England   \n",
       "...      ...      ...              ...   \n",
       "11363  11363  wrecked              NaN   \n",
       "11364  11364  wrecked              NaN   \n",
       "11366  11366  wrecked       arohaonces   \n",
       "11367  11367  wrecked               🇵🇭   \n",
       "11368  11368  wrecked   auroraborealis   \n",
       "\n",
       "                                                    text  target_ground_T  \\\n",
       "0      Communal violence in Bhainsa, Telangana. \"Ston...                1   \n",
       "1      Telangana: Section 144 has been imposed in Bha...                1   \n",
       "4      \"Lord Jesus, your love brings freedom and pard...                0   \n",
       "5      If this child was Chinese, this tweet would ha...                0   \n",
       "6      Several houses have been set ablaze in Ngemsib...                1   \n",
       "...                                                  ...              ...   \n",
       "11363  He's the oxygen that pumps blood to my living ...                0   \n",
       "11364  Had these guys last game n fcked them. Talked ...                0   \n",
       "11366  i feel directly attacked 💀 i consider moonbin ...                0   \n",
       "11367  i feel directly attacked 💀 i consider moonbin ...                0   \n",
       "11368  ok who remember \"outcast\" nd the \"dora\" au?? T...                0   \n",
       "\n",
       "       target  prob_DIS_TRUE  prob_DIS_FALSE      prediction  \n",
       "0           1       0.898958        0.118938        Disaster  \n",
       "1           1       0.809305        0.215982        Disaster  \n",
       "4           0       0.175745        0.791633  NOT a Disaster  \n",
       "5           0       0.248484        0.730252  NOT a Disaster  \n",
       "6           1       0.825068        0.194716        Disaster  \n",
       "...       ...            ...             ...             ...  \n",
       "11363       0       0.217445        0.770807  NOT a Disaster  \n",
       "11364       0       0.170426        0.790354  NOT a Disaster  \n",
       "11366       0       0.165259        0.792880  NOT a Disaster  \n",
       "11367       0       0.174767        0.787528  NOT a Disaster  \n",
       "11368       0       0.163032        0.793749  NOT a Disaster  \n",
       "\n",
       "[7406 rows x 9 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_new_ = df_new[df_new[\"prob_DIS_TRUE\"]>0.85]\n",
    "\n",
    "# or df_new[\"prob_DIS_FALSE\"]>0.85\n",
    "\n",
    "# filtered_values = np.where((dataFrame['Salary']>=100000) & (dataFrame['Age']< 40) & (dataFrame['JOB'].str.startswith('D')))\n",
    "# dataFrame[filtered_values]\n",
    "\n",
    "df_ = df_new.loc[(df_new['prob_DIS_TRUE']>=0.70) | (df_new['prob_DIS_FALSE']>0.70)]\n",
    "\n",
    "# df_new_\n",
    "# predictions_df.loc[predictions_df[\"prob_DIS_TRUE\"] < 0.75, \"target\"] = 0\n",
    "df_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "acae2ec5-4225-4a37-b43f-940f953950c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "new_training_rows :  7406\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\\nnew_training_rows : \",len(df_),end=\"\\n\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1d834413-0a05-41f3-9593-6a44ff5514e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "# import matplotlib.pyplot as plt\n",
    "# import seaborn as sns\n",
    "# import scipy.stats as stats\n",
    "\n",
    "from nltk.tokenize import word_tokenize,sent_tokenize\n",
    "from string import punctuation\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer,LancasterStemmer\n",
    "\n",
    "from contractions import fix \n",
    "from unidecode import unidecode\n",
    "# import pandas as pd \n",
    "\n",
    "# from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
    "# from sklearn.naive_bayes import MultinomialNB,BernoulliNB,GaussianNB\n",
    "# from sklearn.metrics import accuracy_score,classification_report,confusion_matrix \n",
    "# from sklearn.model_selection import train_test_split,GridSearchCV,RandomizedSearchCV, StratifiedKFold\n",
    "\n",
    "# import pandas\n",
    "# from tqdm import tqdm\n",
    "# tqdm.pandas()\n",
    "# from wordcloud import WordCloud\n",
    "# from gensim.models import Word2Vec,doc2vec\n",
    "\n",
    "# from sklearn.linear_model import LogisticRegression\n",
    "# from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\n",
    "# from sklearn.svm import SVC,SVR\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "df_train = pd.read_csv(\"dataset/train.csv\")  #changed from \"train.csv\"\n",
    "df_test = pd.read_csv(\"dataset/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "34ce0c23-a273-4051-af16-4ddc488a2aeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>10869</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Two giant cranes holding a bridge collapse int...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>10870</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@aria_ahrary @TheTawniest The out of control w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>10871</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>10872</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Police investigating after an e-bike collided ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>10873</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Latest: More Homes Razed by Northern Calif...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id keyword location  \\\n",
       "0         1     NaN      NaN   \n",
       "1         4     NaN      NaN   \n",
       "2         5     NaN      NaN   \n",
       "3         6     NaN      NaN   \n",
       "4         7     NaN      NaN   \n",
       "...     ...     ...      ...   \n",
       "7608  10869     NaN      NaN   \n",
       "7609  10870     NaN      NaN   \n",
       "7610  10871     NaN      NaN   \n",
       "7611  10872     NaN      NaN   \n",
       "7612  10873     NaN      NaN   \n",
       "\n",
       "                                                   text  target  \n",
       "0     Our Deeds are the Reason of this #earthquake M...       1  \n",
       "1                Forest fire near La Ronge Sask. Canada       1  \n",
       "2     All residents asked to 'shelter in place' are ...       1  \n",
       "3     13,000 people receive #wildfires evacuation or...       1  \n",
       "4     Just got sent this photo from Ruby #Alaska as ...       1  \n",
       "...                                                 ...     ...  \n",
       "7608  Two giant cranes holding a bridge collapse int...       1  \n",
       "7609  @aria_ahrary @TheTawniest The out of control w...       1  \n",
       "7610  M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...       1  \n",
       "7611  Police investigating after an e-bike collided ...       1  \n",
       "7612  The Latest: More Homes Razed by Northern Calif...       1  \n",
       "\n",
       "[7613 rows x 5 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6a04d930-6ab0-4444-b1eb-ee8df70915c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>Two giant cranes holding a bridge collapse int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>@aria_ahrary @TheTawniest The out of control w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>Police investigating after an e-bike collided ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>The Latest: More Homes Razed by Northern Calif...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0     Our Deeds are the Reason of this #earthquake M...\n",
       "1                Forest fire near La Ronge Sask. Canada\n",
       "2     All residents asked to 'shelter in place' are ...\n",
       "3     13,000 people receive #wildfires evacuation or...\n",
       "4     Just got sent this photo from Ruby #Alaska as ...\n",
       "...                                                 ...\n",
       "7608  Two giant cranes holding a bridge collapse int...\n",
       "7609  @aria_ahrary @TheTawniest The out of control w...\n",
       "7610  M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...\n",
       "7611  Police investigating after an e-bike collided ...\n",
       "7612  The Latest: More Homes Razed by Northern Calif...\n",
       "\n",
       "[7613 rows x 1 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[['text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "4c0ca671-08f7-4dbe-88a0-4cace1f78509",
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove newlines , spaces\n",
    "def remove_blank(data):\n",
    "    text=data.replace(\"\\\\n\",\" \").replace(\"\\t\",\" \")\n",
    "    return text\n",
    "\n",
    "# Contractions mapping\n",
    "def contract_text(data):\n",
    "    text=fix(data)\n",
    "    return text\n",
    "\n",
    "# handling accented character\n",
    "def handling_accented_chr(data):\n",
    "    text=unidecode(data)\n",
    "    return text\n",
    "\n",
    "# remove stopwords\n",
    "stopwords_list=stopwords.words('english')\n",
    "stopwords_list.remove('no')\n",
    "stopwords_list.remove('nor')\n",
    "stopwords_list.remove('not')\n",
    "\n",
    "# clean the text\n",
    "def clean_text(data):\n",
    "    # tokenization\n",
    "    tokens=word_tokenize(data)\n",
    "    # lowercase the data\n",
    "    #lowercase=[word.lower() for word in tokens]\n",
    "    # remove stopwords\n",
    "    #remove_stopwords=[word for word in lowercase if word not in stopwords_list]\n",
    "    # remove punctuations\n",
    "    #remove_punctuations=[word for word in remove_stopwords if word not in punctuation]\n",
    "    # remove len(word)<2\n",
    "    #remove_len_word=[word for word in remove_punctuations if len(word)>2]\n",
    "    # word contains only alphabet not number\n",
    "    #final_text=[word for word in remove_len_word if word.isalpha()]\n",
    "    \n",
    "    clean_data=[i.lower() for i in tokens if (i.lower() not in punctuation) and (i.lower() not in stopwords_list) and (len(i)>2) and (i.isalpha())]\n",
    "\n",
    "    \n",
    "    return clean_data\n",
    "\n",
    "def lemmatization(data):\n",
    "    final_text=[]\n",
    "    lemmatizing=WordNetLemmatizer()\n",
    "    for i in data:\n",
    "        lemma=lemmatizing.lemmatize(i)\n",
    "        final_text.append(lemma)\n",
    "        \n",
    "    return \" \".join(final_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf00ece8-42a7-46a3-a4d1-cd4f38a6e442",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a163f803-5b58-4cb7-9dba-f795f1248546",
   "metadata": {},
   "outputs": [],
   "source": [
    "## For Test Data\n",
    "df_train['text']=df_train['text'].apply(remove_blank)\n",
    "df_train['text']=df_train['text'].apply(contract_text)\n",
    "df_train['text']=df_train['text'].apply(handling_accented_chr)\n",
    "df_train['text']=df_train['text'].apply(clean_text)\n",
    "df_train['text']=df_train['text'].apply(lemmatization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ae2f8776-a357-4399-b740-5da5acfc9031",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>deed reason earthquake may allah forgive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>forest fire near ronge sask canada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>resident asked place notified officer evacuati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>people receive wildfire evacuation order calif...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>got sent photo ruby alaska smoke wildfire pour...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>two giant crane holding bridge collapse nearby...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>thetawniest control wild fire california even ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>utc volcano hawaii http</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>police investigating collided car little portu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>latest home razed northern california wildfire...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text\n",
       "0              deed reason earthquake may allah forgive\n",
       "1                    forest fire near ronge sask canada\n",
       "2     resident asked place notified officer evacuati...\n",
       "3     people receive wildfire evacuation order calif...\n",
       "4     got sent photo ruby alaska smoke wildfire pour...\n",
       "...                                                 ...\n",
       "7608  two giant crane holding bridge collapse nearby...\n",
       "7609  thetawniest control wild fire california even ...\n",
       "7610                            utc volcano hawaii http\n",
       "7611  police investigating collided car little portu...\n",
       "7612  latest home razed northern california wildfire...\n",
       "\n",
       "[7613 rows x 1 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[['text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b1a7b7d-9e9a-400d-927a-b1dddf1e591b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e09d424a-1f53-47a2-9ea0-1766b52e6c93",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                            \n",
      " ______  _____ _______ _______ _______ _______ _______  ______\n",
      " |     \\   |   |______ |_____| |______    |    |______ |_____/\n",
      " |_____/ __|__ ______| |     | ______|    |    |______ |    \\_\n",
      "                                                              \n",
      " _______ _  _  _ _______ _______ _______ _______\n",
      "    |    |  |  | |______ |______    |    |______\n",
      "    |    |__|__| |______ |______    |    ______|\n",
      "                                                \n",
      "\n",
      "____________________________________________________________________________\n",
      "                                                                            \n",
      "*********************  Actual Disaster Tweets Finder  **********************\n",
      "\n",
      "\n",
      "*******  1. Show general summary visualizations for job indicators     *****\n"
     ]
    }
   ],
   "source": [
    "import pyfiglet\n",
    "\n",
    "def entry_menu():\n",
    "    total_characters = 76\n",
    "    fonts = ['delta_corps_priest_1','cyberlarge','bubble','ansi_regular','ansi_shadow','dos_rebel','starwars','calvin_s']\n",
    "    project_name = \"Disaster Tweets\"\n",
    "    application_name = pyfiglet.figlet_format(project_name,font=fonts[1])\n",
    "\n",
    "    print(\"{:^{total_characters}}\".format('',total_characters=total_characters))\n",
    "    print(application_name)\n",
    "    print(\"{:_^{total_characters}}\".format('',total_characters=total_characters))\n",
    "    print(\"{:^{total_characters}}\".format('',total_characters=total_characters))\n",
    "\n",
    "    print(\"{:*^{total_characters}}\".format('  Actual Disaster Tweets Finder  ',total_characters=total_characters))\n",
    "    print(\"\\n\")\n",
    "    print(\"{:*^{total_characters}}\".format('**  1. Show general summary visualizations for job indicators     ',total_characters=total_characters))\n",
    "\n",
    "\n",
    "entry_menu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c7517fc1-3781-4874-a98f-daeb82638219",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "{:*^76}\n",
      "Do you want to continue (y or n) :: \n"
     ]
    }
   ],
   "source": [
    "print(('\\n{:*^76}\\nDo you want to continue (y or n) :: '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2118b302-c6a7-41a2-b04b-c0462aff2c4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select 10% for the validation (local testing) - select this from original training set\n",
    "\n",
    "# merge all csv \n",
    "\n",
    "# append all extracted and filtered to a\n",
    "\n",
    "# return \n",
    "\n",
    "# validate.py to return the F1 score for validation\n",
    "\n",
    "# complete pipeline\n",
    "\n",
    "\n",
    "new_model <- new training job using the entire dataset\n",
    "    run bash through python\n",
    "\n",
    "then at the begining of the key word search adn prediction at the next user input check for the performance of previously created new model\n",
    "\n",
    "\n",
    "then validating the new_model using the local validation dataset\n",
    "    lemos_validation.py ---> model, validation_split()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "df5b10e9-9655-4361-a644-48acc940b15d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['results/csv/arson_23_11_08_16_34_17.csv', 'results/csv/ablaze_23_11_08_16_34_45.csv', 'results/csv/casualty_23_11_08_18_31_28.csv', 'results/csv/fire_23_11_08_16_31_46.csv', 'results/csv/wildfire_23_11_08_16_33_03.csv', 'results/csv/bushfire_23_11_08_16_33_57.csv', 'results/csv/flood_23_11_08_16_33_36.csv']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import re\n",
    "import glob\n",
    "\n",
    "folder_path_for_csv = 'results/csv/' #ablaze_23_10_02_00_37_57.csv\n",
    "# full_path_\n",
    "csv_file_list = glob.glob(f\"{folder_path_for_csv}/*.csv\")\n",
    "\n",
    "print(csv_file_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a2acf465-8fc3-46cb-bb08-d97ecaa20f84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/oruma001/cs834_\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "7f737c18-46c7-4743-8799-fff468d93b01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "results/csv/arson_23_11_08_16_34_17.csv  -  44\n",
      "results/csv/ablaze_23_11_08_16_34_45.csv  -  17\n",
      "results/csv/fire_23_11_08_16_31_46.csv  -  524\n",
      "results/csv/wildfire_23_11_08_16_33_03.csv  -  42\n",
      "results/csv/bushfire_23_11_08_16_33_57.csv  -  119\n",
      "results/csv/flood_23_11_08_16_33_36.csv  -  128\n",
      "874\n",
      "676\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_726757/1309146042.py:9: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  df_all = pd.concat([df_all,df])\n"
     ]
    }
   ],
   "source": [
    "# cs834_/results/csv/selected/wildfire_23_10_02_00_38_32.csv\n",
    "\n",
    "df_all = pd.DataFrame(columns = ['id', 'keyword', 'location', 'text', 'target', 'prob_DIS_TRUE',\n",
    "       'prob_DIS_FALSE', 'prediction'])\n",
    "\n",
    "for csv_file in csv_file_list:\n",
    "    df = pd.read_csv(csv_file)\n",
    "\n",
    "    df_all = pd.concat([df_all,df])\n",
    "    # df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True)\n",
    "\n",
    "    # print(df.columns)\n",
    "    \n",
    "    # print(df.head(5))\n",
    "    print(csv_file,\" - \",len(df))\n",
    "\n",
    "\n",
    "df_all = df_all.sort_values(by=['id'])\n",
    "print(len(df_all))\n",
    "\n",
    "df_unique = df_all.drop_duplicates(subset=['id'])\n",
    "\n",
    "print(len(df_unique))\n",
    "\n",
    "df_all.to_csv(r'dataset/all_extracted_data_in_one.csv',index=False)\n",
    "df_unique.to_csv(r'dataset/all_unique_data_in_one.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2f6cbe64-9872-4645-9b17-d5b12544987e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Empty DataFrame\n",
      "Columns: [id, keyword, location, text, target, prob_DIS_TRUE, prob_DIS_FALSE, prediction]\n",
      "Index: []\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(columns = ['id', 'keyword', 'location', 'text', 'target', 'prob_DIS_TRUE',\n",
    "       'prob_DIS_FALSE', 'prediction'])\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b363b8fa-baa2-4a52-b1d8-3be3c71a9ddc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'keyword', 'location', 'text', 'target'], dtype='object')\n",
      "Index(['id', 'keyword', 'location', 'text', 'target'], dtype='object')\n",
      "initial_training_df :  6851\n",
      "unique_new_records :  676\n",
      "df_all_training :  7527\n"
     ]
    }
   ],
   "source": [
    "#merging new records with initial training dataset\n",
    "initial_training_df = pd.read_csv('dataset/train_split.csv')\n",
    "unique_new_records = pd.read_csv('dataset/all_unique_data_in_one.csv')\n",
    "\n",
    "# remove additional columns from unique_new_records df\n",
    "unique_new_records = unique_new_records.iloc[:,:5]\n",
    "\n",
    "# print(initial_training_df.columns)\n",
    "# print(unique_new_records.columns)\n",
    "\n",
    "df_all_training = pd.concat([initial_training_df,unique_new_records])\n",
    "\n",
    "# initial_training_df.head()\n",
    "print('initial_training_df : ',len(initial_training_df))\n",
    "print('unique_new_records : ',len(unique_new_records))\n",
    "print('df_all_training : ',len(df_all_training))\n",
    "\n",
    "df_all_training.to_csv(r'dataset/extended_training_dataset.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "71ead8c0-5e56-443b-ba40-3566acf537bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7613\n",
      "6851\n",
      "762\n"
     ]
    }
   ],
   "source": [
    "# spliting train.csv into train(0.9) and validation (0.1)\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "kaggle_training_df = pd.read_csv('dataset/train.csv')\n",
    "\n",
    "train, validation = train_test_split(kaggle_training_df, test_size=0.1)\n",
    "\n",
    "print(len(kaggle_training_df))\n",
    "print(len(train))\n",
    "print(len(validation))\n",
    "\n",
    "train = train.sort_values(by=['id'])\n",
    "validation = validation.sort_values(by=['id'])\n",
    "\n",
    "# type(train)\n",
    "train.to_csv(r'dataset/train_split.csv',index=False)\n",
    "validation.to_csv(r'dataset/validation_split.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9be93645-89bf-4ce4-b146-6b6a54d40e98",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "85bded63-111f-4861-a0a6-09a1a135bd9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_new_records_and_extend_training_dataset():\n",
    "    folder_path_for_csv = 'results/csv/' \n",
    "    csv_file_list = glob.glob(f\"{folder_path_for_csv}/*.csv\")\n",
    "\n",
    "    df_all = pd.DataFrame(columns = ['id', 'keyword', 'location', 'text', 'target', 'prob_DIS_TRUE',\n",
    "           'prob_DIS_FALSE', 'prediction'])\n",
    "    \n",
    "    for csv_file in csv_file_list:\n",
    "        df = pd.read_csv(csv_file)\n",
    "        df_all = pd.concat([df_all,df]) \n",
    "    \n",
    "    df_all = df_all.sort_values(by=['id'])   \n",
    "    df_unique = df_all.drop_duplicates(subset=['id'])\n",
    "\n",
    "    df_all.to_csv(r'dataset/all_extracted_data_in_one.csv',index=False)\n",
    "    df_unique.to_csv(r'dataset/all_unique_data_in_one.csv',index=False)    \n",
    "\n",
    "    # merging new records with initial training dataset\n",
    "    initial_training_df = pd.read_csv('dataset/train_split.csv')\n",
    "    \n",
    "    # remove additional columns from unique_new_records df\n",
    "    df_unique = df_unique.iloc[:,:5]   \n",
    "    df_all_training = pd.concat([initial_training_df, df_unique])\n",
    "    \n",
    "    # initial_training_df.head()\n",
    "    print('initial_training_df : ',len(initial_training_df))\n",
    "    print('unique_new_records : ',len(df_unique))\n",
    "    print('df_all_training : ',len(df_all_training))\n",
    "    \n",
    "    df_all_training.to_csv(r'dataset/extended_training_dataset.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "c29623b4-d513-499a-b8cd-7347ebe6e249",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial_training_df :  6851\n",
      "unique_new_records :  676\n",
      "df_all_training :  7527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_726757/1519100502.py:8: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  df_all = pd.concat([df_all,df])\n"
     ]
    }
   ],
   "source": [
    "merge_new_records_and_extend_training_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "cd52feaa-23a0-4e65-94eb-2b5c2f126602",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial_training_df :  6851\n",
      "unique_new_records :  696\n",
      "df_all_training :  7547\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_726757/2830795956.py:10: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  df_all = pd.concat([df_all,df])\n"
     ]
    }
   ],
   "source": [
    "merge_new_records_and_extend_training_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f31b6fa-3eab-44f1-a6b7-a6d5b1cf6501",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "5cbc684e-107a-4170-b85b-3fe428075a33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6851"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "initial_train_recors_count = len(pd.read_csv('dataset/train_split.csv'))\n",
    "initial_train_recors_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9fd85c70-1fdf-4758-b02e-eaebdceb3fad",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'glob' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# find the modt recent model\u001b[39;00m\n\u001b[1;32m      2\u001b[0m folder_path_for_keras \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msaved_models\u001b[39m\u001b[38;5;124m'\u001b[39m \n\u001b[0;32m----> 3\u001b[0m keras_models_list \u001b[38;5;241m=\u001b[39m \u001b[43mglob\u001b[49m\u001b[38;5;241m.\u001b[39mglob(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfolder_path_for_keras\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/*.keras\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      5\u001b[0m keras_models_list\n",
      "\u001b[0;31mNameError\u001b[0m: name 'glob' is not defined"
     ]
    }
   ],
   "source": [
    "# find the modt recent model\n",
    "folder_path_for_keras = 'saved_models' \n",
    "keras_models_list = glob.glob(f\"{folder_path_for_keras}/*.keras\")\n",
    "\n",
    "keras_models_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "a76d3455-9bf3-4e9a-bb92-8d57853be3c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'saved_models/lemos_DT_nlp_bert_049.keras'"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras_models_list.sort(reverse=True)\n",
    "\n",
    "keras_models_list[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "11718439-9f27-4b65-bbf4-ce89878ef02c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6851\n",
      "saved_models/lemos_DT_nlp_bert_049.keras\n",
      "050\n"
     ]
    }
   ],
   "source": [
    "import lemos_validate as lval\n",
    "\n",
    "print(lval.get_recent_model_path())\n",
    "print(lval.get_next_iteration())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "28bc85e6-4512-4132-a0b9-784571a3ae2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "\n",
    "curr_file_full_path = \"saved_models/lemos_DT_nlp_bert_CURR.txt\"\n",
    "best_model_path = \"saved_models/test.txt\"\n",
    "# best_model_path = \"saved_models/lemos_834_001.txt\"\n",
    "\n",
    "dest = shutil.copy(best_model_path, curr_file_full_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f4ead54-172b-4a9c-9f17-71f10d4eb453",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a1669b4e-b9fc-4f1c-818e-949c51925873",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6851\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from subprocess import call\n",
    "call([\"python\", \"lemos_validate.py\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "14a47ac5-bcb0-4d2e-8b5c-8327ba3824e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "050\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'050'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import glob\n",
    "\n",
    "def get_next_iteration():\n",
    "    folder_path_for_keras = 'saved_models' \n",
    "    keras_models_list = glob.glob(f\"{folder_path_for_keras}/*.keras\")\n",
    "    keras_models_list.sort(reverse=True)\n",
    "\n",
    "    path = keras_models_list[1]\n",
    "    val = int(path[path.index(\"bert_\")+5:path.index(\".keras\")])+1\n",
    "\n",
    "    print('{:0>3}'.format(val))\n",
    "    iteration = '{:0>3}'.format(val)\n",
    "\n",
    "    return iteration\n",
    "\n",
    "get_next_iteration()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cbc8735-e6c7-4a13-8fd5-5baafcc1ceb5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6611e70f-c483-4a54-b860-4fea279a139c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7536"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "initial_train_records_count = len(pd.read_csv(r'dataset/train_split.csv')) # 6851\n",
    "\n",
    "int(1.1*(initial_train_records_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "48b4a91f-8dc2-4714-b5fe-898da144c81f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted batch job 2558754\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from subprocess import call\n",
    "\n",
    "itr = lval.get_next_iteration()\n",
    "epochs = '4'\n",
    "\n",
    "dynamic_vals = f\"--export=ALL,iteration={itr},epochs={epochs}\"\n",
    "out_filename = f\"out_files_from_slurm/lemos_834_{itr}.txt\"\n",
    "\n",
    "# call([\"sbatch\", dynamic_vals, \"-o\", out_filename, \"lemos_job_test.sh\"])\n",
    "call([\"sbatch\", dynamic_vals, \"-o\", out_filename, \"training_job_submission.sh\"])\n",
    "\n",
    "# sbatch --export=ALL,iteration='044',epochs=4 -o lemos_834_044.txt lemos_job_test.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3ad4a08e-ceb7-4665-a12a-4b1bc7f0fdee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', './saved_models/lemos_DT_nlp_bert_054.keras', 'records_used_for_training:7547', '0.7874659400544959']\n",
      "['', '23_12_02_17_55_54', './saved_models/lemos_DT_nlp_bert_056.keras', 'records_used_for_training:7547', '0.7896879240162822']\n"
     ]
    }
   ],
   "source": [
    "model_perfom_file = 'results/performances/model_performances.txt'\n",
    "with open(model_perfom_file) as f:\n",
    "    content = f.read()\n",
    "    \n",
    "    # print((content.split(\"\\n\\n\")))\n",
    "\n",
    "    _2nd_last_item = content.split(\"\\n\\n\")[-3]\n",
    "    last_item = content.split(\"\\n\\n\")[-2]\n",
    "\n",
    "    print(_2nd_last_item.split(\"\\n\"))\n",
    "    print(last_item.split(\"\\n\"))\n",
    "    \n",
    "\n",
    "    # [print(a) for a in content.split(\"\\n\\n\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7e1fa44-be00-41bb-ad42-8daae499d392",
   "metadata": {},
   "outputs": [],
   "source": [
    "23_12_02_17_55_54\n",
    "./saved_models/lemos_DT_nlp_bert_056.keras\n",
    "records_used_for_training:7547\n",
    "0.7896879240162822"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "55e330a3-14d5-4531-9077-585dc7e9dab5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', '23_12_02_17_55_54', './saved_models/lemos_DT_nlp_bert_056.keras', 'records_used_for_training:7547', '0.7896879240162822']\n",
      "['', '23_12_02_18_13_59', './saved_models/lemos_DT_nlp_bert_057.keras', 'records_used_for_training:7547', '0.8005698005698005']\n"
     ]
    }
   ],
   "source": [
    "model_perfom_file = 'results/performances/model_performances.txt'\n",
    "with open(model_perfom_file) as f:\n",
    "    content = f.read()\n",
    "    \n",
    "    # print((content.split(\"\\n\\n\")))\n",
    "\n",
    "    _2nd_last_item = content.split(\"\\n\\n\")[-3]\n",
    "    last_item = content.split(\"\\n\\n\")[-2]\n",
    "\n",
    "    print(_2nd_last_item.split(\"\\n\"))\n",
    "    print(last_item.split(\"\\n\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96b502ad-a2e1-486a-8e62-e8ce636104e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def get_last_items_from_model_perfromance():\n",
    "#     model_perform_file = 'results/performances/model_performances.txt'\n",
    "    \n",
    "#     with open(model_perform_file) as f:\n",
    "#         content = f.read()\n",
    "        \n",
    "#         # print((content.split(\"\\n\\n\")))\n",
    "    \n",
    "#         _2nd_last_item = content.split(\"\\n\\n\")[-3]\n",
    "#         last_item = content.split(\"\\n\\n\")[-2]\n",
    "    \n",
    "#         # print(_2nd_last_item.split(\"\\n\"))\n",
    "#         # print(last_item.split(\"\\n\"))\n",
    "\n",
    "#     return _2nd_last_item.split(\"\\n\"),last_item.split(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fe7b625d-6170-4ba1-9265-7194e44d1a40",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_last_item_from_model_perfromance():\n",
    "    model_perform_file = 'results/performances/model_performances.txt'\n",
    "    \n",
    "    with open(model_perform_file) as f:\n",
    "        content = f.read()\n",
    "        \n",
    "        # print((content.split(\"\\n\\n\")))\n",
    "    \n",
    "        last_item = content.split(\"\\n\\n\")[-2].split(\"\\n\")\n",
    "        print(last_item)\n",
    "        model_path = last_item[2]\n",
    "        records_used_for_training = int(last_item[3].split(\":\")[1])\n",
    "        f1_score_local_validation = float(last_item[4])\n",
    "    \n",
    "        # print(_2nd_last_item.split(\"\\n\"))\n",
    "        # print(last_item.split(\"\\n\"))\n",
    "\n",
    "    return model_path,records_used_for_training,f1_score_local_validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "cad879d1-1b2b-4de9-8f3d-9038b8c11bf0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', '23_12_02_18_13_59', './saved_models/lemos_DT_nlp_bert_057.keras', 'records_used_for_training:7547', '0.8005698005698005']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('./saved_models/lemos_DT_nlp_bert_057.keras', 7547, 0.8005698005698005)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_last_item_from_model_perfromance()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b1644280-b5cb-4f9b-a3d5-05bd7909ce91",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_current_loaded_model_details():\n",
    "    model_perform_file = 'results/performances/current_model_details.txt'\n",
    "    \n",
    "    with open(model_perform_file) as f:\n",
    "        content = f.read()\n",
    "        \n",
    "        content_list = content.split(\"\\n\")\n",
    "\n",
    "    model_path = content_list[1]\n",
    "    records_used_for_training = int(content_list[2].split(\":\")[1])\n",
    "    f1_score_local_validation = float(content_list[3])\n",
    "    \n",
    "        # last_item = content.split(\"\\n\\n\")[-2]\n",
    "    \n",
    "    return model_path,records_used_for_training,f1_score_local_validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7da809e3-bc4a-42fe-a6d6-ccabe8d07c57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./saved_models/lemos_DT_nlp_bert_CURR.keras', 6587, 0.7874659400544959)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_current_loaded_model_details()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4d833b82-7d63-4394-916c-14ac259a2ee3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./saved_models/lemos_DT_nlp_bert_057.keras', 7547, 0.8005698005698005)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_current_loaded_model_details()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "404f794e-a7fe-4b2e-9fb1-78a7deef7046",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_last_item_details_to_current_model_details_txt():\n",
    "    model_perform_file = 'results/performances/model_performances.txt'\n",
    "    \n",
    "    with open(model_perform_file) as f:\n",
    "        content = f.read()\n",
    "        \n",
    "        current_model_details_file = 'results/performances/current_model_details.txt'\n",
    "        last_item = content.split(\"\\n\\n\")[-2].replace(\"\\n\",\"\",1)\n",
    "\n",
    "        with open(current_model_details_file,'w') as f1:\n",
    "            f1.write(last_item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5b454ba9-155b-429c-bce1-4cb3091c83d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "write_last_item_details_to_current_model_details_txt()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4415b2f2-b618-42cd-a7e8-267191373789",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "506dc3d6-fada-41cd-bb4a-804a01f29521",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d953415-1053-49db-b61b-64f96e27e3fa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5ada608b-c3ea-46fe-ab67-d507bd4bc790",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_f1_for_last_2_items():\n",
    "    \n",
    "    item_list_2nd_last,item_list_last = get_last_2_items_from_model_perfromance()\n",
    "\n",
    "    f1_last = float(item_list_last[-1])\n",
    "    f1_2nd_last = float(item_list_2nd_last[-1])\n",
    "        \n",
    "    return f1_2nd_last,f1_last"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9f26809e-97d7-4bad-a43a-515d24fa5740",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.7896879240162822, 0.8005698005698005)\n"
     ]
    }
   ],
   "source": [
    "print(get_f1_for_last_2_items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f97dc19d-0e46-4ab6-b7a1-badd8c277d63",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_training_records_count_for_training_job():\n",
    "    \n",
    "    _,item_list_last = get_last_2_items_from_model_perfromance()\n",
    "\n",
    "    training_records_count = int(item_list_last[-2].split(':')[-1])\n",
    "        \n",
    "    return training_records_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d4dd9737-6578-4f16-89a2-1769a3115106",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7547\n"
     ]
    }
   ],
   "source": [
    "print(get_training_records_count_for_training_job())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e422358a-6ce0-4ced-b88b-6477b3100713",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "folder_path_for_csv = 'results/csv/' \n",
    "csv_file_list = glob.glob(f\"{folder_path_for_csv}/*.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b955932-2837-476a-8f35-ab57cbca6c33",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}