m2f3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import re\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import operator\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from collections import Counter\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    \n",
    "    #Read pre-processed train_paras.txt file into codes\n",
    "    \n",
    "    with open('train_paras.txt','r') as f:\n",
    "        codes=f.readlines()\n",
    "    \n",
    "    for i in range(0,len(codes)):\n",
    "        codes[i]=codes[i][2:].split(\"', '\")\n",
    "        codes[i][-1]=codes[i][-1].replace(\"']\\n\",'')\n",
    "    \n",
    "    #Read tags.pkl as tags\n",
    "    with open(\"tags.pkl\", \"rb\") as fp:\n",
    "        tags = pickle.load(fp)\n",
    "        \n",
    "    #for i in range(0,len(tags)):\n",
    "        #print(tags[i])\n",
    "        #tags[i]=tags[i][0]\n",
    "        \n",
    "    for i in range(0,len(codes)):\n",
    "        codes[i]=\" \".join(word for word in codes[i])\n",
    "    \n",
    "    #Call code_classification\n",
    "    code_classification(codes,tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('MyData.csv')\n",
    "\n",
    "def all_label(raw_data):\n",
    "    l = len(raw_data)\n",
    "    label_list = [] #np.empty((raw_data.shape[0],len(tagCol)))\n",
    "    ll =[]\n",
    "    tagCol = [col for col in raw_data.columns if 'Tag_' in col]\n",
    "    #print(tagCol)\n",
    "    for j in range(0,raw_data.shape[0]):\n",
    "        for i in range(0,len(tagCol)):\n",
    "            label_list.append(tagCol[i])\n",
    "        #print(label_list)\n",
    "    return label_list\n",
    "\n",
    "label_entire = all_label(df)\n",
    "label_entire = list(dict.fromkeys(label_entire)) \n",
    "#label_entire\n",
    "\n",
    "x = []\n",
    "for i in range(0,len(label_entire)):\n",
    "    x.append(df[label_entire[i]].sum())\n",
    "\n",
    "SelectMax= pd.DataFrame()\n",
    "SelectMax['Tag']=label_entire\n",
    "SelectMax['Count']=x\n",
    "\n",
    "SelectMax = SelectMax.sort_values(by='Count', ascending=False)\n",
    "#print(SelectMax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(SelectMax.loc[1:9,:])\n",
    "def code_classification(codes,tags):\n",
    "    \n",
    "    # Chose top tags to train into two different model. Made box plots to look for outliers\n",
    "    \n",
    "    #Top 9 most frequent tags\n",
    "    new_tags1=SelectMax.iloc[1:9,:]\n",
    "    new_tags1 = new_tags1['Tag'].tolist()\n",
    "    #print(new_tags1)\n",
    "    #print(type(new_tags1))\n",
    "    \n",
    "    #print(type(tags))\n",
    "    \n",
    "    #Next 11 most frequent tags\n",
    "    new_tags2=SelectMax.iloc[10:20,:]\n",
    "    #new_tags2 = np.asarray(new_tags2)\n",
    "    new_tags2 = new_tags2['Tag'].tolist()\n",
    "    \n",
    "    #Indices of specified frequent tags in new_tags1, new_tags2\n",
    "    indexes_1=[]\n",
    "    indexes_2=[]\n",
    "    for i in range(len(tags)):\n",
    "        for j in range(len(tags[i])):\n",
    "            #print(tags[i][j])\n",
    "            if tags[i][j] in new_tags1:\n",
    "                indexes_1.append(i)\n",
    "            elif tags[i][j] in new_tags2:\n",
    "                indexes_2.append(i)\n",
    "\n",
    "    #Separating frequent tags to perform classification to maintain class balance\n",
    "    new_codes1=operator.itemgetter(*indexes_1)(codes)\n",
    "    tags1=operator.itemgetter(*indexes_1)(tags)\n",
    "    \n",
    "    #Model building for top 9 frequent tags\n",
    "    vectorizer,model=model_building(new_codes1,tags1)\n",
    "    \n",
    "    #Saving the vectorizer and model\n",
    "    with open('vectorizer_1.pkl', 'wb') as f:\n",
    "        pickle.dump(vectorizer, f)\n",
    "    with open('model_codes_1.pkl', 'wb') as f:\n",
    "        pickle.dump(model, f)\n",
    "    \n",
    "    #Separating next most frequent tags to perform classification to maintain class balance\n",
    "    new_codes2=operator.itemgetter(*indexes_2)(codes)\n",
    "    tags2=operator.itemgetter(*indexes_2)(tags)\n",
    "    \n",
    "    #Building model for next 11 frequent tags\n",
    "    vectorizer,model=model_building(new_codes2,tags2)\n",
    "    \n",
    "    #Saving model and vectorizer\n",
    "    with open('vectorizer_2.pkl', 'wb') as f:\n",
    "        pickle.dump(vectorizer, f)\n",
    "    with open('model_codes_2.pkl', 'wb') as f:\n",
    "        pickle.dump(model, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_building(new_codes,tags):\n",
    "    \n",
    "    # Use TF-IDF vectorizer and extract features to give it as an inout to the LogisticRegression Model\n",
    "    \n",
    "    \n",
    "    #Building TF-IDF\n",
    "    vectorizer=TfidfVectorizer(analyzer='word',stop_words='english')\n",
    "    transformed=vectorizer.fit_transform(new_codes)\n",
    "    \n",
    "    #Building Logistic regression to classify codes\n",
    "    model=LogisticRegression()\n",
    "    model.fit(transformed,tags)\n",
    "    \n",
    "    #Return model and vectorizer\n",
    "    return vectorizer, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "0",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-4be4dd58a851>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0m__name__\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;34m'__main__'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mmain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-2-b9af0bbfe54d>\u001b[0m in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m     \u001b[1;31m#Call code_classification\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m     \u001b[0mcode_classification\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodes\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtags\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-12-ff120d99b39a>\u001b[0m in \u001b[0;36mcode_classification\u001b[1;34m(codes, tags)\u001b[0m\n\u001b[0;32m     23\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtags\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m             \u001b[1;31m#print(tags[i][j])\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m             \u001b[1;32mif\u001b[0m \u001b[0mtags\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mnew_tags1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m                 \u001b[0mindexes_1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m             \u001b[1;32melif\u001b[0m \u001b[0mtags\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mnew_tags2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Public\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m    765\u001b[0m         \u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_apply_if_callable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    766\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 767\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    768\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    769\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Public\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_value\u001b[1;34m(self, series, key)\u001b[0m\n\u001b[0;32m   3116\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3117\u001b[0m             return self._engine.get_value(s, k,\n\u001b[1;32m-> 3118\u001b[1;33m                                           tz=getattr(series.dtype, 'tz', None))\n\u001b[0m\u001b[0;32m   3119\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3120\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minferred_type\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'integer'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'boolean'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 0"
     ]
    }
   ],
   "source": [
    "if __name__=='__main__':\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}