quora-nlp-text-classification-v1.ipynb

{"cells":[{"metadata":{"_uuid":"50045bee3b3c26a3992d79124b3e619bb1aa3c6e"},"cell_type":"markdown","source":"**Objective: ** \nPredicting if a question asked on Quora is sincere or not."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport string\nimport re, string, unicodedata\nimport os\nimport time\nstart_time = time.time()\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f03838b2f233023388fa5ea66aea21e9042b1044"},"cell_type":"code","source":"# Data Import\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\n# Shape for train and test\nprint(\"--- %s seconds for Data Loading ---\" % (time.time() - start_time))\nprint('Shape of train:',train.shape)\nprint('Shape of test:',test.shape)\ntrain['Dataset'] = 'train'\ntest['Dataset'] = 'test'\n\nall_data =  pd.concat([train, test], axis= 0, ignore_index= True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ea47e0f3b827213ec5d15faa5548159555df0de"},"cell_type":"markdown","source":"**Pre-Processing**\nUsing the library NLTK, we are going to start with text pre-processing.  It is predominantly comprised of three steps:\n* Noise Removal\n* Lexicon Normalization\n* Object Standardization\nTo go into more detail we have taken the following steps to clean and process the text:\n* Create Tokens\n* Remove Non-Ascii characters, convert to lower case, remove punctuation and whitespace. Replace numbers with their words equivalent.\n* Removing Stop words: NLTK corpus contains 179 stop words such as \"for\", \"having\", \"yours\" and so on.\n* Stem / Lemmenize words"},{"metadata":{"trusted":true,"_uuid":"01e904711ecb92c971acd396d216ff82e9327fb6"},"cell_type":"code","source":"import nltk\n#!pip install inflect #Make sure the Kernal has Internet Connected (Check Settings)\n#import inflect\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import sent_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem.wordnet import WordNetLemmatizer ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a44cba2a736b52c5a3950b245b955a2be0b71684"},"cell_type":"code","source":"eng_stopwords = set(stopwords.words(\"english\"))\n## Number of words in the text ##\nall_data[\"num_words\"] = all_data[\"question_text\"].apply(lambda x: len(str(x).split()))\n\n## Number of unique words in the text ##\nall_data[\"num_unique_words\"] = all_data[\"question_text\"].apply(lambda x: len(set(str(x).split())))\n\n## Number of characters in the text ##\nall_data[\"num_chars\"] = all_data[\"question_text\"].apply(lambda x: len(str(x)))\n\n## Number of stopwords in the text ##\nall_data[\"num_stopwords\"] = all_data[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\n\n## Number of punctuations in the text ##\nall_data[\"num_punctuations\"] =all_data['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\n\n## Number of title case words in the text ##\nall_data[\"num_words_upper\"] = all_data[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n\n## Number of title case words in the text ##\nall_data[\"num_words_title\"] = all_data[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n\n## Average length of the words in the text ##\nall_data[\"mean_word_len\"] = all_data[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b8e94189dfb79afe3d0c69724faea24ed0087ef"},"cell_type":"code","source":"#lower case\nall_data['question_text'] = all_data['question_text'].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n#Removing Punctuation\nall_data['question_text'] = all_data['question_text'].str.replace('[^\\w\\s]','')\n#Removing numbers\nall_data['question_text'] = all_data['question_text'].str.replace('[0-9]','')\n#Remooving stop words and words with length <=2\nfrom nltk.corpus import stopwords\nstop = stopwords.words('english')\nall_data['question_text'] = all_data['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop and len(x)>2))\n# Lemmatize\nfrom nltk.stem import WordNetLemmatizer\nwl = WordNetLemmatizer()\nall_data['question_text'] = all_data['question_text'].apply(lambda x: \" \".join(wl.lemmatize(x,'v') for x in x.split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6a5c5b113de0cedef485730460483e38c05451e"},"cell_type":"code","source":"print(\"--- %s seconds for Data Transformation ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d76c742030fcd618f85d558550a4df0145c2583"},"cell_type":"code","source":"from tqdm import tqdm, tqdm_notebook\n# NLTK sentiment cell\nprint('\\nGetting sentiments...')\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer as SIA\nsia = SIA()\nsentiments = np.zeros(len(all_data))\n\nfor i, (_, row) in tqdm_notebook(enumerate(all_data.iterrows()), total=len(all_data)):\n    sentiments[i] = sia.polarity_scores(row.question_text)['compound']\n# Bringing Sentiment scores to [0,1] range\nall_data['sentiment'] = pd.Series(sentiments)\nall_data['sentiment_target'] = (all_data['sentiment'] + 1) / 2\n\nprint(\"--- %s seconds for Adding Sentiment ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"affbe268c7934ec35387e3610f7c44611c049725"},"cell_type":"code","source":"print(train.shape)\ntrain = all_data[all_data.Dataset == 'train']\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1caec4a016ed1f3d8edca7c68ab4fc2da436b815"},"cell_type":"markdown","source":"LDA"},{"metadata":{"trusted":true,"_uuid":"323dc7b06aedce59a1f82c78460a4025745f25b2"},"cell_type":"code","source":"import gensim\nfrom gensim import corpora\ndct = corpora.Dictionary(  nltk.word_tokenize(i) for i in  train.question_text )\ndct.filter_extremes(no_below=20, no_above=0.5)\n\ntopic_cnt = 100\n\n# Reindexes the remaining words after filtering\ndct.compactify()\nprint(\"Left with {} words.\".format(len(dct.values())))\n\nall_data[\"question_text_tokens\"] = list(map(nltk.word_tokenize, all_data.question_text))\n\n#Make a BOW for every document\ndef document_to_bow(df):\n    df['bow'] = list(map(lambda doc: dct.doc2bow(doc), df.question_text_tokens))\n    \ndocument_to_bow(all_data)\n\ntrain = all_data[all_data.Dataset == 'train']\n\nprint(\"Created BOW\")\n# model imports\nfrom gensim.models.ldamulticore import LdaMulticore\ncorpus = train.bow\nnum_topics = topic_cnt\n#A multicore approach to decrease training time\nLDAmodel = LdaMulticore(corpus=corpus,\n                        id2word=dct,\n                        num_topics=num_topics,\n                        workers=4,\n                        chunksize=4000,\n                        passes=7,\n                        alpha='asymmetric')\nprint(\"--- %s seconds for LDA ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cfb8409587b2c96aab1864a4617744edd98ff3f"},"cell_type":"code","source":"def document_to_lda_features(lda_model, document):\n    \"\"\" Transforms a bag of words document to features.\n    It returns the proportion of how much each topic was\n    present in the document.\n    \"\"\"\n    topic_importances = LDAmodel.get_document_topics(document, minimum_probability=0)\n    topic_importances = np.array(topic_importances)\n    return topic_importances[:,1]\n\nall_data['lda_features'] = list(map(lambda doc:\n                                      document_to_lda_features(LDAmodel, doc),\n                                      all_data.bow))\n\n\nprint(\"--- %s seconds for LDA to Features ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57727539a3d2c3dcef05865823fdccc03b6c2d14"},"cell_type":"code","source":"all_data[\"lda_features\"][all_data[\"lda_features\"].isnull()] = all_data[\"lda_features\"].apply( lambda d: d if isinstance(d, list) else [0] *topic_cnt  ) \nall_data = pd.concat([all_data,pd.DataFrame(np.array(list(map(np.array, all_data.lda_features))))], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe0695bdfa4a72625b3513e56dee5ad4d7d3216a"},"cell_type":"code","source":"print(train.shape)\ntrain = all_data[all_data.Dataset == 'train']\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09de55bba5e4544e6419a6d1fb39f45583cfea7a"},"cell_type":"code","source":"Insincere_topic_distribution = train.loc[train.target == 1, 'lda_features'].mean()\nSincere_topic_distribution = train.loc[train.target == 0, 'lda_features'].mean()\n\ndef get_topic_top_words(lda_model, topic_id, nr_top_words=20):\n    \"\"\" Returns the top words for topic_id from lda_model.\n    \"\"\"\n    id_tuples = lda_model.get_topic_terms(topic_id, topn=nr_top_words)\n    word_ids = np.array(id_tuples)[:,0]\n    words = map(lambda id_: lda_model.id2word[id_], word_ids)\n    return words\n\ntop_topics = []\nfor target, distribution in zip([1,0], [Insincere_topic_distribution, Sincere_topic_distribution]):\n    for x in sorted(np.argsort(distribution)[-5:]):\n        top_topics.append(x)\n        \n# Get the top 15 words from each topic\ntopic_words = []\nfor i in top_topics:\n    terms = LDAmodel.get_topic_terms(i,15)\n    topic_words.append([dct[pair[0]] for pair in terms])\n\n# Create an empty data frame\ntopic_cols = []\nfor i in top_topics:\n    topic_cols.append(\"Topic_%d_count\" % (i))\n\ntopic_count = pd.DataFrame( index = range(len(all_data.index)) ,columns = topic_cols)\n\nfor i in range( topic_count.shape[1]):\n    topic_count[topic_count.columns[i]] = all_data['question_text_tokens'].apply(lambda x: len(  list(set( (x)  ) & set( topic_words[i]    ) )     )   )\n\nprint(\"--- %s seconds for get the number of words in question i from topic j as a feature ---\" % (time.time() - start_time))    \n# Get the number of words in question i from topic j as a feature\n\nall_data = pd.concat([ all_data, topic_count], axis=1)\ndel topic_count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f097888c8852b74be55d280a2ac4c4eb8f8b648"},"cell_type":"code","source":"print(\"Modeling Section\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9dcbb744714687b754237ebfaec9e7fe31f05ddb"},"cell_type":"code","source":"del LDAmodel","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7855e726572336e2ce8ddabbcaa5179f7deb36dd"},"cell_type":"markdown","source":"**Modeling Section:**\n"},{"metadata":{"trusted":true,"_uuid":"bb153509f6b31e2de84347534659635f6b63d703"},"cell_type":"code","source":"print(test.shape)\nprint(train.shape)\ntest = all_data[all_data.Dataset == 'test']\nprint(test.shape)\ntrain = all_data[all_data.Dataset == 'train']\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13e2aac3199b812848dac91a6d67155b5c8eec71"},"cell_type":"code","source":"col_rm_list = ['qid','question_text','bow','Dataset','sentiment','target','question_text_tokens','lda_features']\ntopic_features = [col for col in all_data.filter(regex='Topic').columns ]\ntopic_features = list(set(topic_features))\nother_features = ['num_words', 'num_unique_words', 'num_chars', \n                'num_stopwords', 'num_punctuations', 'num_words_upper', \n                'num_words_title', 'mean_word_len','sentiment_target']\neng_features = [col for col in train.columns if col not in (col_rm_list + topic_features)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5f417c1f67c119301ea1e77f5ce4b99213faf93"},"cell_type":"code","source":"from sklearn.metrics import f1_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4df14a7f9fa0eb56c90e34a351a4de08d2941e66"},"cell_type":"markdown","source":"Model With Basic Features + Sentiment"},{"metadata":{"trusted":true,"_uuid":"d674ebecd1c1d8f9d14ad124b9b3c48e24810c9a","collapsed":true},"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=43)\ntest_pred_ots = 0\noof_pred_ots = np.zeros([train.shape[0],])\n\ntrain_target = train['target'].values\n\nx_test = test[other_features].values\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = train.loc[train_index][other_features].values, train.loc[val_index][other_features].values\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    classifier = LogisticRegression(C= 0.1)\n    classifier.fit(x_train, y_train)\n    val_preds = classifier.predict_proba(x_val)[:,1]\n    preds = classifier.predict_proba(x_test)[:,1]\n    test_pred_ots += 0.2*preds\n    oof_pred_ots[val_index] = val_preds\nprint(\"--- %s seconds for Model Other Features ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78512102a266468bb0c9be84443ff63941e73611"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt_ots = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_ots.astype(float) >thresh).astype(int))\n    #print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt_ots = thresh\nprint(thresh_opt_ots)\npred_train_ots = (oof_pred_ots > thresh_opt_ots).astype(np.int)\nf1_score(train_target, pred_train_ots)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66763086126021000ea7a4b9a9a59b3f3099e58d"},"cell_type":"markdown","source":"Model with Topic Counts"},{"metadata":{"trusted":true,"_uuid":"f4bf3165ea1d298b7601a028a636ae5c5b92681a","collapsed":true},"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=43)\ntest_pred_tps = 0\noof_pred_tps = np.zeros([train.shape[0],])\n\ntrain_target = train['target'].values\n\nx_test = test[topic_features].values\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = train.loc[train_index][topic_features].values, train.loc[val_index][topic_features].values\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    classifier = LogisticRegression(C= 0.1)\n    classifier.fit(x_train, y_train)\n    val_preds = classifier.predict_proba(x_val)[:,1]\n    preds = classifier.predict_proba(x_test)[:,1]\n    test_pred_tps += 0.2*preds\n    oof_pred_tps[val_index] = val_preds\nprint(\"--- %s seconds for Model Topic Count Features ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"046c004e6dd85cdc18a9d687c23e3be9c31d71cf"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt_tps = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_tps.astype(float) >thresh).astype(int))\n    #print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt_tps = thresh\nprint(thresh_opt_tps)\npred_train_tps = (oof_pred_tps > thresh_opt_tps).astype(np.int)\nf1_score(train_target, pred_train_tps)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3df06a6c00abbae64ad2a0a46a379ff6d06f792"},"cell_type":"markdown","source":"Model For LDA"},{"metadata":{"trusted":true,"_uuid":"f150d714878150ef0fae5fd327d9540d814013b1"},"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=43)\ntest_pred_lda = 0\noof_pred_lda = np.zeros([train.shape[0],])\n\ntrain_target = train['target'].values\n\nx_test = test[eng_features].values\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = train.loc[train_index][eng_features].values, train.loc[val_index][eng_features].values\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    classifier = LogisticRegression(C= 0.1)\n    classifier.fit(x_train, y_train)\n    val_preds = classifier.predict_proba(x_val)[:,1]\n    preds = classifier.predict_proba(x_test)[:,1]\n    test_pred_lda += 0.2*preds\n    oof_pred_lda[val_index] = val_preds\nprint(\"--- %s seconds for Model LDA ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ffe47adafd92907596df5bbc8c3514e1ada3081"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt_lda = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_lda.astype(float) >thresh).astype(int))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt_lda = thresh\nprint(thresh_opt_lda)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d7ba8793ba496c9631d124334de05257dad6e9b"},"cell_type":"code","source":"pred_train_lda = (oof_pred_lda > thresh_opt_lda).astype(np.int)\nf1_score(train_target, pred_train_lda)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06f9f2d1498bb36fc1679fb3f2a3bfac091be3ee"},"cell_type":"code","source":"## Clean UP\nall_data = all_data[col_rm_list]\nall_data = all_data.drop(['sentiment','lda_features'], axis = 1)\nprint(test.shape)\nprint(train.shape)\ntest = all_data[all_data.Dataset == 'test']\nprint(test.shape)\ntrain = all_data[all_data.Dataset == 'train']\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3448549fd00284a9ab36904a45ebbd2793c1dab"},"cell_type":"markdown","source":"Model TFIDF 1"},{"metadata":{"trusted":true,"_uuid":"0a1e28167efe42157bdbcdefa3eb2179ea78f505"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nword_vectorizer = TfidfVectorizer(\n    sublinear_tf=True,\n    strip_accents='unicode',\n    analyzer='word',\n    token_pattern=r'\\w{1,}',\n    stop_words='english',\n    ngram_range=(1, 3),\n    max_features=10000)\n\nword_vectorizer.fit(all_data.question_text)\ndel all_data\ntrain_word_features = word_vectorizer.transform(train.question_text)\ntest_word_features = word_vectorizer.transform(test.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2f03498ee601b42fda900780025bfbb1e025009"},"cell_type":"code","source":"\ntrain_target = train['target'].values\n\nkf = KFold(n_splits=5, shuffle=True, random_state=187)\ntest_pred_tf = 0\noof_pred_tf = np.zeros([train.shape[0],])\n\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = train_word_features[train_index,:], train_word_features[val_index,:]\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    classifier = LogisticRegression(class_weight = \"balanced\", C=0.5, solver='sag')\n    classifier.fit(x_train, y_train)\n    val_preds = classifier.predict_proba(x_val)[:,1]\n    preds = classifier.predict_proba(test_word_features)[:,1]\n    test_pred_tf += 0.2*preds\n    oof_pred_tf[val_index] = val_preds\nprint(\"--- %s seconds for Model TFIDF 1 ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7777e27502353729f9eea180acbac70611f51670"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_tf.astype(float) >thresh).astype(int))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt = thresh\nprint(thresh_opt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"caafad0be7b6cd3cb91f57c83c24184e3cf24d52"},"cell_type":"code","source":"pred_train = (oof_pred_tf > thresh_opt).astype(np.int)\nf1_score(train_target, pred_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"24d4e13a5ffff164eb23b061577f75cc940c439e"},"cell_type":"markdown","source":"Model TFIDF 2"},{"metadata":{"trusted":true,"_uuid":"40d125d8411df088e1033b8628789a32cbbc9e95"},"cell_type":"code","source":"import lightgbm as lgb\n\ndef lgb_f1_score(y_hat, data):\n    y_true = data.get_label()\n    y_hat = np.round(y_hat) # scikits f1 doesn't like probabilities\n    return 'f1', f1_score(y_true, y_hat), True\n\nparams = {'learning_rate': 0.05,\n          'application': 'binary',\n          'max_depth': 9,\n          'num_leaves': 100,\n          'verbosity': -1,\n          'metric': 'lgb_f1_score',\n          'data_random_seed': 3,\n          'bagging_fraction': 0.8,\n          'feature_fraction': 0.4,\n          #'nthread': 16,\n          'lambda_l1': 1,\n          'lambda_l2': 1,\n          'num_rounds': 2700,\n          'verbose_eval': 100}\n\n\nkf = KFold(n_splits=5, shuffle=True, random_state=420)\ntest_pred_lgb = 0\noof_pred_lgb = np.zeros([train.shape[0],])\n\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = train_word_features[train_index,:], train_word_features[val_index,:]\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    \n    d_train = lgb.Dataset(x_train, label=y_train)\n    d_valid = lgb.Dataset(x_val, label=y_val)\n\n    num_rounds = 2500\n    model = lgb.train(params,\n                  train_set=d_train,\n                  num_boost_round=num_rounds,\n                  valid_sets=[d_train, d_valid],\n                  valid_names=['train', 'val'],\n                  verbose_eval=0)\n    \n    val_preds = model.predict(x_val)\n    preds = classifier.predict(test_word_features)\n    test_pred_lgb += 0.2*preds\n    oof_pred_lgb[val_index] = val_preds\nprint(\"--- %s seconds for Model 2 ---\" % (time.time() - start_time))    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae727872eeb7b9dfcf77dd4a668ee58e0eef0c38"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt_lgb = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_lgb.astype(float) >thresh).astype(int))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt_lgb = thresh\nprint(thresh_opt_lgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"547705eeff9b1c2a7042f911a37c9a584aa033b2"},"cell_type":"code","source":"pred_train_lgb = (oof_pred_lgb > thresh_opt_lgb).astype(np.int)\nf1_score(train_target, pred_train_lgb)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"569d27b0765bd7c6a73af7b9f247a49d7e6f5b9b"},"cell_type":"markdown","source":"Model Count Vectorizer"},{"metadata":{"trusted":true,"_uuid":"f0548255e509d4cb441d4cab1e9b6a443a55eef2"},"cell_type":"code","source":"# Count Vectorizor\nfrom sklearn.feature_extraction.text import CountVectorizer \n\nbow = CountVectorizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40c7231a49892b4993494f5c6e80e21f5481adca"},"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=43)\ntest_pred_cv = 0\noof_pred_cv = np.zeros([train.shape[0],])\n\n\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = train.loc[train_index]['question_text'].values, train.loc[val_index]['question_text'].values\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    x_test = test['question_text'].values\n    \n    bow = CountVectorizer()\n    x_train = bow.fit_transform(x_train)\n    x_val = bow.transform(x_val)\n    x_test = bow.transform(x_test)\n\n    classifier = LogisticRegression(penalty = \"l1\", C = 1.25, class_weight = \"balanced\")\n    \n    classifier.fit(x_train, y_train)\n    val_preds = classifier.predict_proba(x_val)[:,1]\n    preds = classifier.predict_proba(x_test)[:,1]\n    test_pred_cv += 0.2*preds\n    oof_pred_cv[val_index] = val_preds\nprint(\"--- %s seconds for Model 3 ---\" % (time.time() - start_time))    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e69a52008ebcc1c23456379584fd4a9eaefd4c0"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt_cv = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_cv.astype(float) >thresh).astype(int))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt_cv = thresh\nprint(thresh_opt_cv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c28fa8ca1f49df416760e68e4314121d75752c9c"},"cell_type":"code","source":"pred_train_cv = (oof_pred_cv > thresh_opt_cv).astype(np.int)\nf1_score(train_target, pred_train_cv)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9de551ab31f012252783eb98c1551a7da1798c40"},"cell_type":"markdown","source":"NB Count Vectorizer"},{"metadata":{"trusted":true,"_uuid":"4663745f88ae7ae91d4b83f55541e41de0a29acc"},"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\nfrom sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e19a0f04c2efffcc88bc7da65bb7f5519ab1dda"},"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=43)\ntest_pred_cv_2 = 0\noof_pred_cv_2 = np.zeros([train.shape[0],])\ntest_pred_cv_3 = 0\noof_pred_cv_3 = np.zeros([train.shape[0],])\n\n\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = train.loc[train_index]['question_text'].values, train.loc[val_index]['question_text'].values\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    x_test = test['question_text'].values\n    \n    bow = CountVectorizer()\n    x_train = bow.fit_transform(x_train)\n    x_val = bow.transform(x_val)\n    x_test = bow.transform(x_test)\n    \n    classifier2 = MultinomialNB()\n    classifier3 = BernoulliNB()\n    \n    classifier2.fit(x_train, y_train)\n    val_preds = classifier2.predict_proba(x_val)[:,1]\n    preds = classifier2.predict_proba(x_test)[:,1]\n    test_pred_cv_2 += 0.2*preds\n    oof_pred_cv_2[val_index] = val_preds\n    \n    classifier3.fit(x_train, y_train)\n    val_preds = classifier3.predict_proba(x_val)[:,1]\n    preds = classifier3.predict_proba(x_test)[:,1]\n    test_pred_cv_3 += 0.2*preds\n    oof_pred_cv_3[val_index] = val_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7042d80732b1e06d8415d23318aa47770a2f3cf"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt_cv_2 = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_cv_2.astype(float) >thresh).astype(int))\n    #print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt_cv_2 = thresh\nprint(thresh_opt_cv_2)\npred_train_cv_2 = (oof_pred_cv_2 > thresh_opt_cv_2).astype(np.int)\nf1_score(train_target, pred_train_cv_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0751cf79b4d8af261132fefb6efdc8d01a34386"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt_cv_3 = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_cv_3.astype(float) >thresh).astype(int))\n    #print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt_cv_3 = thresh\nprint(thresh_opt_cv_3)\npred_train_cv_2 = (oof_pred_cv_3 > thresh_opt_cv_3).astype(np.int)\nf1_score(train_target, pred_train_cv_2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08c27784aaaab4bab7bdd81633e1be5271a13f56"},"cell_type":"markdown","source":"Stacking:"},{"metadata":{"trusted":true,"_uuid":"218252d75ba68cf11f39791e649dfdea5b7b9a7e"},"cell_type":"code","source":"stack_train = np.hstack((oof_pred_ots.reshape(-1,1),oof_pred_tps.reshape(-1,1),oof_pred_lda.reshape(-1,1),oof_pred_tf.reshape(-1,1), oof_pred_lgb.reshape(-1,1), \n                         oof_pred_cv.reshape(-1,1),oof_pred_cv_2.reshape(-1,1),oof_pred_cv_3.reshape(-1,1)))\nstack_test = np.hstack((test_pred_ots.reshape(-1,1),test_pred_tps.reshape(-1,1),test_pred_lda.reshape(-1,1),test_pred_tf.reshape(-1,1), test_pred_lgb.reshape(-1,1), \n                         test_pred_cv.reshape(-1,1),test_pred_cv_2.reshape(-1,1),test_pred_cv_3.reshape(-1,1)))\n\nprint(stack_train.shape)\nprint(stack_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f98ecf2b65e59fbf8288c2da909999ec0ee28df"},"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=43)\ntest_pred_stack = 0\noof_pred_stack = np.zeros([train.shape[0],])\n\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = stack_train[train_index,:], stack_train[val_index,:]\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    classifier = LogisticRegression(class_weight = \"balanced\", C=0.5, solver='sag')\n    classifier.fit(x_train, y_train)\n    val_preds = classifier.predict_proba(x_val)[:,1]\n    preds = classifier.predict_proba(stack_test)[:,1]\n    test_pred_stack += 0.2*preds\n    oof_pred_stack[val_index] = val_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bec71de0ac8703218633d3126f1d883860d25bef"},"cell_type":"code","source":"from sklearn import metrics\nthresh_opt_stack = 0.5\nf1_opt = 0\nfor thresh in np.arange(0.1, 0.91, 0.05):\n    thresh = np.round(thresh, 2)\n    f1 = metrics.f1_score(train_target, (oof_pred_stack.astype(float) >thresh).astype(int))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    if f1_opt < f1:\n        f1_opt = f1\n        thresh_opt_stack = thresh\nprint(thresh_opt_stack)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e39b33753838d71f3796095eecaa1250b837efd"},"cell_type":"code","source":"pred_train_stack = (oof_pred_stack > thresh_opt_stack).astype(np.int)\nf1_score(train_target, pred_train_stack)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d640b03f5b2b19100165a23c0032ea2808fe170"},"cell_type":"code","source":"# Confusion Matrix\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn import metrics\nimport matplotlib.pyplot as plt\nimport scikitplot as skplt\nskplt.metrics.plot_confusion_matrix(train_target, pred_train_stack,labels = [1,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"deabb8278bb8f755f0bb748592756cb66c20d257"},"cell_type":"code","source":"pred_test_final = ( test_pred_stack> thresh_opt_stack).astype(np.int)\nsubmission = pd.DataFrame.from_dict({'qid': test['qid']})\nsubmission['prediction'] = pred_test_final\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a9fca6107bbf24b76a41846f65a6e0c81920a0c"},"cell_type":"code","source":"print(np.sum(submission.prediction)/(stack_test.shape[0]))\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}