Supervises_h2o.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    h2o.remove_all()\n",
    "    h2o.shutdown()\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.util import ngrams\n",
    "from collections import Counter\n",
    "from itertools import chain\n",
    "import h2o\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from h2o.estimators.kmeans import H2OKMeansEstimator\n",
    "from h2o.estimators.word2vec import H2OWord2vecEstimator\n",
    "pd.options.display.max_columns= None\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import nltk\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import word_tokenize\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import class_weight\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "np.set_printoptions(threshold=sys.maxsize)\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from collections import Counter\n",
    "from h2o.automl import H2OAutoML\n",
    "import shap\n",
    "import vaderSentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install pandas_profiling --user --index-url=http://pypi.appdev.proj.coe.ic.gov/simple/ --trusted-host=pypi.appdev.proj.coe.ic.gov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "h2o.init()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "source": [
    "## All Frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "#join to get themes\n",
    "categories=pd.read_excel(\"Theme_Key.xlsx\")\n",
    "data=pd.read_excel(\"All Comments.xls\",sheets='All Comments')\n",
    "data.reset_index()\n",
    "to_w=data.melt(id_vars=['All Comments','RecNum'], value_vars=['Theme 1','Theme 2','Theme 3','Theme 4','Theme 5','Theme 6','Theme 7','Theme 8','Theme 9','Theme 10','Theme 11','Theme 12'])\n",
    "to_w=to_w.merge(categories,on='value',how='inner')\n",
    "to_w=to_w.drop(['variable','RecNum','value'],axis=1)\n",
    "dat=to_w\n",
    "data=dat\n",
    "h2o_data=h2o.H2OFrame(data)\n",
    "h2o_data=h2o_data.ascharacter()\n",
    "h2o_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gl=h2o.import_file('glove.6B.300d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "#Sentiment\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "analyzer = SentimentIntensityAnalyzer()\n",
    "def sentiment_analyzer_scores(sentence):\n",
    "    score=analyzer.polarity_scores(sentence)\n",
    "    return(score)\n",
    "Dat=data\n",
    "J=Dat.dropna()['All Comments'].apply(str).apply(sentiment_analyzer_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "#dynamiclly generate subsets\n",
    "frames=[]\n",
    "k=0\n",
    "for i in dat['Theme'].unique():\n",
    "    temp=dat\n",
    "    temp2=temp[temp['Theme']==i]\n",
    "    temp2['Bool']='True'\n",
    "    temp3=temp[~temp.isin(temp2)].dropna()\n",
    "    temp3['Bool']='False'\n",
    "    temp2.reset_index(drop=True,inplace=True)\n",
    "    temp3.reset_index(drop=True,inplace=True)\n",
    "    temp=pd.concat([temp2,temp3],axis=0)\n",
    "\n",
    "    frame_name=i.strip().replace(\" \",'').replace(\"/\",'').replace(\",\",'').replace(\"-\",'')+'_FRAME'+'=temp'\n",
    "    try:\n",
    "        exec(frame_name)\n",
    "    except:\n",
    "        print(frame_name)\n",
    "    frames.append(frame_name[:-5])\n",
    "    k=k+1\n",
    "    #if(k>0):\n",
    "    #    break\n",
    "\n",
    "frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pretrained with glove\n",
    "\n",
    "STOP_WORDS = [\"i\",\"you\",\"us\",\"s\",\"t\",\"m\",\"subject\",\"can\",\"lines\",\"re\",\"what\",\n",
    "               \"there\",\"all\",\"we\",\"one\",\"the\",\"a\",\"an\",\"of\",\"or\",\"in\",\"for\",\"by\",\"on\",\n",
    "               \"but\",\"is\",\"in\",\"a\",\"not\",\"with\",\"as\",\"was\",\"if\",\"they\",\"are\",\"this\",\"and\",\"it\",\"have\",\n",
    "               \"from\",\"at\",\"my\",\"be\",\"by\",\"not\",\"that\",\"to\",\"from\",\"com\",\"org\",\n",
    "              \"like\",\"likes\",\"so\",\"(U)\",\"(U//FOUO)\",\"apos\",\"quot\",\"has\",\"been\",\"do\",\"had\",\"was\",\"were\",\n",
    "             \"they\",\"their\",\"them\",\"am\",\"will\",\"than\",\"when\",\"who\",\"where\",\"our\",\"me\",\"your\",\"would\",'don','get','more',\n",
    "             'nga','nro','nsa','new','adf','here','no','see','day','get','why','how','ve','dont','its','ive','work','people',\n",
    "              'out','which','should','ADF-C','leadership','ADFC']\n",
    "\n",
    "\n",
    "#This converts sentenses into vectors\n",
    "def tokenize(sentences, stop_word = STOP_WORDS):\n",
    "    tokenized = sentences.tokenize(\"\\\\W+\")\n",
    "    tokenized_lower = tokenized.tolower()\n",
    "    tokenized_filtered = tokenized_lower[(tokenized_lower.nchar() >= 2) | (tokenized_lower.isna()),:]\n",
    "    tokenized_words = tokenized_filtered[tokenized_filtered.grep(\"[0-9]\",invert=True,output_logical=True),:]\n",
    "    tokenized_words = tokenized_words[(tokenized_words.isna()) | (~ tokenized_words.isin(STOP_WORDS)),:]\n",
    "    return tokenized_words\n",
    "words=tokenize(h2o_data)\n",
    "\n",
    "w2v_model = H2OWord2vecEstimator(pre_trained=gl)\n",
    "w2v_model.train(training_frame=words)\n",
    "survey= w2v_model.transform(words, aggregate_method = \"AVERAGE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['All Comments']\n",
    "survey"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%capture output\n",
    "#dynamically do all frames\n",
    "\n",
    "\n",
    "h2o_all=h2o.H2OFrame(data).ascharacter()\n",
    "#h2o_new=h2o.H2OFrame(data['All Comments'].ascharacter()) #this will be new data\n",
    "words_all=tokenize(h2o_all)\n",
    "#words_all=tokenize(h2o_new)\n",
    "\n",
    "\n",
    "\n",
    "test_results=[]\n",
    "survey_all= w2v_model.transform(words_all, aggregate_method = \"AVERAGE\")\n",
    "\n",
    "\n",
    "k=0\n",
    "for frame in frames:\n",
    "    try:\n",
    "        exec('h2o_data=h2o.H2OFrame('+frame+').ascharacter()')\n",
    "        words=tokenize(h2o_data)\n",
    "        survey= w2v_model.transform(words, aggregate_method = \"AVERAGE\")\n",
    "        master=h2o_data.concat(survey)\n",
    "        x=master.columns[2:]\n",
    "        y='Bool'\n",
    "        master['Bool']=master['Bool'].asfactor()\n",
    "        master_train, test=master.split_frame(ratios=[.90])\n",
    "        aml_frame='aml_'+frame\n",
    "        exec(aml_frame+'=H2OAutoML(max_models=3,seed=1, balance_classes=True,nfolds=5,include_algos=[\\'DeepLearning\\',\\'XGBoost\\',\\'StackedEnsemble\\'])')\n",
    "        exec(aml_frame+'.train(x=x,y=y, training_frame=master_train)')\n",
    "\n",
    "        exec(aml_frame+\"_model\"+\"=\"+aml_frame+\".leader\")\n",
    "\n",
    "        \n",
    "        exec('print('+aml_frame+'.leader.accuracy)')\n",
    "        \n",
    "        exec('pred_test='+aml_frame+'.leader.predict(test)')\n",
    "        test_results.append(np.mean((pred_test['predict']==test['Bool']).as_data_frame()))\n",
    "        exec('preds='+aml_frame+'.leader.predict(survey_all)') #Change to new data\n",
    "        \n",
    "        preds=preds.drop(1)\n",
    "        col_names=[frame[:-5]+'_pred',frame[:-5]+'_true_prob']\n",
    "        preds.set_names(col_names)\n",
    "        h2o_all=preds.cbind( h2o_all)\n",
    "    except:\n",
    "        pass\n",
    "    k=k+1\n",
    "    #if(k>0):\n",
    "    #    break\n",
    "\n",
    "#h2o_all\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"Models have average error-\"+str(1-np.mean(test_results)))\n",
    "print(\"With error bound-\"+str(2*(np.var([1-x for x in test_results]))**.5/len(test_results)**.5))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.distplot(test_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RedFlagcomments_FRAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(H2OAutoML)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cell_out=str(output)\n",
    "text_file=open(\"models.txt\",\"w\")\n",
    "text_file.write(cell_out)\n",
    "text_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "h2o.export_file(h2o_all,\"debug.csv\", force=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "h2o.export_file(h2o_all,\"all_preds.csv\",force=True)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "change=pd.read_csv(\"all_preds.csv\")\n",
    "names=[col for col in change.columns if 'true' not in col]\n",
    "d1=change[names]\n",
    "d2=change['All Comments']\n",
    "#d1.reset_index()\n",
    "#d2.reset_index()\n",
    "#d=pd.concat([d2,d1],axis=1)\n",
    "d1=d1.drop(['Theme'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "import pandas_profiling as pf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "profile=pf.ProfileReport(d1)\n",
    "profile.to_file(output_file='Report.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {},
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "d1"
   ]
  }
 ],
 "metadata": {
  "extensions": {
   "jupyter_dashboards": {
    "activeView": "grid_default",
    "version": 1,
    "views": {
     "grid_default": {
      "cellMargin": 10,
      "defaultCellHeight": 20,
      "maxColumns": 12,
      "name": "grid",
      "type": "grid"
     },
     "report_default": {
      "name": "report",
      "type": "report"
     }
    }
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}