WordEmbeddings_randomStuff.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing & playing with Word Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf8 -*-\n",
    "import gensim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tweets Embeddings (https://github.com/bakrianoo/aravec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the model\n",
    "model = gensim.models.Word2Vec.load('C:\\\\Users\\\\oae15\\\\Downloads\\\\Compressed\\\\Twt-SG\\\\Twt-SG')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "204448\n",
      "<class 'dict'>\n"
     ]
    }
   ],
   "source": [
    "# model length\n",
    "print(len(model.wv.vocab))\n",
    "\n",
    "#print(len(model.vocab))\n",
    "\n",
    "print(type(model.wv.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(300,)\n",
      "[-0.10839421  0.09138662 -0.35472968  0.39736253 -0.09697727  0.22064325\n",
      " -0.21969014 -0.14091577  0.08716107  0.21912374  0.42986417 -0.00326329\n",
      " -0.12883724  0.09176949 -0.20214131 -0.12370671 -0.13858204  0.0138018\n",
      " -0.11687897  0.2257821  -0.21549632 -0.2341325   0.12628528 -0.23617908\n",
      "  0.20452431  0.11834913 -0.24664535 -0.01505378  0.06017434 -0.05228602\n",
      " -0.13811375  0.01898501  0.13786076  0.09435657  0.08092331 -0.09469444\n",
      "  0.4267219  -0.30736586 -0.0498416   0.1247642   0.03759022  0.50961906\n",
      " -0.37867928 -0.02934707  0.08601025  0.17956714  0.17601164  0.47291353\n",
      " -0.22509554  0.22936483  0.26485905  0.06402382 -0.00897211  0.31328171\n",
      " -0.00905758 -0.04255282 -0.27249449  0.1157776   0.25323051 -0.23520882\n",
      " -0.23988013 -0.25885782 -0.1309604   0.1842871  -0.26517582  0.0581301\n",
      " -0.05798087  0.38387588  0.11803581  0.25754991  0.30073762  0.28911319\n",
      "  0.01451335  0.17272291  0.27696607  0.20779043 -0.11682101 -0.1930891\n",
      " -0.33148226  0.15588762  0.3880358  -0.00920448  0.17637585  0.09538542\n",
      "  0.04176452  0.10688386  0.23868728  0.02344269 -0.04644856  0.14604217\n",
      " -0.18104486  0.01717399 -0.29405865  0.08286926 -0.06056172 -0.46287796\n",
      "  0.10845517  0.12828642  0.36128676 -0.06015698  0.1092208  -0.03571925\n",
      "  0.0339113  -0.37432882  0.05988164  0.37458631 -0.29328203  0.12264429\n",
      "  0.18462332 -0.00197115 -0.30573487 -0.15375809  0.04877048 -0.17120758\n",
      " -0.13579568  0.18552513  0.26828355  0.49017656 -0.10699464 -0.19186155\n",
      " -0.32036552 -0.0437063  -0.34810194  0.04784707  0.23995574  0.09946968\n",
      "  0.27851498 -0.15353477 -0.15936126 -0.0230229   0.09572729 -0.39223328\n",
      " -0.10107268 -0.15752174  0.21352844  0.02403676  0.03400167 -0.18713427\n",
      " -0.2662248   0.14494875 -0.1753971   0.26048863  0.16093715  0.13561141\n",
      "  0.20660797  0.47485387 -0.08444978  0.27518559 -0.00464113 -0.4890579\n",
      "  0.11087438 -0.47915688 -0.15964049  0.28312558  0.13989441 -0.26467717\n",
      " -0.21900447 -0.3321991  -0.3754091   0.09144904  0.04217119 -0.08405124\n",
      "  0.05047801  0.10945081 -0.27738753  0.0039044  -0.10519777 -0.12031917\n",
      " -0.14424059  0.10658421 -0.26347056 -0.13666935  0.09522433 -0.00072094\n",
      " -0.08024392 -0.07682858 -0.30260208  0.13311905  0.27505645 -0.24274676\n",
      " -0.11612188  0.11566649  0.20595703  0.1769949  -0.46296269  0.17729147\n",
      "  0.11710378 -0.12083489  0.36699528 -0.12523025 -0.13603729  0.09890535\n",
      " -0.29977381 -0.13898747  0.16285129  0.1954446  -0.14303343  0.0090642\n",
      " -0.01857325  0.30938378  0.09385683 -0.29291236 -0.15118068 -0.01785085\n",
      " -0.14031081 -0.38183179 -0.21872355 -0.20565191  0.21982922 -0.28339845\n",
      " -0.46907172 -0.01585826 -0.10276913 -0.14230193 -0.27321586 -0.05177723\n",
      "  0.39385027  0.20155331  0.01865192 -0.0757165   0.0952103   0.25170314\n",
      "  0.33444151  0.29666167  0.35091558 -0.14270553 -0.19215275  0.51476544\n",
      "  0.04485594 -0.26802537 -0.00852204 -0.33822855 -0.02592558 -0.47308385\n",
      "  0.17497638 -0.37511218 -0.30827358  0.30800766  0.08613658  0.04617155\n",
      " -0.12745482  0.02523875 -0.30258003  0.36919069  0.04262444  0.12869924\n",
      " -0.05327822  0.18410857 -0.19001545 -0.25873825  0.58023953 -0.09887242\n",
      " -0.48909959 -0.02619072 -0.08560052 -0.0329175   0.0627302   0.0500334\n",
      " -0.46294871 -0.19386292 -0.31841829  0.26936191 -0.04193562 -0.17566024\n",
      "  0.07069106  0.46760777 -0.06867281 -0.12424562  0.33826068 -0.59815502\n",
      "  0.13077089  0.16952778  0.05888782  0.02731603  0.18783665  0.13189737\n",
      "  0.28601518  0.43351227 -0.21542346 -0.1202192   0.1569055   0.00725203\n",
      "  0.09914091  0.22776116 -0.3626906  -0.02590812  0.02144776 -0.33768526\n",
      " -0.30850896  0.04968406 -0.11019913 -0.09924591 -0.06454578  0.29973486\n",
      "  0.2459036   0.49684262  0.12456509  0.08802407 -0.13642617 -0.1841177 ]\n"
     ]
    }
   ],
   "source": [
    "# get a word vector\n",
    "word_vector = model.wv['جامعه']\n",
    "print(word_vector.shape)\n",
    "print(word_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "اشوي 0.7315714359283447\n",
      "شويا 0.6618170738220215\n",
      "شويات 0.6333547830581665\n",
      "اشويه 0.6227505207061768\n",
      "ششوي 0.6178196668624878\n",
      "شويه 0.608310341835022\n",
      "شؤي 0.594508171081543\n",
      "وشوي 0.5906755924224854\n",
      "شويتين 0.5700851678848267\n",
      "حبتين 0.5528500080108643\n"
     ]
    }
   ],
   "source": [
    "# find and print the most similar terms to a word\n",
    "most_similar = model.wv.most_similar('شوي')\n",
    "for term, score in most_similar:\n",
    "    print(term, score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['الانستقرام', 'مقدمتهم', 'اموال', 'الصليبيخات', 'نقبله']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('عاوز' in model.wv.vocab)\n",
    "list(model.wv.vocab.keys())[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FaceBook Embeddings (wikipedia)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.wrappers  import FastText\n",
    "\n",
    "FB_trained_model = gensim.models.KeyedVectors.load_word2vec_format('C:\\\\Users\\\\oae15\\\\Downloads\\\\Compressed\\\\wiki.ar.vec', binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(FB_trained_model.wv['بقا'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "بعا 0.6326510310173035\n",
      "قا 0.6192600131034851\n",
      "سبقا 0.5837240815162659\n",
      "زقا 0.5790398120880127\n",
      "حا 0.5701289176940918\n",
      "نائيا 0.5657782554626465\n",
      "بشعا 0.5649769306182861\n",
      "بطيا 0.556555986404419\n",
      "بقال 0.5545694828033447\n",
      "ببرا 0.5521543025970459\n"
     ]
    }
   ],
   "source": [
    "# find and print the most similar terms to a word\n",
    "most_similar = FB_trained_model.wv.most_similar('بقا')\n",
    "for term, score in most_similar:\n",
    "    print(term, score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### News Embeddings (https://github.com/iamaziz/ar-embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.keyedvectors import KeyedVectors\n",
    "# load the model\n",
    "word_vectors = KeyedVectors.load_word2vec_format('C:\\\\Users\\\\oae15\\\\Downloads\\\\Compressed\\\\arabic-news\\\\arabic-news.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(159175, 300)\n"
     ]
    }
   ],
   "source": [
    "print(word_vectors.syn0.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ميته 0.37885844707489014\n",
      "متبعين 0.3571765124797821\n",
      "ليظهره 0.33383163809776306\n",
      "لأسلم 0.32984691858291626\n",
      "بباق 0.32429784536361694\n",
      "وآلامها 0.3199229836463928\n",
      "يضئ 0.31953853368759155\n",
      "المعزى 0.3181782364845276\n",
      "المعزي 0.31528738141059875\n",
      "آباءهم 0.3103591203689575\n"
     ]
    }
   ],
   "source": [
    "# find and print the most similar terms to a word\n",
    "most_similar = word_vectors.wv.most_similar('كره')\n",
    "for term, score in most_similar:\n",
    "    print(term, score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('resources/ArSEL_arabic_lemmas.csv', header=0)\n",
    "ar_lemmas = df['Ar_Lemma'].tolist()\n",
    "scores = df['HAPPY'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "no_emb = []\n",
    "for lemma in ar_lemmas:\n",
    "    if lemma in model.wv.vocab: count += 1\n",
    "    else: no_emb.append(lemma)\n",
    "print(count)\n",
    "print(len(ar_lemmas) - count)\n",
    "print(no_emb[:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = [], []\n",
    "for (score, lemma) in zip(scores, ar_lemmas):\n",
    "    if lemma in model.wv.vocab:\n",
    "        X.append(model.wv[lemma])\n",
    "        y.append(score)\n",
    "#print(len(X_train[0]), len(y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Neural Net with embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           index                                              tweet  \\\n",
      "0             ID                                              Tweet   \n",
      "1  2018-Ar-01452  تاني ايام العيد #حفله\\n#تامر_حسني اللي عايز يي...   \n",
      "2  2018-Ar-04326  المهم مبروك الدرع مش عايزين نخرج عن احتفالية ا...   \n",
      "3  2018-Ar-00273                  صوتك أمان يحضن قلبي قبل مسمعي 😴 ❤   \n",
      "4  2018-Ar-00918  للصبح كلمة فرح تفتح جسور الأمل و الحب كلمة شوق...   \n",
      "\n",
      "            emotion            score  \n",
      "0  Affect Dimension  Intensity Score  \n",
      "1               joy             NONE  \n",
      "2               joy             NONE  \n",
      "3               joy             NONE  \n",
      "4               joy             NONE  \n",
      "5473\n",
      "5467\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from resources.arabic_preprocessing import Arabic_preprocessing\n",
    "\n",
    "def count_embeddings(words_embeddings, tweets_list):\n",
    "    count = 0\n",
    "    for tweet in tweets_list:\n",
    "        for word in tweet.split():\n",
    "            if word in words_embeddings.wv:\n",
    "                count += 1\n",
    "    return count\n",
    "\n",
    "# load the embeddings models\n",
    "tweeter_embeddings = gensim.models.Word2Vec.load('resources/Twt-SG') # tweeter\n",
    "prep = Arabic_preprocessing()\n",
    "# prepare train data\n",
    "sample=pd.read_csv('data/updated/2018-EI-reg-Ar-joy-test.txt', delimiter='\\t', header=None, names=['index', 'tweet','emotion','score'])\n",
    "print(sample.head())\n",
    "sample['tweet_preprocessed_no_stemming'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False, replace_emojis=True, normalize_arabic=True))\n",
    "sample['tweet_preprocessed_stemming'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=True, replace_emojis=True, normalize_arabic=True))\n",
    "tweets = sample['tweet_preprocessed_no_stemming'].tolist()\n",
    "print(count_embeddings(tweeter_embeddings,tweets))\n",
    "tweets = sample['tweet_preprocessed_stemming'].tolist()\n",
    "print(count_embeddings(tweeter_embeddings,tweets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(800, 300)\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "from gensim.models.keyedvectors import KeyedVectors\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from resources.arabic_preprocessing import Arabic_preprocessing\n",
    "\n",
    "seed = 222\n",
    "np.random.seed(seed)\n",
    "\n",
    "def is_emoji(token):\n",
    "    emojis_unicode = r'([\\U0001F600-\\U0001F64F\\U00002000-\\U00003000]+)'\n",
    "    match = re.match(emojis_unicode, token)\n",
    "    return True if match else False\n",
    "\n",
    "def featurize(words_embeddings, sentence):\n",
    "    \"\"\"average words' vectors\"\"\"\n",
    "    feature_vec = np.zeros((300,), dtype=\"float32\")\n",
    "    words = sentence.split(' ')\n",
    "    retrieved_words = 0\n",
    "    for token in words:\n",
    "        try:\n",
    "            if include_emoji_embeddings and is_emoji(token): feature_vec = np.add(feature_vec, emoji_embeddings.wv[token])\n",
    "            else: feature_vec = np.add(feature_vec, words_embeddings.wv[token])\n",
    "            retrieved_words += 1\n",
    "        except KeyError:\n",
    "            pass  # if a word is not in the embeddings' vocabulary discard it\n",
    "    np.seterr(divide='ignore', invalid='ignore')\n",
    "    feature_vec = np.divide(feature_vec, retrieved_words)\n",
    "    return feature_vec\n",
    "\n",
    "def read_date_from_file(emotion, train_or_test, preprocess):\n",
    "    sample=pd.read_csv('data/updated/2018-EI-reg-Ar-' + emotion + '-' + train_or_test + '.txt', delimiter='\\t', header=None, names=['index', 'tweet','emotion','score'])\n",
    "    sample['tweet'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False, replace_emojis=True, normalize_arabic=True))\n",
    "    X = sample['tweet'].values\n",
    "    y = sample['score'].values\n",
    "    return X, y\n",
    "\n",
    "def get_train_test_preprocessed_and_normalized_data(emotion, preprocess, word_embeddings):\n",
    "    X1, y1 = read_date_from_file('joy', 'train', preprocess)\n",
    "    training_samples = X1.shape[0]\n",
    "    X2, y2 = read_date_from_file('joy', 'dev', preprocess)\n",
    "#     test_samples = X2.shape[0]\n",
    "    X = np.array([featurize(word_embeddings, tweet) for tweet in np.append(X1,X2)])\n",
    "    y = np.append(y1, y2)\n",
    "#     df = pd.DataFrame(X)\n",
    "#     df['score'] = y\n",
    "#     df = df.dropna()\n",
    "#     X = df.iloc[:,:300].values\n",
    "#     y = df['score'].values\n",
    "    X_train, X_test = X[:training_samples,:], X[training_samples:,:]\n",
    "    y_train, y_test = y[:training_samples], y[training_samples:]\n",
    "    return X_train, y_train, X_test, y_test\n",
    "\n",
    "# load the embeddings models\n",
    "tweeter_embeddings = gensim.models.Word2Vec.load('C:\\\\Users\\\\oae15\\\\Downloads\\\\Compressed\\\\Twt-SG\\\\Twt-SG') # tweeter\n",
    "#emoji_embeddings = KeyedVectors.load_word2vec_format('C:\\\\Users\\\\oae15\\\\Downloads\\\\Compressed\\\\emoji2vec.txt', binary=False) # emojis\n",
    "prep = Arabic_preprocessing()\n",
    "# prepare train & test data\n",
    "emotion = 'joy'\n",
    "X_train, y_train, X_test, y_test = get_train_test_preprocessed_and_normalized_data(emotion, prep, tweeter_embeddings)\n",
    "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)\n",
    "\n",
    "\n",
    "\n",
    "# sample=pd.read_csv('data/joy_train.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "# sample['tweet'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False, replace_emojis=False, normalize_arabic=True))\n",
    "# tweets = sample['tweet'].tolist()\n",
    "# # prepare test data\n",
    "# sample_test=pd.read_csv('data/joy_test.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "# sample_test['tweet'] = sample_test['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False, replace_emojis=False, normalize_arabic=True))\n",
    "# tweets_test = sample_test['tweet'].tolist()\n",
    "\n",
    "# all_tweets = tweets + tweets_test\n",
    "\n",
    "# X = np.zeros((800,300))\n",
    "# for i,tweet in enumerate(all_tweets):\n",
    "#     X[i,:] = featurize(tweeter_embeddings, emoji_embeddings, tweet, False)\n",
    "# print(X.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 2]\n",
      " [3 4]]\n"
     ]
    }
   ],
   "source": [
    "print(np.array([np.array([1,2]), np.array([3,4])]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 799 entries, 0 to 799\n",
      "Columns: 301 entries, 0 to score\n",
      "dtypes: float64(301)\n",
      "memory usage: 1.8 MB\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(X)\n",
    "y = sample['score'].tolist() + sample_test['score'].tolist()\n",
    "df['score'] = y\n",
    "df = df.dropna()\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.iloc[:,:300].values\n",
    "y = df['score'].values\n",
    "X_train, X_test = X[:600,:], X[600:,:]\n",
    "y_train, y_test = y[:600], y[600:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normalizing data (MaxMin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "X_new = scaler.fit_transform(X)\n",
    "X_train, X_test = X_new[:600,:], X_new[600:,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare training & test date (updated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data:  (728, 300) (728,)\n",
      "test data:  (224, 300) (224,)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import gensim\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from resources.arabic_preprocessing import Arabic_preprocessing\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "seed = 222\n",
    "np.random.seed(seed)\n",
    "    \n",
    "def featurize(words_embeddings, sentence):\n",
    "    \"\"\"average words' vectors\"\"\"\n",
    "    feature_vec = np.zeros((300,), dtype=\"float32\")\n",
    "    words = sentence.split(' ')\n",
    "    retrieved_words = 0\n",
    "    for token in words:\n",
    "        try:\n",
    "            feature_vec = np.add(feature_vec, words_embeddings.wv[token])\n",
    "            retrieved_words += 1\n",
    "        except KeyError:\n",
    "            pass  # if a word is not in the embeddings' vocabulary discard it\n",
    "    np.seterr(divide='ignore', invalid='ignore')\n",
    "    feature_vec = np.divide(feature_vec, retrieved_words)\n",
    "    return feature_vec\n",
    "\n",
    "def read_dataset_preprocess_featurize(csv_path, preprocess_object, words_embeddings, words_embeddings_size=300):\n",
    "    sample=pd.read_csv(csv_path, delimiter='\\t', skiprows=1, header=None, names=['index', 'tweet','emotion','score'])\n",
    "    sample['tweet'] = sample['tweet'].apply(lambda x : preprocess_object.preprocess_arabic_text(x, stem=False, replace_emojis=True, normalize_arabic=True))\n",
    "    sample = sample.dropna()\n",
    "    tweets = sample['tweet'].tolist()\n",
    "    scores = sample['score'].values\n",
    "    tweets_number = len(tweets)\n",
    "    tweets_embeddings = np.zeros((tweets_number,words_embeddings_size))\n",
    "    for i,tweet in enumerate(tweets):\n",
    "        tweets_embeddings[i,:] = featurize(words_embeddings, tweet)\n",
    "    return tweets_embeddings, scores\n",
    "\n",
    "prep = Arabic_preprocessing()\n",
    "tweeter_embeddings = gensim.models.Word2Vec.load('resources\\\\Twt-SG') # load tweeter word embeddings\n",
    "\n",
    "# prepare train data\n",
    "X_train, y_train = read_dataset_preprocess_featurize('data/2018-EI-reg-Ar-joy-train.txt', prep, tweeter_embeddings, 300)\n",
    "# prepare test data\n",
    "X_test, y_test = read_dataset_preprocess_featurize('data/2018-EI-reg-Ar-joy-dev.txt', prep, tweeter_embeddings, 300)\n",
    "\n",
    "#normalize data\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(np.append(X_train, X_test, axis=0))\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "print('training data: ', X_train.shape, y_train.shape)\n",
    "print('test data: ', X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt0XeV95vHvo7tk+W5hG1+wASfE\n5AKt4oRCSCYFapoWsjppCm2mZIUMpYWmM5l2CtM2aUlmJkmnado1tAkrIU3ToTSXtnEpDaEhSRsu\nicU9Bhxkc5HwBdmSL7KuR/rNH3sf+0g68pFlbR1dns9aWmdf3n38biP24/d99363IgIzM7OTqSh3\nBczMbOZzWJiZWUkOCzMzK8lhYWZmJTkszMysJIeFmZmV5LAwO02SPivpD06yPySdO511Mptq8nMW\nZqVJehFYCQwB3cA3gZsjonsCxwawKSJaM62kWYbcsjCbuJ+PiEbgAuBC4NYy18ds2jgszE5RROwD\n7iMJDST9laSP5/dL+h1JeyXtkfSBwmMlLZf0T5KOSNou6eOSvl+w/zxJ90vqlLRT0nun67zMTsZh\nYXaKJK0FrgTGdCtJ2gr8NnA5sAm4bFSR24FjwCrguvQnf+wC4H7gLuAM4FrgLySdP/VnYXZqHBZm\nE/ePko4CbcCrwEeLlHkv8MWI+FFEHAP+ML9DUiXwH4GPRkRPRDwDfKng2J8DXoyIL0ZELiIeA74O\nvCeb0zGbOIeF2cS9OyIWAu8AzgNWFClzJkmY5L1UsNwEVI3aX7h8FvAWSYfyP8CvkLRCzMrKYWF2\niiLie8BfAf+nyO69wLqC9fUFyx1ADlhbsK2wbBvwvYhYUvDTGBG/PjU1N5s8h4XZ5HwGuFzSBaO2\nfwV4v6TNkhoo6KqKiCHg74E/lNQg6TzgVwuOvQd4jaT/JKk6/XmzpNdlfC5mJTkszCYhIjqAvwb+\nYNT2fyEJkgdIBsAfGHXozcBiYB/wZeBvgf702KPAFcA1wJ60zCeB2qzOw2yi/FCeWRlJ+iSwKiKu\nK1nYrIzcsjCbRulzFG9UYgtwPfAP5a6XWSlV5a6A2TyzkKTr6UyS22//BPhGWWtkNgHuhjIzs5Iy\n7YaStDWdsqBV0i1F9t8o6WlJT0j6vqTN6fYNknrT7U9I+myW9TQzs5PLrGWRPq36Y5JpD9qB7cC1\n6VOr+TKLIuJIunwV8BsRsVXSBuCeiHj9RP+8FStWxIYNG6buBMzM5oFHH330QEQ0lSqX5ZjFFqA1\nInYDSLobuBo4Hhb5oEgtACadXBs2bKClpWWyh5uZzUuSXipdKttuqDWMnMqgPd02gqSbJO0CPgV8\nqGDXRkmPS/qepLdlWE8zMyshy7BQkW1jWg4RcXtEnAP8LvD76ea9wPqIuBD4MHCXpEVj/gDpBkkt\nklo6OjqmsOpmZlYoy7BoZ+S8N2tJnkodz93AuwEioj8iDqbLjwK7gNeMPiAi7oiI5ohobmoq2eVm\nZmaTlGVYbAc2SdooqYZkCoNthQUkbSpYfRfwfLq9KR0gR9LZJO8F2J1hXc3M7CQyG+COiJykm0ne\nKFYJ3BkROyTdBrRExDbgZkmXAYNAFydeBHMpcJukHMk7j2+MiM6s6mpmZic3Zx7Ka25uDt8NZWZ2\naiQ9GhHNpcp5bigzMyvJc0PZSeWGhjnQPcC+I33sO9zH/iN9dPUMUF1ZQW1VBXXVleN/VldQVzXy\ns7aqksqKYjfKmdlM5rCYx7r7c8cDYN/hPvYdObG8/0iy3nG0n+Ep7qmsrtSI8Mh/1lWPDKAl9TWs\nWFjDisZaljfWsqIxWV7RWMuS+moqHDpm08ZhMQcNDQcHuvvHBMDIMOinuz835thFdVWsWlzHykV1\nvGblwuPLqxbVHV9etqCGwaFh+nPD9A8O0Z8bpm+cz/7cEH2DSbm+3DD9g8P05YZGfB4vk653HRug\nd3CIQz2HOHhsgKEiaVVZIZYtyIdHGigLalix8MTnigW1rFhYw/IFtdRUucfV7HQ4LOaAQz0DfGvH\nfu790V6e23uUju7+MRfYqgpxxsJaVi5OQuBtm5pYtTgJgZXHg6CWhpqJ/UpUVlRSV10J9dVZnNJx\nw8PB4d5BDnT3c6B7IP3s5+Dx5eTzxYPHOHA0CZliFtVVjQmQFY21NC2sZfWSOlYvrmP14noW1VUh\nucViNprDYpbqOjbAt57Zxz8/vY+HWg+QGw7WLq3n4nNXsHpxHSvTIFi1qI6Vi5OL5GzstqmoEEsX\n1LB0QQ2bVpYuf6w/x8HuATq6+zlYECaFyzv3HeXgsYMc6hkcc/yCmkpWL6lPwyMJkDOXjPxcUOv/\nbWz+8W/9LNJ5bID7duzj3qf38tCugwwNB+uXNfDBt53Nu96wmtevWTTv/1W8oLaKBbVVrF/eULLs\nQG6YA9397D3cy55DSffcnsO97D3Ux97DvTy37ygHuvsZfXf5wroqzlxcn7ZITgTLmcdDpp76msqM\nztCsPBwWM1xEsO3JPXy1pZ2HdycBcdbyBm64NAmI8890QExWTVUFZy6p58wl9fzkWcXLDOSG2X+k\nj72H+wpCpZc96frT7Yc5eGxgzHFLGqqT1sjiOlYvqWPNkgbWLq1nzdJ61i6tp6mx1v/dbFZxWMxg\nR/oGueXrT3Hv0/vYuGIBN779bH72DavZvNoBMV1qqipYt6yBdcvGb6n0DQ6x/0gfe9IWST5Y9h7q\nY8/hPh59uWtMl1dNVQVrl5wIj7VLG1izpP54oJyxsM63GNuM4rCYoZ5uP8xNdz3GK4d6ufXK8/jP\nbzt7Vo45zAd11ZWctXwBZy1fMG6Z7v4cr3T18sqhHtq7enmlq5f2rl7au3q4f+8RDnSPbJ1UV4rV\ni/NBUj+mZbJqUR1Vlb7Dy6aPw2KGiQi+/MhLfPyeZ1neWMNXfu2t/ORZy8pdLTtNjbVVvHbVQl67\namHR/b0DQ7xyKAmP5LM3DZUevruzg1eP9o8oX1khVi2qG9EyyQfLuqUNrF7sMLGp5bCYQQq7nd55\n3hn8yS++iaULaspdLZsG9TWVnHtGI+ee0Vh0f9/gEHsP96UtkrR1kobLI7sOsu/IKyMenqysEKsX\n17EuDZGkKy0JlXVLGzhj4ey8O87Kx2ExQ7jbyU6mrrqSjSsWsHFF8a6ugdww+w730dbVQ3tXD22d\nvelyL9/78diWSU1lxYhWybpl9SOCZfmCGo+L2QgOizJzt5NNhZqqCtYvbxj3luG+waSbq60zCZC2\nrh7aO5OWyX179tE56o6u+urK48GR79o63kJZ2sDihmwfxrSZx2FRRu52sulSV13JOU2NnNNUvJsr\nPwDf1tlzvEWSLPey/YVOjo6aGmZhXVXapZUPkHwLJQkVP7g49/i/aJm428lmklID8Id7Bot2cb1w\n4Bj/9nwHfYPDI8ovX1CTdHGlLZET4yXJHV21VX5ocbZxWEwzdzvZbLS4oZrFDYt5/ZrFY/ZFBAe6\nB5Ig6cp3dSVhsuOVw3xrxz4Gh06MvkuwcmFdQbdWEiq+k2tmc1hMI3c72VwkiaaFyaSMF65fOmb/\n0HCw/0hfQddW0jpp7+rhhy908o0nekveyVU4EO8HFsvDYTFN3O1k81VlhY5Pq7Jl49hW9EBumL2H\ne0eESf45k2J3clVXKn3avUiYLK1nRaNvC86Cw2IaPNF2iPd+9mF3O5kVUVNVcdIn4PN3cuWfeM+3\nStq6evnXZ/ePefq9tip/W/CJgffCUPFtwZOTaVhI2gr8GVAJfD4iPjFq/43ATcAQ0A3cEBHPpPtu\nBa5P930oIu7Lsq5Z6Rsc4sNfeYLljTXc85uXsLyxttxVMptVSt3J1TswdHyM5ESrJAmVp9sP0TVq\nXq78bcEnpk8ZOS+XJ3ksLrOwkFQJ3A5cDrQD2yVty4dB6q6I+Gxa/irg08BWSZuBa4DzgTOBf5X0\nmogo/mabGexT39zJ7o5j/M31b3FQmGWgvqaSTSsXsmll8Tu5uvtzSZh0jg2Tx14+xOHek0/yuGbJ\nyFBZuWh+jplk2bLYArRGxG4ASXcDVwPHwyIijhSUXwDkh7muBu6OiH7gBUmt6fc9nGF9p9zDuw5y\n54Mv8KsXncUlm1aUuzpm81JjbRXnrVrEeasWFd1/tG+QVw71phM95ufkGn+Sx6oKpdPOnwiQ40/D\nL2lg9ZI6qufg3VxZhsUaoK1gvR14y+hCkm4CPgzUAO8sOPaRUceuKXLsDcANAOvXr5+SSk+Vo32D\n/PZXn2TD8gZuufK8clfHzMaxsK6a81ZVjxsm+Ukej0/0WBAq33/+APuP9o14QVaFYOWiuhFdW4Wh\nsmZJffJK4lkmy7Ao1k6LMRsibgdul/TLwO8D153CsXcAdwA0NzeP2V9O//Ofn2Xv4V6+euNFE36v\ntZnNPKUmeczfzXV82vlDJ1omLS918U9P7WVoeOTlaUVjbUFr5ESorFnSwJql9TTOwCfgs6xRO7Cu\nYH0tsOck5e8G/nKSx84oDzy3n7u3t3Hj28/xnU9mc1ypu7lyQ8PsP9p/PEAKWybP7DnC/Tv2MzA0\n8gn4JQ3VJ8ZLCt5lsmZJ8uDiovqqaR+EzzIstgObJG0EXiEZsP7lwgKSNkXE8+nqu4D88jbgLkmf\nJhng3gT8MMO6TpnDPYP87tef5rxVC/mvl28qd3XMrMyqKivSi37x50yGh4MD3f20F4yX5F+Stbvj\nGP/24wP0Do68t6extmrE4Pvm1Yu4Zku2XfGZhUVE5CTdDNxHcuvsnRGxQ9JtQEtEbANulnQZMAh0\nkXRBkZb7CslgeA64abbcCfXXD79Ix9F+7rzuzZ7/xsxKqqgQZyyq44xFdfxEkSfgI4KunsETLZPC\nl2Md6uWHL3by3L6jmYeFImZUV/+kNTc3R0tLS1nr0J8b4uJPfIfzz1zElz6wpax1MbP5o29waNKD\n5pIejYjmUuXm3v1dZfRPT+7lQHc/H3zbxnJXxczmkem4u8phMUUigs//+25eu3Ihl5zrZyrMbG5x\nWEyRh3Yd5Ll9R7n+ko2eKsDM5hyHxRT5wvdfYEVjDVddcGa5q2JmNuUcFlOg9dVuHnjuVd731rNm\n5ZOZZmalOCymwBcffIGaqgre99azyl0VM7NMOCxOU+exAb7+WDu/cOEaVnhWWTOboxwWp+muH7xE\n3+AwH7jEt8ua2dzlsDgN/bkhvvTwS1z6miZeM85c+mZmc4HD4jR8+9lX6Tjaz/VuVZjZHOewOA3/\n9uMOFtZV+SE8M5vzHBan4cFdB7jo7OXz8hWLZja/OCwmqa0zeYfvxW5VmNk84LCYpAdbDwBw8bnL\ny1wTM7PsOSwm6futBzhjYS3nNBV/1aKZ2VzisJiE4eHg4V0HueTcFZ400MzmBYfFJOzcf5SDxwb4\nKY9XmNk84bCYBI9XmNl847CYhId2HeTsFQtYvbi+3FUxM5sWDotTNDg0zA92H+Sn3Kows3kk07CQ\ntFXSTkmtkm4psv/Dkp6R9JSkb0s6q2DfkKQn0p9tWdbzVDzZdohjA0NcfI7HK8xs/qjK6oslVQK3\nA5cD7cB2Sdsi4pmCYo8DzRHRI+nXgU8Bv5Tu642IC7Kq32Q92HoQCS46xy0LM5s/smxZbAFaI2J3\nRAwAdwNXFxaIiO9ERE+6+giwNsP6TIkHdx3g9WcuZklDTbmrYmY2bbIMizVAW8F6e7ptPNcD/1Kw\nXiepRdIjkt5d7ABJN6RlWjo6Ok6/xiX0DOR4/OUuj1eY2byTWTcUUOxptShaUHof0Ay8vWDz+ojY\nI+ls4AFJT0fErhFfFnEHcAdAc3Nz0e+eSttf7GJwKDxeYWbzTpYti3ZgXcH6WmDP6EKSLgN+D7gq\nIvrz2yNiT/q5G/gucGGGdZ2Qh1oPUFNZwZs3LCt3VczMplWWYbEd2CRpo6Qa4BpgxF1Nki4EPkcS\nFK8WbF8qqTZdXgFcDBQOjJfF91sPcOH6JdTXVJa7KmZm0yqzsIiIHHAzcB/wLPCViNgh6TZJV6XF\n/hhoBL466hbZ1wEtkp4EvgN8YtRdVNOu69gAz+w94hcdmdm8lOWYBRFxL3DvqG0fKVi+bJzjHgLe\nkGXdTtUjuw8SgeeDMrN5yU9wT9Cz+44iwRvWLC53VczMpp3DYoLaO3s4c3E9NVX+KzOz+cdXvgl6\nubOHtUs9caCZzU8Oiwlq6+ph3bKGclfDzKwsHBYT0Dc4xP4j/axb6rAws/nJYTEBrxzqBWD9cndD\nmdn85LCYgJc7k7kO3bIws/nKYTEB7fmw8JiFmc1TDosJaOvqpaaqgqbG2nJXxcysLBwWE9DW2cO6\npfVUVBSbSNfMbO5zWEyAb5s1s/nOYTEBLx/s8eC2mc1rDosSDvcOcqQvx7plvm3WzOYvh0UJbb5t\n1szMYVFKe5dvmzUzc1iU8LKfsTAzc1iU0tbZy6K6KhbXV5e7KmZmZeOwKMG3zZqZOSxKauvsYb3D\nwszmOYfFSQwPB21dvW5ZmNm8l2lYSNoqaaekVkm3FNn/YUnPSHpK0rclnVWw7zpJz6c/12VZz/F0\ndPczkBtmnd+QZ2bzXGZhIakSuB24EtgMXCtp86hijwPNEfFG4GvAp9JjlwEfBd4CbAE+KmlpVnUd\nT/4Zi7VuWZjZPJdly2IL0BoRuyNiALgbuLqwQER8JyJ60tVHgLXp8s8A90dEZ0R0AfcDWzOsa1Ft\nXX4gz8wMsg2LNUBbwXp7um081wP/cirHSrpBUouklo6OjtOs7lhtnckb8ta6G8rM5rksw6LYfN5R\ntKD0PqAZ+ONTOTYi7oiI5ohobmpqmnRFx/NyZw8rF9VSV1055d9tZjabZBkW7cC6gvW1wJ7RhSRd\nBvwecFVE9J/KsVlL3mPhLigzsyzDYjuwSdJGSTXANcC2wgKSLgQ+RxIUrxbsug+4QtLSdGD7inTb\ntGr3bbNmZgBUZfXFEZGTdDPJRb4SuDMidki6DWiJiG0k3U6NwFclAbwcEVdFRKekj5EEDsBtEdGZ\nVV2LGcgNs/eww8LMDDIMC4CIuBe4d9S2jxQsX3aSY+8E7syudie351Avw4GfsTAzw09wj6vNU5Ob\nmR3nsBhH/rZZh4WZmcNiXG1dPVRXilWL6spdFTOzsnNYjOPlzh7WLKmnsqLYIx9mZvOLw2Ic7Z1+\nj4WZWZ7DYhxtXb2s9QN5ZmaAw6Ko7v4cnccGWLfMt82amUGJsJBUKenXJH1M0sWj9v1+tlUrn/zU\n5H5DnplZolTL4nPA24GDwJ9L+nTBvl/IrFZllg8LzwtlZpYoFRZbIuKXI+IzJC8iapT095JqKT4z\n7JzQ1uVnLMzMCpUKi5r8QkTkIuIG4AngAZI5neak/Uf6qK2qYGlDdbmrYmY2I5QKixZJI95QFxG3\nAV8ENmRVqXLrOjbA0oYa0skNzczmvZOGRUS8LyK+WWT75yNizv6zu6tnkCVuVZiZHTehW2clzatX\nxR3qGWDZgprSBc3M5omSYSFpIfCNaajLjNHVk3RDmZlZotRzFquBfwXumJ7qzAzuhjIzG6nUy4/+\nHfid9K1288LwcHDILQszsxFKdUN1AWumoyIzxdG+HMOBWxZmZgVKhcU7gCsl3TQNdZkRunoGANyy\nMDMrUOrW2WPAVcCF01Od8suHhe+GMjM7oeTdUBExFBEfnMyXS9oqaaekVkm3FNl/qaTHJOUkvWfU\nviFJT6Q/0zZmcqhnEHA3lJlZoUlNUZ7ORvsrpcoAtwNXApuBayVtHlXsZeD9wF1FvqI3Ii5If66a\nTD0no/OYu6HMzEYrdevsIkm3Svq/kq5Q4jeB3cB7S3z3FqA1InZHxABwN3B1YYGIeDEingKGT+Mc\nppTHLMzMxirVsvgy8FrgaeCDwLeA9wBXR8TVJzuQ5C6qtoL1dk7tzqo6SS2SHpH07mIFJN2Qlmnp\n6Og4ha8e36GeQSoEC+tK3VVsZjZ/lLoinh0RbwCQ9HngALA+Io5O4LuLzcIXp1C39RGxR9LZwAOS\nno6IXSO+LOIO0gcGm5ubT+W7x9XVM8CShhoqKjyJoJlZXqmWxWB+ISKGgBcmGBSQtCTWFayvBfZM\ntGIRsSf93A18l2m6I+tQz6CnJjczG6VUWLxJ0pH05yjwxvyypCMljt0ObJK0UVINcA0wobuaJC1N\nX7CEpBXAxcAzEzn2dHleKDOzsU7aDRURk55tNiJykm4G7gMqgTsjYoek24CWiNgm6c3APwBLgZ+X\n9EcRcT7wOuBzkoZJAu0TETEtYXG4d5BVi+qm448yM5s1Mh3FjYh7gXtHbftIwfJ2ku6p0cc9BLwh\ny7qNp7s/58FtM7NRJvWcxVzW3Zej0WFhZjaCw2KUo/05Gms9wG1mVshhUaA/N8RAbtjdUGZmozgs\nChzrHwKgsdZhYWZWyGFR4Ghf8liJw8LMbCSHRYGjfTkAD3CbmY3isCjQ3Z+ExUK3LMzMRnBYFOhO\nWxYL63w3lJlZIYdFgXzLwt1QZmYjOSwKHM2HhbuhzMxGcFgUONEN5bAwMyvksCjQ3T9IVYWorfJf\ni5lZIV8VC+TnhZL84iMzs0IOiwJH+3IerzAzK8JhUSCZRNBhYWY2msOiQHdfjkV+xsLMbAyHRYHu\nfr/LwsysGIdFgW53Q5mZFeWwKHDUb8kzMyvKYVGgu3/QkwiamRWRaVhI2ippp6RWSbcU2X+ppMck\n5SS9Z9S+6yQ9n/5cl2U9AQaHhukbHHY3lJlZEZmFhaRK4HbgSmAzcK2kzaOKvQy8H7hr1LHLgI8C\nbwG2AB+VtDSrugL0pG/Ja3BYmJmNkWXLYgvQGhG7I2IAuBu4urBARLwYEU8Bw6OO/Rng/ojojIgu\n4H5ga4Z1pXcwDYuayiz/GDOzWSnLsFgDtBWst6fbpuxYSTdIapHU0tHRMemKwomwqK92WJiZjZZl\nWBSbYCmm8tiIuCMimiOiuamp6ZQqN1rvQBIWdQ4LM7MxsgyLdmBdwfpaYM80HDspvYPJ9OTuhjIz\nGyvLsNgObJK0UVINcA2wbYLH3gdcIWlpOrB9RbotM70DybBJvcPCzGyMzMIiInLAzSQX+WeBr0TE\nDkm3SboKQNKbJbUDvwh8TtKO9NhO4GMkgbMduC3dlhmPWZiZjS/T+0Qj4l7g3lHbPlKwvJ2ki6nY\nsXcCd2ZZv0L5sPCYhZnZWH6CO9WXDnC7G8rMbCyHRcrdUGZm43NYpHoGHBZmZuNxWKROjFn4r8TM\nbDRfGVN9g0PUV1ciFXse0MxsfnNYpHoHhjy4bWY2DodFqjdtWZiZ2VgOi1TvwJDHK8zMxuGrY6p3\n0N1QZmbjcVikegeGaKj2i4/MzIpxWKR6B4eoc8vCzKwoh0UquXXWfx1mZsX46pjqGfDdUGZm43FY\npDzAbWY2PodFqm9gyNOTm5mNw2GR6h0c8itVzczG4bAABoeGyQ2HxyzMzMbhsMBvyTMzK8VhQXLb\nLEBtlf86zMyK8dURGMgNA1DjsDAzKyrTq6OkrZJ2SmqVdEuR/bWS/i7d/wNJG9LtGyT1Snoi/fls\nlvV0WJiZnVxmkyFJqgRuBy4H2oHtkrZFxDMFxa4HuiLiXEnXAJ8EfindtysiLsiqfoUGhtKwqPSY\nhZlZMVn+U3oL0BoRuyNiALgbuHpUmauBL6XLXwN+WmV4Vd1gLgC3LMzMxpPl1XEN0Faw3p5uK1om\nInLAYWB5um+jpMclfU/S24r9AZJukNQiqaWjo2PSFR0YSga4HRZmZsVleXUs1kKICZbZC6yPiAuB\nDwN3SVo0pmDEHRHRHBHNTU1Nk65of37MotJhYWZWTJZXx3ZgXcH6WmDPeGUkVQGLgc6I6I+IgwAR\n8SiwC3hNVhX1ALeZ2clleXXcDmyStFFSDXANsG1UmW3Adenye4AHIiIkNaUD5Eg6G9gE7M6qovmw\n8HMWZmbFZXY3VETkJN0M3AdUAndGxA5JtwEtEbEN+ALwZUmtQCdJoABcCtwmKQcMATdGRGdWdT1+\nN5TDwsysqEzfIxoR9wL3jtr2kYLlPuAXixz3deDrWdat0IDHLMzMTspXRzxmYWZWiq+OuBvKzKwU\nXx1xy8LMrBRfHfFzFmZmpfjqiAe4zcxK8dWRZMyiulJUVEz7tFRmZrOCw4KkZeFWhZnZ+HyFJA0L\nD26bmY3LV0gcFmZmpfgKSTJm4bAwMxufr5B4zMLMrBRfIUmes6ip8itVzczG47DA3VBmZqX4CgkM\n5IaodTeUmdm4fIXEd0OZmZXiKyTuhjIzK8VXSHw3lJlZKb5C4m4oM7NSfIXEYWFmVkqmV0hJWyXt\nlNQq6ZYi+2sl/V26/weSNhTsuzXdvlPSz2RZT49ZmJmdXGZXSEmVwO3AlcBm4FpJm0cVux7oiohz\ngT8FPpkeuxm4Bjgf2Ar8Rfp9mej3mIWZ2UlleYXcArRGxO6IGADuBq4eVeZq4Evp8teAn5akdPvd\nEdEfES8Aren3ZWIgN0ytWxZmZuPK8gq5BmgrWG9PtxUtExE54DCwfILHTomIcDeUmVkJWV4hi712\nLiZYZiLHIukGSS2SWjo6OiZRRcgNBxF+paqZ2clkeYVsB9YVrK8F9oxXRlIVsBjonOCxRMQdEdEc\nEc1NTU2TquTx92+7ZWFmNq4sr5DbgU2SNkqqIRmw3jaqzDbgunT5PcADERHp9mvSu6U2ApuAH2ZR\nSYeFmVlpVVl9cUTkJN0M3AdUAndGxA5JtwEtEbEN+ALwZUmtJC2Ka9Jjd0j6CvAMkANuioihLOpZ\nUSHe9cbVnN3UmMXXm5nNCUr+IT/7NTc3R0tLS7mrYWY2q0h6NCKaS5Vz34uZmZXksDAzs5IcFmZm\nVpLDwszMSnJYmJlZSQ4LMzMryWFhZmYlOSzMzKykOfNQnqQO4KVJHr4CODCF1Sknn8vMNFfOZa6c\nB/hc8s6KiJKT682ZsDgdklom8gTjbOBzmZnmyrnMlfMAn8upcjeUmZmV5LAwM7OSHBaJO8pdgSnk\nc5mZ5sq5zJXzAJ/LKfGYhZmZleSWhZmZleSwMDOzkuZ9WEjaKmmnpFZJt5S7PqdC0p2SXpX0o4Jt\nyyTdL+n59HNpOes4EZLWSfqOpGcl7ZD0W+n22XgudZJ+KOnJ9Fz+KN2+UdIP0nP5u/RVw7OCpEpJ\nj0u6J12fleci6UVJT0t6QlJiHHC+AAAEw0lEQVRLum02/o4tkfQ1Sc+l/89cNB3nMa/DQlIlcDtw\nJbAZuFbS5vLW6pT8FbB11LZbgG9HxCbg2+n6TJcD/ltEvA54K3BT+t9hNp5LP/DOiHgTcAGwVdJb\ngU8Cf5qeSxdwfRnreKp+C3i2YH02n8t/iIgLCp5JmI2/Y38GfDMizgPeRPLfJvvziIh5+wNcBNxX\nsH4rcGu563WK57AB+FHB+k5gdbq8GthZ7jpO4py+AVw+288FaAAeA95C8nRtVbp9xO/dTP4B1qYX\nn3cC9wCaxefyIrBi1LZZ9TsGLAJeIL05aTrPY163LIA1QFvBenu6bTZbGRF7AdLPM8pcn1MiaQNw\nIfADZum5pN02TwCvAvcDu4BDEZFLi8ym37PPAP8dGE7XlzN7zyWAb0l6VNIN6bbZ9jt2NtABfDHt\nGvy8pAVMw3nM97BQkW2+l7hMJDUCXwf+S0QcKXd9JisihiLiApJ/lW8BXles2PTW6tRJ+jng1Yh4\ntHBzkaIz/lxSF0fET5B0O98k6dJyV2gSqoCfAP4yIi4EjjFNXWfzPSzagXUF62uBPWWqy1TZL2k1\nQPr5apnrMyGSqkmC4v9FxN+nm2flueRFxCHguyTjMEskVaW7Zsvv2cXAVZJeBO4m6Yr6DLPzXIiI\nPennq8A/kAT5bPsdawfaI+IH6frXSMIj8/OY72GxHdiU3t1RA1wDbCtznU7XNuC6dPk6kv7/GU2S\ngC8Az0bEpwt2zcZzaZK0JF2uBy4jGYD8DvCetNisOJeIuDUi1kbEBpL/Nx6IiF9hFp6LpAWSFuaX\ngSuAHzHLfsciYh/QJum16aafBp5hOs6j3AM25f4Bfhb4MUm/8u+Vuz6nWPe/BfYCgyT/4riepE/5\n28Dz6eeyctdzAudxCUlXxlPAE+nPz87Sc3kj8Hh6Lj8CPpJuPxv4IdAKfBWoLXddT/G83gHcM1vP\nJa3zk+nPjvz/67P0d+wCoCX9HftHYOl0nIen+zAzs5LmezeUmZlNgMPCzMxKcliYmVlJDgszMyvJ\nYWFmZiU5LMxOQzqT6YrTLWM20zkszMysJIeF2QRJ+sd0ErodBRPR5fdtSN8v8CVJT6XvG2goKPKb\nkh5L36dwXnrMFkkPpRPCPZR/KlfS+ek7MZ5Iv2vTNJ6mWVEOC7OJ+0BE/CTQDHxI0vJR+18L3BER\nbwSOAL9RsO9AJJPY/SXw2+m254BLI5kQ7iPA/0q33wj8WSSTETaTPJ1vVlYOC7OJ+5CkJ4FHSCag\nHP0v/raIeDBd/huSaUzy8pMjPkryDhKAxcBX0zcd/ilwfrr9YeB/SPpd4KyI6J3SszCbBIeF2QRI\negfJpIAXRfIWvMeBulHFRs+dU7jen34OkUwzDfAx4DsR8Xrg5/PfFxF3AVcBvcB9kt45RadhNmkO\nC7OJWQx0RURPOubw1iJl1ku6KF2+Fvj+BL7zlXT5/fmNks4GdkfEn5PMJvrG06m42VRwWJhNzDeB\nKklPkbQIHilS5lngurTMMpLxiZP5FPC/JT0IVBZs/yXgR+nb9s4D/vp0K292ujzrrNkUSF8He0/a\npWQ257hlYWZmJbllYWZmJbllYWZmJTkszMysJIeFmZmV5LAwM7OSHBZmZlbS/wdOE6Eoq36chQAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x27d912c02e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maximum R^2:  0.336794600698\n",
      "Best alpha:  15\n",
      "Pearson score = 0.585259933443\n"
     ]
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.stats import pearsonr\n",
    "\n",
    "alphas = [0.1,0.3,0.5,0.75,1,2,3,4,5,8,10,15,20,25,30,35,40,45,50,60]\n",
    "r2 = []\n",
    "max_r2, best_alpha = 0, 0\n",
    "for alpha in alphas:\n",
    "    model = linear_model.Ridge(alpha=alpha, random_state=seed)\n",
    "    model.fit(X_train, y_train) # Train the model using the training sets\n",
    "    r = model.score(X_test, np.array(y_test))\n",
    "    if r > max_r2: max_r2, best_alpha = r, alpha\n",
    "    r2.append(r)\n",
    "\n",
    "plt.plot(alphas,r2)\n",
    "plt.xlabel('alphas')\n",
    "plt.ylabel('R^2')\n",
    "plt.title('Ridge')\n",
    "plt.show()\n",
    "\n",
    "print('maximum R^2: ', max_r2)\n",
    "print('Best alpha: ', best_alpha)\n",
    "\n",
    "best_ridge_model = linear_model.Ridge(alpha=best_alpha, random_state=seed)\n",
    "best_ridge_model.fit(X_train, y_train) # Train the model using the training sets\n",
    "y_pred = best_ridge_model.predict(X_test) # Make predictions using the testing set\n",
    "pearson = pearsonr(y_test, y_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt83HWd7/HXO0nTFnpJSwv0arkU\nSrG0xVC8chcLaqkICMoKyh5EZddzPCpw8LAuu16Qdb3sIgtHAREULYJ2FRYQC96AbVqalhZKQ29J\n00sKTa9p0ySf88f8UqZhcutkMrm8n4/HPGbm+/v+fvOZ6XTe+V2/igjMzMwOVUG+CzAzs97NQWJm\nZllxkJiZWVYcJGZmlhUHiZmZZcVBYmZmWXGQmB0iSWdJqsp3HWb55iCxPkXSWkl1knZJ2iTpPklD\n8l1XtiSFpN3J+9olqbabX9+haa1ykFhf9OGIGALMAGYCN+W5nq4yPSKGJLeSzs4sqSgXRZk5SKzP\niohNwBOkAgUASR+U9KKkHZIqJX0tbdqk5C//qyStl7RV0s1p0wcnazjbJK0ATkt/PUknSXpGUq2k\n5ZLmpE27T9IPJT2erFH8RdLRkr6XLO8VSTMP5X1K+h+SKiS9IWm+pLFp00LS5yWtAlYlbVMkPZX0\nXynpsrT+F0paIWmnpA2SviTpcOBxYGzaGtHYtxRi/ZaDxPosSeOBC4CKtObdwCeBEuCDwGclzW0x\n63uBE4FzgVsknZS0/wNwXHL7AHBV2msNAP4TeBI4Evg74EFJJ6Yt9zLgq8AoYB/wHLA4ef4w8K+H\n8B7PAb6ZLHsMsA54qEW3ucDpwNQkFJ4CfpbUeQXwQ0knJ31/DHwmIoYCbwf+EBG7SX2O1WlrRNWd\nrdX6LgeJ9UW/lrQTqAS2kAoAACLimYhYFhFNEbEU+DlwZov5/zEi6iKiHCgHpiftlwFfj4g3IqIS\n+EHaPO8EhgDfioj6iPgD8FtSP9TNHo2IRRGxF3gU2BsR90dEI/ALUpvh2rI4WdupldT82p8A7omI\nxRGxj9RmvHdJmpQ23zeTmuuADwFrI+LeiGiIiMXAr4BLkr77SQXOsIjYlkw3a5ODxPqiuclf1GcB\nU0j9xQ+ApNMlLZBUI2k7cF369MSmtMd7SAUEwFhS4dRsXdrjsUBlRDS1mD4u7fnmtMd1GZ63d1DA\nqRFRktz+Pu11D9QREbuA11u8bnrNbwNOTwukWlJhdHQy/aPAhcA6Sc9Kelc7NZk5SKzviohngfuA\nf0lr/hkwH5gQEcOB/wDUwUVuBCakPZ+Y9rgamCCpoMX0DZ0su7OqSYUDAMmmqyNavG76Jb4rgWfT\nAqkk2VT1WYCIWBgRF5Ha7PVr4JcZlmF2EAeJ9XXfA94vqXmH+1DgjYjYK2kW8PFOLOuXwE2SRiT7\nX/4ubdoLpPa/fEXSAElnAR/mrfsrutrPgE9JmiFpIPAN4IWIWNtK/98CJ0j6m6TOAZJOSw4UKJb0\nCUnDI2I/sANoTObbDBwhaXiO34/1Qg4S69Mioga4H/i/SdPngFuTfSi38OZf3B3xj6Q2I60htVP9\np2mvUw/MIbVTeivwQ+CTEfFKtu+hLRHxNKn39itSa0zHAZe30X8ncH7Sp5rUZrzbgIFJl78B1kra\nQWqz35XJfK+Q2p+0Otkk5qO27AB5YCszM8uG10jMzCwrDhIzM8uKg8TMzLLiIDEzs6z0i4u4jRo1\nKiZNmpTvMszMepVFixZtjYjR7fXrF0EyadIkysrK8l2GmVmvImld+728acvMzLLkIDEzs6w4SMzM\nLCsOEjMzy4qDxMzMsuIgMTOzrDhIzMwsK/3iPBKz7hAR1O7Zz4baOqpr69hQW0ftnv0UFxUwsKiA\n4qICigsLGDiggOLCwoPbk8cDi1LTUn3enFZUIKSOjr9l1r0cJGYdtL+xic079rJhWx3V2+vYsK2O\nDbV7D4RGdW0de+ob21/QISgQaUFUmLpPC6DUfWGLsGrRnh5WhxhqA4sKKCr0hgw7mIPELLFz736q\nk2Coal6r2Pbm2sXmHXtpajF8zxGHFzO2ZDDHjT6cMyaPZmzJIMaPGMzYksGMKxnMiMOKqW9sor6x\niX37U/f1DU3sa2ikvqHpwG1fckv1a0zrl94nmacxrX/atD31DdTWtXydN/vUNzZlfuOd1BxqBwVX\nlqH2ZrgdHGoDCgsoEBRIKLlP1SAKCkCIAoEEklDztKR/c3uB0vu+uazm/ohWX6f5efO89lYOEusX\nmpqCml37qEoLhurk1ty2Y2/DQfMUFYgxJYMYVzKYdx13BONLUgExtmQw40YMZuzwwQwuLmz3tQcV\nFDJoQCEMytW765impjgQaq2GVItQyzStvkXgtQy1+oamA6H21tfp2lDLhyR30sIqc1AJKCjQgcB6\nM8BaCbW0cGvZBm8NNQ68biu1JEF72yWnMK5kcE4/EweJ9Um79jXw4PPrWLByCxtq69i0fS/7Gw9e\nnRg2qOjAmsOsY0YeeNx8P3roQAoL+s5foAUFejPU8qxlqLW19hUBQdDUBAE0RaTaImhqnpY8T+/b\nFEHwZnumvk1p9yTLTk1Pa2uKA6/blFpgi2U113Lw66TX+NZa0vpGHPS+0vum3vPBrxMH1RktPp/m\n9iaisXmZuR8F10FifUrtnnru++ta7v3LWrbX7WfauOHMnDCCsdNSaxHjSgYxruQwxpYMYuigAfku\nt9/qSaFm2XOQWJ9Qs3MfP/rzah54bh276xs576SjuP6c45kxoSTfpZn1eQ4S69U21NZx97Ov8dDC\nSvY3NvHBU8by+bOPY8rRw/Jdmlm/4SCxXmnN1t3c+UwFj764gQi4+NRxfPas4zlm1OH5Ls2s33GQ\nWK+yctNO7lhQwW+XVlNUWMAVsyZy7RnHMn7EYfkuzazfcpBYr1BeWcu/L6jgqRWbOby4kP/xvmO5\n5n3HcOTQPB9Ta2YOEuvZXlj9Ov++oII/rdrKsEFFfOHcyXzqPZMoOaw436WZWcJBYj1ORPDsqzXc\nsaCChWu3MWpIMTfMnsKV75zoQ3bNeiAHifUYTU3Bkys2cceC11i2YTtjhg/iax+eysdOm9ihM8jN\nLD8cJJZ3DY1N/HbpRu5YUMGqLbuYdMRh3PbRaXxk5niKi3yBQLOeLqdBImk28H2gEPhRRHyrxfQv\nAn8LNAA1wKcjYp2ks4HvpnWdAlweEb+WdB9wJrA9mXZ1RCzJ5fuw3NjX0Mgjizdw5zOvsf6NPZxw\n1BC+f/kMPjhtjK8wa9aL5CxIJBUCdwDvB6qAhZLmR8SKtG4vAqURsUfSZ4FvAx+LiAXAjGQ5I4EK\n4Mm0+b4cEQ/nqnbLrbr6Rn7+3+u5+4+r2bRjL6eMH87NH3wH7z/pKAr60LWtzPqLXK6RzAIqImI1\ngKSHgIuAA0GSBEaz54ErMyznEuDxiNiTw1qtG+zcu5/7n1vHPX9ew+u765l1zEi+fckpvG/yKF+e\n26wXy2WQjAMq055XAae30f8a4PEM7ZcD/9qi7euSbgGeBm6MiH0tZ5J0LXAtwMSJEztRtnW1bbvr\nufcva7j3r2vZubeBM04YzfVnH8+sY0bmuzQz6wK5DJJMf2JmvJ6xpCuBUlL7PtLbxwDTgCfSmm8C\nNgHFwN3ADcCtb3mhiLuT6ZSWlub+Osr2Flt37eOuZ1/jwRfWs6e+kQ+cfBSfP/t4ThnvCyma9SW5\nDJIqYELa8/FAdctOks4DbgbOzLBmcRnwaETsb26IiI3Jw32S7gW+1KVVW9Yigl8v2cA//ucKdtTt\nZ870sXzu7OM54aih+S7NzHIgl0GyEJgs6RhgA6lNVB9P7yBpJnAXMDsitmRYxhWk1kDS5xkTERuV\n2qg+F3gpF8XboamurePmR5exYGUNp04s4baPnsJkB4hZn5azIImIBknXk9osVQjcExHLJd0KlEXE\nfOB2YAgwL9nZuj4i5gBImkRqjebZFot+UNJoUpvOlgDX5eo9WMdFBD//70q+8djLNDYFt3xoKle9\ne1KfGmHQzDJTdwzDmG+lpaVRVlaW7zL6rPWv7+GGXy3ludWv8+7jjuBbF5/CxCN8NV6z3k7Soogo\nba+fz2y3Q9bYFPzkr2u5/YmVFBaIb148jctPm+BDec36GQeJHZKKLbv4ysPlLF5fy9knjuYbF09j\nzPDB+S7LzPLAQWKdsr+xibv/uJrvP72Kw4oL+e7HpjN3xjivhZj1Yw4S67Dl1dv5ysNLWV69gwun\nHc0/znk7o4cOzHdZZpZnDhJr176GRu74QwU/fOY1Sg4r5s5PnMoF08bkuywz6yEcJNamF9dv4ysP\nL2XVll1cfOo4bvnQVI9OaGYHcZBYq3785zV8/XcrOGrYIO791GmcfeKR+S7JzHogB4ll9MzKLfzz\n71Zw/tSj+JdLp3uIWzNrlYPE3mL963v4wkNLmHL0ML73sZke5tbM2uRh6OwgdfWNfOaBRUQEd135\nDoeImbXLayR2QERw86PLeGXTDu65+jRf5sTMOsRrJHbA/c+t45EXN/C/zjvBO9bNrMMcJAZA2do3\n+KffruC8k47k+rOPz3c5ZtaLOEiMLTv28tkHFzN+xGC+c9kMCnzpdzPrBO8j6efqG5r43IOL2bW3\ngQeuOZ3hg32Yr5l1joOkn/vGYy9Ttm4b/3bFTE482iMZmlnnedNWP/boi1Xc99e1/O17j+HD08fm\nuxwz66UcJP3Uiuod3PTIMk4/ZiQ3XjAl3+WYWS/mIOmHavfU85kHyigZXMy/f/xUigr9NTCzQ+d9\nJP1MY1PwhYeWsGn7Xn7xmXd5PBEzy1pO/xSVNFvSSkkVkm7MMP2LklZIWirpaUlvS5vWKGlJcpuf\n1n6MpBckrZL0C0m+pnknfP/3r/LsqzV8bc7JnDpxRL7LMbM+IGdBIqkQuAO4AJgKXCFpaotuLwKl\nEXEK8DDw7bRpdRExI7nNSWu/DfhuREwGtgHX5Oo99DW/X7GZH/yhgkvfMZ6Pz5qY73LMrI/I5RrJ\nLKAiIlZHRD3wEHBReoeIWBARe5KnzwPj21qgUgODn0MqdAB+Aszt0qr7qDVbd/O/frGEaeOG809z\n3+4x1s2sy+QySMYBlWnPq5K21lwDPJ72fJCkMknPS2oOiyOA2ohoaG+Zkq5N5i+rqak5tHfQR+ze\n18B1P11EUaG488pTGTTAV/Q1s66Ty53tmf7kjYwdpSuBUuDMtOaJEVEt6VjgD5KWATs6usyIuBu4\nG6C0tDRjn/4gIrjhV0tZtWUn93/6dMaP8BV9zaxr5XKNpAqYkPZ8PFDdspOk84CbgTkRsa+5PSKq\nk/vVwDPATGArUCKpOQAzLtPe9OM/r+G3Szfy5Q9M4b2TR+W7HDPrg3IZJAuByclRVsXA5cD89A6S\nZgJ3kQqRLWntIyQNTB6PAt4DrIiIABYAlyRdrwJ+k8P30Ks9v/p1vvn4K8w++WiuO/PYfJdjZn1U\nzoIk2Y9xPfAE8DLwy4hYLulWSc1HYd0ODAHmtTjM9ySgTFI5qeD4VkSsSKbdAHxRUgWpfSY/ztV7\n6M32NzZx0yPLmDjyMG6/9BTvXDeznMnpCYkR8RjwWIu2W9Ien9fKfH8FprUybTWpI8KsDT97YT1r\ntu7mnqtLGTrIV/Q1s9zxtTH6oB179/P9p1fxrmOP8EiHZpZzDpI+6D+eeY03dtfzfy48yZu0zCzn\nHCR9THVtHT/+8xrmzhjLtPHD812OmfUDDpI+5jtPvkoAX/rAifkuxcz6CQdJH7KiegePvFjFp949\nyScemlm3cZD0Id98/GWGDx7A584+Pt+lmFk/4iDpI/74ag1/WrWVvztnMsMH+3BfM+s+DpI+oLEp\n+MZjLzNx5GH8zTvf1v4MZmZdyEHSBzyyuIpXNu3kK7NPpLjI/6Rm1r38q9PL1dU38p0nX2XGhBI+\nOG1Mvssxs37IQdLL3fOXNWzasdcnH5pZ3jhIerGtu/Zx5zOv8f6pRzHrmJH5LsfM+ikHSS/2g6dX\nUbe/kRsvmJLvUsysH3OQ9FKra3bxsxfWc8WsCRw3eki+yzGzfsxB0kvd9l+vMLCogC+ce0K+SzGz\nfs5B0gstXPsGTyzfzHVnHsfooQPzXY6Z9XMOkl4mInXy4VHDBvK37/PwuWaWfw6SXuaxZZt4cX0t\n//v9JzK4uDDf5ZiZOUh6k/qGJr79xCtMOXooH33H+HyXY2YGOEh6lQeeX8e61/dw4wVTKCzwyYdm\n1jPkNEgkzZa0UlKFpBszTP+ipBWSlkp6WtLbkvYZkp6TtDyZ9rG0ee6TtEbSkuQ2I5fvoafYXref\nH/xhFe89fhRnnjA63+WYmR2QsyCRVAjcAVwATAWukDS1RbcXgdKIOAV4GPh20r4H+GREnAzMBr4n\nqSRtvi9HxIzktiRX76En+eEzFWyv289NF07xpVDMrEfJ5RrJLKAiIlZHRD3wEHBReoeIWBARe5Kn\nzwPjk/ZXI2JV8rga2AL02z/Dq7bt4d6/rOUjM8dx8liPw25mPUsug2QcUJn2vCppa801wOMtGyXN\nAoqB19Kav55s8vqupIwnUki6VlKZpLKamprOV9+DfOfJVxHwpfM9DruZ9Ty5DJJM218iY0fpSqAU\nuL1F+xjgp8CnIqIpab4JmAKcBowEbsi0zIi4OyJKI6J09OjeuzLz0obtPPriBj793mMYWzI43+WY\nmb1FLoOkCpiQ9nw8UN2yk6TzgJuBORGxL619GPA74KsR8Xxze0RsjJR9wL2kNqH1Sc0nH444bACf\nPeu4fJdjZpZRLoNkITBZ0jGSioHLgfnpHSTNBO4iFSJb0tqLgUeB+yNiXot5xiT3AuYCL+XwPeTV\nMytr+Otrr/P3505m2CCPw25mPVNRrhYcEQ2SrgeeAAqBeyJiuaRbgbKImE9qU9YQYF5yJNL6iJgD\nXAacARwh6epkkVcnR2g9KGk0qU1nS4DrcvUe8u2OBRVMHHkYnzjd47CbWc+VsyABiIjHgMdatN2S\n9vi8VuZ7AHiglWnndGWNPdXqml2UrdvGDbOneBx2M+vR/AvVQz28qIoCwcWntnWgm5lZ/jlIeqDG\npuCRxRs484TRHDVsUL7LMTNrk4OkB/rTqho27djLpaUT2u9sZpZnDpIeaN6iKkoOG8C5Jx2Z71LM\nzNrlIOlhavfU89TyzcydMY6BRR5vxMx6PgdJDzO/vJr6xiYu8XgjZtZLOEh6mHllVZw0ZhhvH+eL\nM5pZ7+Ag6UFe2bSDZRu2e23EzHoVB0kP8nBZFUUFYu6MsfkuxcyswxwkPcT+xiZ+vWQD5550JEcM\nyXhlfDOzHslB0kMseGULW3fVc+k7fO6ImfUubQaJpEJJn5H0T5Le02LaV3NbWv8yb1EVo4YM5KwT\ne+/YKWbWP7W3RnIXcCbwOvADSf+aNu3inFXVz2zdtY8Fr2zh4lPHUVTolUQz613a+9WaFREfj4jv\nAacDQyQ9kgxvm2kERDsEv35xAw1NwaU+WsvMeqH2gqS4+UFENETEtaTGAPkDqXFELEsRwbyyKqZP\nKGHyUUPzXY6ZWae1FyRlkmanN0TEraSGuJ2Uq6L6k2UbtrNy806vjZhZr9VmkETElRHxXxnafxQR\nHvu1C8wrq6K4qIAPT/e5I2bWO3Voz64kXz0wB/bub2R+eTUfOPlohg92LptZ79RukEgaCvymG2rp\nd37/8ma21+33Zi0z69XaO49kDPB74O5DWbik2ZJWSqqQdGOG6V+UtELSUklPS3pb2rSrJK1Kblel\ntb9D0rJkmT+Q1GuPHptXVsWY4YN4z/Gj8l2Kmdkha2+N5E/AtyJifmcXnGwOuwO4AJgKXCFpaotu\nLwKlEXEK8DDw7WTekcA/kDrkeBbwD5JGJPPcCVwLTE5us+mFNm3fy59W1fDRU8dTWNBrs9DMrN0g\n2QaMO8RlzwIqImJ1RNQDDwEXpXeIiAURsSd5+jzQvI3nA8BTEfFGRGwDngJmJ2tIwyLiuYgI4H5g\n7iHWl1e/WlxFU+Ar/ZpZr9dekJwFXCDp84ew7HFAZdrzKtoOpWuAx9uZd1zyuN1lSrpWUpmkspqa\nmk6WnlsRwcOLqpg1aSSTRh2e73LMzLLS3uG/u4E5wMxDWHam7TWRsaN0JVAK3N7OvB1eZkTcHRGl\nEVE6enTPun7VonXbWLN1N5eUem3EzHq/do/aiojGiPjbQ1h2FZB+KdvxQHXLTpLOA24G5kTEvnbm\nreLNzV+tLrOnm1dWxWHFhXxw2ph8l2JmlrVDukJgclXgT7TTbSEwWdIxkoqBy4GDdtpLmknqwpBz\nImJL2qQngPMljUh2sp8PPBERG4Gdkt6ZHK31SXrZocl76hv47dJqLpw2hsMHFuW7HDOzrLV3+O8w\nSTdJ+ndJ5yvl74DVwGVtzRsRDcD1pELhZeCXEbFc0q2S5iTdbid1za55kpZImp/M+wbwT6TCaCFw\na9IG8FngR0AF8Bpv7lfpFR5ftond9Y0+d8TM+gylDn5qZaL0G1JHbj0HnAuMIHUhxy9ExJJuqbAL\nlJaWRllZWb7LAODyu5+junYvz375LHrxKTBm1g9IWhQRpe31a2/byrERMS1Z4I+ArcDEiNjZBTX2\nO5Vv7OH51W/wxfef4BAxsz6jvX0k+5sfREQjsMYhcugeXlSFBB/1Zi0z60PaWyOZLmlH8ljA4OS5\ngIiIYTmtrg9pakqdO/Ke40YxrmRwvssxM+sybQZJRPiqv13k+dWvs6G2jq/MPjHfpZiZdSkPEN5N\n5i2qYuigIj5w8tH5LsXMrEs5SLrBjr37efyljXx4+lgGDfBKnpn1LQ6SbvC7pRvZu7/J546YWZ/k\nIOkG88oqOf7IIcyYUJLvUszMupyDJMcqtuxi8fpaLn3HeJ87YmZ9koMkxx5eVEVhgfjIzEMd1sXM\nrGdzkORQY1Pw6ItVnHnCaI4cNijf5ZiZ5YSDJIf+uKqGzTv2eSe7mfVpDpIc+s2LGxhx2ADOPemo\nfJdiZpYzDpIcKlu3jXcfP4riIn/MZtZ3+RcuR7bu2kfVtjqmjx+e71LMzHLKQZIjS6tqAZg+3ueO\nmFnf5iDJkSWV2ykQvH2c10jMrG9zkORIeWUtJxw11OOym1mf5yDJgYhgaVWtN2uZWb/gIMmByjfq\n2LZnP6dM8GYtM+v7chokkmZLWimpQtKNGaafIWmxpAZJl6S1ny1pSdptr6S5ybT7JK1JmzYjl+/h\nUCzxjnYz60dytgFfUiFwB/B+oApYKGl+RKxI67YeuBr4Uvq8EbEAmJEsZyRQATyZ1uXLEfFwrmrP\nVnllLQOLCjjx6KH5LsXMLOdyuSd4FlAREasBJD0EXAQcCJKIWJtMa2pjOZcAj0fEntyV2rXKK2t5\n+7jhDCj0lkMz6/ty+Us3DqhMe16VtHXW5cDPW7R9XdJSSd+VNDDTTJKulVQmqaympuYQXvbQNDQ2\n8VL1dm/WMrN+I5dBkmnwjejUAqQxwDTgibTmm4ApwGnASOCGTPNGxN0RURoRpaNHj+7My2bl1c27\n2Lu/iene0W5m/UQug6QKmJD2fDxQ3cllXAY8GhH7mxsiYmOk7APuJbUJrcco9452M+tnchkkC4HJ\nko6RVExqE9X8Ti7jClps1krWUlBquMG5wEtdUGuXKa+sZfjgAbztiMPyXYqZWbfIWZBERANwPanN\nUi8Dv4yI5ZJulTQHQNJpkqqAS4G7JC1vnl/SJFJrNM+2WPSDkpYBy4BRwD/n6j0ciiWVtUyfUOJh\ndc2s38jp9Tsi4jHgsRZtt6Q9Xkhqk1emedeSYed8RJzTtVV2nT31Dby6eSfnT/X4I2bWf/j41C60\nvHoHTQHTJ3j/iJn1Hw6SLlRemdrRfop3tJtZP+Ig6UJLKmsZVzKY0UMzntpiZtYnOUi6UHlVrc8f\nMbN+x0HSRV7ftY/KN+p8/oiZ9TsOki6ydMN2wDvazaz/cZB0kfLKWgoE0zy0rpn1Mw6SLlJeWcvx\nRw7x0Lpm1u84SLpARFBe5Sv+mln/5CDpAlXb6nhjd733j5hZv+Qg6QLNV/yd4SAxs37IQdIFyitr\nKfbQumbWTzlIukB55XbePnaYh9Y1s37Jv3xZamhsYtmG7b6+lpn1Ww6SLK3asou6/Y3eP2Jm/ZaD\nJEtLm4fWdZCYWT/lIMnSksrtDBtUxCQPrWtm/ZSDJEvlHlrXzPo5B0kW6uobWbl5p89oN7N+zUGS\nheXV22lsCu8fMbN+LadBImm2pJWSKiTdmGH6GZIWS2qQdEmLaY2SliS3+Wntx0h6QdIqSb+QVJzL\n99CWJcnQutPH+4q/ZtZ/5SxIJBUCdwAXAFOBKyRNbdFtPXA18LMMi6iLiBnJbU5a+23AdyNiMrAN\nuKbLi++gpVXbGTt8EEcOG5SvEszM8i6XaySzgIqIWB0R9cBDwEXpHSJibUQsBZo6skCl9mifAzyc\nNP0EmNt1JXdOamhdb9Yys/4tl0EyDqhMe16VtHXUIEllkp6X1BwWRwC1EdHQ3jIlXZvMX1ZTU9PZ\n2tu1bXc9617f4yAxs34vl6MwZToeNjox/8SIqJZ0LPAHScuAHR1dZkTcDdwNUFpa2pnX7ZDmK/6e\n4v0jZtbP5XKNpAqYkPZ8PFDd0Zkjojq5Xw08A8wEtgIlkpoDsFPL7ErllduRh9Y1M8tpkCwEJidH\nWRUDlwPz25kHAEkjJA1MHo8C3gOsiIgAFgDNR3hdBfymyyvvgKVVtRw/eghDBw3Ix8ubmfUYOQuS\nZD/G9cATwMvALyNiuaRbJc0BkHSapCrgUuAuScuT2U8CyiSVkwqOb0XEimTaDcAXJVWQ2mfy41y9\nh9akhtb1jnYzM8jtPhIi4jHgsRZtt6Q9Xkhq81TL+f4KTGtlmatJHRGWNxtq69i6y0PrmpmBz2w/\nJOWV2wGY4UujmJk5SA5FeVUtxYUeWtfMDBwkh6S8spapY4dRXOSPz8zMv4Sd1NgULNuw3SMimpkl\nHCSdVLFlF3vqG5k+weePmJmBg6TTyg9c8ddrJGZm4CDptCVVtQwdVMSkIw7PdylmZj2Cg6STllbV\nMn18CQUFHlrXzAwcJJ2yd38jr2zc6f0jZmZpHCSdsLx6Bw1N4f0jZmZpHCSd0Lyj3Yf+mpm9yUHS\nCeVVtYzx0LpmZgdxkHTC0qou9LRrAAAJgklEQVTtHsjKzKwFB0kH1e6pZ83W3b7ir5lZCw6SDlpa\n5Sv+mpll4iDpoPLKWiR4uzdtmZkdxEHSQeVVtRw3egjDPLSumdlBHCQdEBEsqdzu80fMzDJwkHTA\nxu172bprn89oNzPLIKdBImm2pJWSKiTdmGH6GZIWS2qQdEla+wxJz0laLmmppI+lTbtP0hpJS5Lb\njFy+B/AVf83M2lKUqwVLKgTuAN4PVAELJc2PiBVp3dYDVwNfajH7HuCTEbFK0lhgkaQnIqI2mf7l\niHg4V7W3tCQZWnfKGA+ta2bWUs6CBJgFVETEagBJDwEXAQeCJCLWJtOa0meMiFfTHldL2gKMBmrJ\ng/LKWk4aO4yBRYX5eHkzsx4tl5u2xgGVac+rkrZOkTQLKAZeS2v+erLJ67uSBrYy37WSyiSV1dTU\ndPZlD2hsCpZVbWeGD/s1M8sol0GSacCO6NQCpDHAT4FPRUTzWstNwBTgNGAkcEOmeSPi7ogojYjS\n0aNHd+ZlD7K6Zhe76xs5xftHzMwyymWQVAET0p6PB6o7OrOkYcDvgK9GxPPN7RGxMVL2AfeS2oSW\nM0uad7T70ihmZhnlMkgWApMlHSOpGLgcmN+RGZP+jwL3R8S8FtPGJPcC5gIvdWnVLZRX1TJ0YBHH\njvLQumZmmeQsSCKiAbgeeAJ4GfhlRCyXdKukOQCSTpNUBVwK3CVpeTL7ZcAZwNUZDvN9UNIyYBkw\nCvjnXL0HgPLK7ZwyYbiH1jUza0Uuj9oiIh4DHmvRdkva44WkNnm1nO8B4IFWlnlOF5fZqr37G3l5\n4w6uPePY7npJM7Nex2e2t+HljcnQut4/YmbWKgdJG3xGu5lZ+xwkbSiv2s5RwwZy9HAPrWtm1pqc\n7iPp7SYfNcQhYmbWDgdJGz531vH5LsHMrMfzpi0zM8uKg8TMzLLiIDEzs6w4SMzMLCsOEjMzy4qD\nxMzMsuIgMTOzrDhIzMwsK4ro1KCFvZKkGmBdvutoxShga76LaIPry47ry47ry0629b0tItodYrZf\nBElPJqksIkrzXUdrXF92XF92XF92uqs+b9oyM7OsOEjMzCwrDpL8uzvfBbTD9WXH9WXH9WWnW+rz\nPhIzM8uK10jMzCwrDhIzM8uKg6QbSJogaYGklyUtl/SFDH3OkrRd0pLkdks317hW0rLktcsyTJek\nH0iqkLRU0qndWNuJaZ/LEkk7JP3PFn269fOTdI+kLZJeSmsbKekpSauS+xGtzHtV0meVpKu6sb7b\nJb2S/Ps9KqmklXnb/C7ksL6vSdqQ9m94YSvzzpa0Mvku3tiN9f0irba1kpa0Mm93fH4Zf1Py9h2M\nCN9yfAPGAKcmj4cCrwJTW/Q5C/htHmtcC4xqY/qFwOOAgHcCL+SpzkJgE6kTpfL2+QFnAKcCL6W1\nfRu4MXl8I3BbhvlGAquT+xHJ4xHdVN/5QFHy+LZM9XXku5DD+r4GfKkD//6vAccCxUB5y/9Luaqv\nxfTvALfk8fPL+JuSr++g10i6QURsjIjFyeOdwMvAuPxW1WkXAfdHyvNAiaQxeajjXOC1iMjrlQoi\n4o/AGy2aLwJ+kjz+CTA3w6wfAJ6KiDciYhvwFDC7O+qLiCcjoiF5+jwwvqtft6Na+fw6YhZQERGr\nI6IeeIjU596l2qpPkoDLgJ939et2VBu/KXn5DjpIupmkScBM4IUMk98lqVzS45JO7tbCIIAnJS2S\ndG2G6eOAyrTnVeQnDC+n9f/A+fz8AI6KiI2Q+o8OHJmhT0/5HD9Nag0zk/a+C7l0fbLp7Z5WNsv0\nhM/vfcDmiFjVyvRu/fxa/Kbk5TvoIOlGkoYAvwL+Z0TsaDF5ManNNdOBfwN+3c3lvSciTgUuAD4v\n6YwW05Vhnm49dlxSMTAHmJdhcr4/v47qCZ/jzUAD8GArXdr7LuTKncBxwAxgI6nNRy3l/fMDrqDt\ntZFu+/za+U1pdbYMbVl9hg6SbiJpAKl/8Acj4pGW0yNiR0TsSh4/BgyQNKq76ouI6uR+C/AoqU0I\n6aqACWnPxwPV3VPdARcAiyNic8sJ+f78EpubN/cl91sy9Mnr55jsWP0Q8IlINpi31IHvQk5ExOaI\naIyIJuD/tfK6+f78ioCLgV+01qe7Pr9WflPy8h10kHSDZJvqj4GXI+JfW+lzdNIPSbNI/du83k31\nHS5paPNjUjtlX2rRbT7wyeTorXcC25tXobtRq38J5vPzSzMfaD4C5irgNxn6PAGcL2lEsunm/KQt\n5yTNBm4A5kTEnlb6dOS7kKv60ve5faSV110ITJZ0TLKGejmpz727nAe8EhFVmSZ21+fXxm9Kfr6D\nuTyywLcDR0m8l9Sq41JgSXK7ELgOuC7pcz2wnNRRKM8D7+7G+o5NXrc8qeHmpD29PgF3kDpiZhlQ\n2s2f4WGkgmF4WlvePj9SgbYR2E/qL7xrgCOAp4FVyf3IpG8p8KO0eT8NVCS3T3VjfRWkto03fwf/\nI+k7Fnisre9CN9X30+S7tZTUD+KYlvUlzy8kdZTSa91ZX9J+X/N3Lq1vPj6/1n5T8vId9CVSzMws\nK960ZWZmWXGQmJlZVhwkZmaWFQeJmZllxUFiZmZZcZCYdaHkCrZfOoT5ZqRf7fZQl2OWDw4Ss55h\nBqnzAMx6HQeJWZYk3ZyMj/F74MSk7ThJ/5VcuO9PkqYk7fdJ+o+k7VVJH0rO0L4V+FgyhsXHkkVP\nlfSMpNWS/j6Z/3BJv0suTvlSWl+zvCnKdwFmvZmkd5C6TMdMUv+fFgOLgLtJnQG9StLpwA+Bc5LZ\nJgFnkrpA4QLgeOAWUlcLuD5Z7teAKcDZpMabWCnpTlKX+66OiA8m/Ybn/l2atc1BYpad9wGPRnLt\nKknzgUHAu4F5yeW/AAamzfPLSF2YcJWk1aQCI5PfRcQ+YJ+kLcBRpC4h8i+SbiM1kNefuvwdmXWS\ng8Qsey2vM1QA1EbEjA72b+06RfvSHjeSGt3w1WQt6ELgm5KejIhbO12xWRfyPhKz7PwR+IikwclV\nXz8M7AHWSLoUDox3Pz1tnkslFUg6jtRF/lYCO0ltwmqTpLHAnoh4APgXUsPBmuWV10jMshARiyX9\ngtTVV9cBzZuaPgHcKemrwABSQ8KWJ9NWAs+S2lR1XUTslbQAuFHSEuCbbbzkNOB2SU2krkz72a5+\nT2ad5av/mnUjSfeR2rfxcL5rMesq3rRlZmZZ8RqJmZllxWskZmaWFQeJmZllxUFiZmZZcZCYmVlW\nHCRmZpaV/w9yKmptK+VzOwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1eb1c435588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maximum R^2:  0.289902005832\n",
      "Best depth:  10\n",
      "Pearson score = 0.579968397137\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "depths = [1,2,3,4,5,6,8,10,15,20]\n",
    "r2 = []\n",
    "max_r2, best_depth = 0, 0\n",
    "for depth in depths:\n",
    "    model = RandomForestRegressor(n_estimators=1000, max_depth=depth, random_state=seed)\n",
    "    model.fit(X_train, y_train) # Train the model using the training sets\n",
    "    r = model.score(X_test, np.array(y_test))\n",
    "    if r > max_r2: max_r2, best_depth = r, depth\n",
    "    r2.append(r)\n",
    "\n",
    "plt.plot(depths,r2)\n",
    "plt.xlabel('depths')\n",
    "plt.ylabel('R^2')\n",
    "plt.title('Random Forest')\n",
    "plt.show()\n",
    "\n",
    "print('maximum R^2: ', max_r2)\n",
    "print('Best depth: ', best_depth)\n",
    "\n",
    "best_random_forest_model = RandomForestRegressor(n_estimators=1000, max_depth=best_depth, random_state=seed)\n",
    "best_random_forest_model.fit(X_train, y_train) # Train the model using the training sets\n",
    "y_pred = best_random_forest_model.predict(X_test) # Make predictions using the testing set\n",
    "pearson = pearsonr(y_test, y_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SRV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VOW9x/HPLxuBsCUQ9iWsyr4F\nFPcFFVzAVq1ArdTW7Spqr7b3amu12tqr9ta61mpRq7aKS0vl4r6LokBYlB3CHlDWENaELL/7x0zo\nmAYmCZnMTPJ9v17zypwz58z8job55jnPeZ5j7o6IiMiRJES7ABERiX0KCxERCUthISIiYSksREQk\nLIWFiIiEpbAQEZGwFBYiIhKWwkLkKJjZSWY2y8wKzGynmX1mZieb2T4za1bJ9gvMbLKZZZmZm9ne\n4GOdmd0ajWMQqQqFhUgNmVlzYAbwCJABdATuAgqAPOCiCtv3B/oCL4asbunuTYGLgV+a2Vl1ULpI\ntSksRGquN4C7v+jupe5+wN3fcfevgGeByytsfznwurvvqPhG7p4DLAEGR7pokZpQWIjU3Eqg1Mye\nNbMxZpYe8trzwMlm1gXAzBKAicBzlb2RmR0P9AdyI1yzSI0oLERqyN13AycBDvwZ2GZm082srbtv\nBD4GLgtufiaQCrxe4W22m9kB4HPgj8A/66R4kWpSWIgcBXdf5u4/dPdOBFoGHYAHgy+Hnor6AfCC\nuxdXeIvWQFPgp8BpQHLEixapAYWFSC1x9+XAXwiEBsA/gI5mdjrwXQ5zCirY3/F7oBC4rg5KFak2\nhYVIDZnZsWZ2i5l1Ci53BiYAXwC4+z7gVeAZYH2wE/tI7gX+y8xSI1i2SI0oLERqbg9wHDDbzPYR\nCInFwC0h2zwLdOUwrYoKXgfygatquU6Ro2a6+ZGIiISjloWIiISlsBARkbAUFiIiEpbCQkREwkqK\ndgG1pXXr1p6VlRXtMkRE4sq8efO2u3tmuO3qTVhkZWWRkxPuMnYREQllZuursp1OQ4mISFgKCxER\nCUthISIiYSksREQkLIWFiIiEpbAQEZGwFBYiIhKWwuIoFOwv5uWcjRSXlkW7FBGRiKo3g/Lqmrvz\n01e/5N2lW1iyqYC7xvUPv5OISJyKaMvCzEab2QozyzWzWyt5/VozW2RmC83sUzPrG1yfZWYHgusX\nmtmfIllnTfx9/ibeXbqFPu2b8+zn65k6Z0O0SxIRiZiIhYWZJQKPAWOAvsCE8jAI8YK7D3D3wcD9\nwAMhr61298HBx7WRqrMmNu06wF3TlzAiK4PXrj+Rk3u15pevLWbe+p3RLk1EJCIi2bIYAeS6+xp3\nPwhMBcaFbuDuu0MW04CYv21fWZnzs1e+pMyd/71kEClJCTw6YSgdWzbmmufn83XBgWiXKCJS6yIZ\nFh2BjSHLecF132Jm15vZagItixtDXupmZgvM7GMzO7myDzCzq80sx8xytm3bVpu1H9azn69j1uod\n3H5+X7q0agJAiybJ/PnybAqLS7nm+XkUFpfWSS0iInUlkmFhlaz7t5aDuz/m7j2A/wZuD67+Guji\n7kOAm4EXzKx5Jfs+6e7Z7p6dmRl2ht2jlrt1L/e+uZzTj8lk/PDO33qtV9tmPHjpYBZtKuC2fyxC\n9zYXkfokkmGRB4R+o3YCNh9h+6nAhQDuXuTuO4LP5wGrgd4RqrNKSkrLuOXlhTROSeS+iwZi9u9Z\nOKpvW245qzfTFmzipbkbK3kXEZH4FMmwmAv0MrNuZpYCjAemh25gZr1CFs8DVgXXZwY7yDGz7kAv\nYE0Eaw3rlXl5fJlXwK/H9adN89TDbnfdaT0Z2b0Vv3l9GXn5++uwQhGRyIlYWLh7CTAZeBtYBrzs\n7kvM7G4zGxvcbLKZLTGzhQRON00Krj8F+MrMvgReBa5196heavTm4m/o1jqNCwZ1OOJ2CQnG/RcP\nxN35r1e/oqxMp6NEJP5FdFCeu78BvFFh3R0hz286zH5/B/4eydqqY29RCV+s3sHlI7tWafvOGU24\n/fy+3PaPRfx19nouH5kV2QJFRCJM031UwaertnGwtIwz+7St8j7jh3fm1N6Z/M8by1m3fV8EqxMR\niTyFRRW8t2wrzVOTyM5Kr/I+Zsa9Fw0gKdH42atfUqrTUSISxxQWYZSWOR8u38ppx7QhObF6/7na\nt2jMry7ox9x1+Tzz2doIVSgiEnkKizAWbtzFjn0HObNPmxrt/92hHRnVpy33v72C1dv21nJ1IiJ1\nQ2ERxvvLtpCUYJzWu2ZhYWb89rv9aZSUwJ2vLdFgPRGJSwqLMN5ftpXhWRm0aJJc4/do0yyVW87q\nzae523lr8Te1WJ2ISN1QWBzBxp37WbFlT41PQYW67PiuHNuuGb+esZT9B0tqoToRkbqjsDiC95Zt\nAWBUNS6ZPZykxAR+fWF/NhcU8scPVx/1+4mI1CWFxRG8v2wrPTLTyGqdVivvNzwrg+8M6ciTn6xh\nrcZeiEgcUVgcxp7CYmav3VErrYpQt405lpSkBO76P3V2i0j8UFgcxicrt1Nc6tUatV0VbZqn8pNR\nvfhoxTbeXbqlVt9bRCRSFBaH8f6yLbRskszQLi1r/b0nnZBF77ZNuXvGUt0oSUTigsKiEqVlzocr\ntnL6MW1Iquao7apITkzgrrH9ycs/wOMfqbNbRGKfwqIS8zfkk7+/uFYumT2ckT1accGgDjz+8Wo2\n7NB9L0QktiksKvHmom9ITjRO6R3ZW7X+4tw+JCUYd89YGtHPERE5WgqLCopKSpm2II+z+7ajeWrN\nR21XRbsWqdx4Zi/eW7aFD5dvjehniYgcDYVFBW8v2UL+/mIuHd45/Ma14EcndqNHZhq/+r8l6uwW\nkZilsKhg2vw8OrZszEk9W9fJ56UkJfCrsf1Yv2M/U2ZG9TbjIiKHpbCoYGP+AQZ2akFCgtXZZ57c\nK5Mx/dvx6Ie55OWrs1tEYo/CooKCA8W0PIoZZmvq9vP7Yhi/mbGszj9bRCQchUUId6fgQDHNG9d9\nWHRs2ZjJZ/TkrSXf8MnKbXX++SIiRxLRsDCz0Wa2wsxyzezWSl6/1swWmdlCM/vUzPqGvHZbcL8V\nZnZOJOssV1hcxsGSMlpEISwArjy5G1mtmvCr6Us4WFIWlRpERCoTsbAws0TgMWAM0BeYEBoGQS+4\n+wB3HwzcDzwQ3LcvMB7oB4wG/hh8v4gqOFAMQMvGKZH+qEo1SkrkzrH9WLN9H099qnt2i0jsiGTL\nYgSQ6+5r3P0gMBUYF7qBu+8OWUwDyqdhHQdMdfcid18L5AbfL6LKwyJaLQuA049pw1l92/LIB6v4\nuuBA1OoQEQkVybDoCGwMWc4LrvsWM7vezFYTaFncWJ19a9uu/QeB6IYFwB3n96W0zLnndXV2i0hs\niGRYVHbt6b/dwMHdH3P3HsB/A7dXZ18zu9rMcswsZ9u2o+8UPnQaKgpXQ4XqnNGE/zitBzO++ppZ\nq7dHtRYREYhsWOQBocOgOwGbj7D9VODC6uzr7k+6e7a7Z2dmHv08Trti4DRUuWtP7UHnjMbc+doS\nikvV2S0i0RXJsJgL9DKzbmaWQqDDenroBmbWK2TxPGBV8Pl0YLyZNTKzbkAvYE4EawVgd3lYRLll\nAZCanMid5/dj1da9/OWzddEuR0QauKRIvbG7l5jZZOBtIBF42t2XmNndQI67Twcmm9kooBjIByYF\n911iZi8DS4ES4Hp3j/jESQUHikkwaJoSsf8s1XJmnzacfkwmD763knGDO9CmeWq0SxKRBiqi4yzc\n/Q137+3uPdz9nuC6O4JBgbvf5O793H2wu5/u7ktC9r0nuN8x7v5mJOsst2t/YEBeXU71cSRmxp0X\n9KO41Pn5tEW6Z7eIRI1GcIcoOFBMyxjorwiV1TqNW8ccy3vLtjJlpsZeiEh0KCxCFBwojonO7Yqu\nODGL0f3acd9by5m3Pj/a5YhIA6SwCLHrQDEtmkRn9PaRmBn3XTyQ9i1TmfzCfHbuOxjtkkSkgVFY\nhNgdoy0LCFzO+8eJw9ix9yA3v7yQsjL1X4hI3VFYhNi1/yAtGsfGlVCVGdCpBb88vw8frdjGnz5Z\nHe1yRKQBUVgEuTu7C0uiNolgVV12fFfOH9ie37+zkjlrd0a7HBFpIBQWQXuLSigtc5rHcMsCAv0X\n//PdAXTJaMINL85n+96iaJckIg2AwiJob1EJAM1SY7PPIlSz1GQenTiE/P3F/OdL6r8QkchTWATt\nLQyERdNGsd2yKNevQwt+dUE/Zq7azmMf5ka7HBGp5xQWQeUti6ap8REWABNGdGbc4A784b2Vmp1W\nRCJKYRF06DRUnLQsINB/8dvvDCCrdRo3TV3I1j2F0S5JROophUXQodNQcdSyAEhrlMQfvz+UPYXF\n3PTiQkrVfyEiEaCwCNoTbFmkxciMs9VxbLvm3D2uP5+v2cH9by/XhIMiUuvi75sxQspbFs3irGVR\n7nvZnVmwYRdPfLwGHG4dcyxmsTF7rojEv/j8ZoyA8j6LtDjqs6jongv7k5gAT3yyhr1FJfx6XP+Y\nmW5dROJb/H4z1rK9RSWkJieQnBi/Z+YSEoxfj+tPWkoST3yyhgMHS7n/4oEkxfExiUhsUFgE7Sks\noWmj2B+QF46ZceuYY0lrlMQD767kQHEpD40fQkqSAkNEak7fIEH7ikritr+iIjPjxjN7cft5fXhz\n8Tdc/XwOhcURvyutiNRjCougvUUlcTN6u6quPLk7v/3OAD5euY1JT8851C8jIlJdCougvYUlpDVK\njHYZtW7icV34w/cGk7M+n8umzKZgf3G0SxKROKSwCNpTVD/6LCpz4ZCO/PH7Q1m6eTfj//yFZqoV\nkWpTWATtLSquN30WlTmnXzumTMpm7fa9fO+Jz/m64EC0SxKROBLRsDCz0Wa2wsxyzezWSl6/2cyW\nmtlXZva+mXUNea3UzBYGH9MjWScETkPVtz6Lik7pnclzPzqOrbuLuORPn7Nhx/5olyQicSJiYWFm\nicBjwBigLzDBzPpW2GwBkO3uA4FXgftDXjvg7oODj7GRqhMCd8nbW1QSd/NC1cSIbhn87crj2FtU\nwiVPzCJ3655olyQicSCSLYsRQK67r3H3g8BUYFzoBu7+obuX/3n7BdApgvUcVlFJGcWlXu9bFuUG\ndW7J1KuPp7QMvvfEFyzZXBDtkkQkxkUyLDoCG0OW84LrDufHwJshy6lmlmNmX5jZhZXtYGZXB7fJ\n2bZtW40LLR+D0Di5/l0NdTjHtmvOy9ccT2pSAhOe/IL5G/KjXZKIxLBIhkVlkxJVOh2qmV0GZAO/\nC1ndxd2zgYnAg2bW49/ezP1Jd8929+zMzMwaF1oSnNY7ObFhzaPUPbMpL187kvS0FC6bMls3UBKR\nw4pkWOQBnUOWOwGbK25kZqOAXwBj3f3QNZ3uvjn4cw3wETAkUoWW3wMiMaHhXRzWKb0Jr1wzkk7p\njbnimbl8uHxrtEsSkRgUyW/HuUAvM+tmZinAeOBbVzWZ2RDgCQJBsTVkfbqZNQo+bw2cCCyNVKHl\nLYukBjpDa5vmqUy9eiS92jbl6udzeGPR19EuSURiTMTCwt1LgMnA28Ay4GV3X2Jmd5tZ+dVNvwOa\nAq9UuES2D5BjZl8CHwL3unvEwqK0tLxl0TDDAiAjLYUXrjqeQZ1aMvmF+bw6Ly/aJYlIDIno5T/u\n/gbwRoV1d4Q8H3WY/WYBAyJZW6iSsjIAkhpYn0VFzVOTee7HI7j6uXn89JUvOXCwhB+MzIp2WSIS\nAxreSfpKlPdZJOjOcjRJSWLKpGxG9WnLL19bwp8+Xh3tkkQkBigsUJ9FRanJiTx+2VAuGNSBe99c\nzgPvrNB9vUUauIYxCi2Mf10NpbAol5yYwIOXDqZJciIPf5DLvoOl3H5eH93XW6SBUljwr7Bo6H0W\nFSUmGP/z3QE0TknkqU/Xsv9gCb+5cIBCVaQBUljwr9NQDXGcRTgJCcadF/SlaaMkHv0wl/0HS/nf\nSwbF9b3KRaT6FBaEtCz0F3OlzIyfnnMMaY2SuO+t5ew/WMpjE4fqvt4iDYj+tfOvS2d1euXI/uO0\nHtw1th/vLt3Cf7608FDIikj9p5YF6uCujkknZFFcWsZvXl9Gk5RE7rtoIAn67yZS7yksCO2z0Jde\nVVx5cnd2F5bw8PuraJqaxB3n99VVUiL1nMICKFOfRbX956he7C0s4enP1tIsNZmbz+od7ZJEJIIU\nFqhlURNmxi/P78PeomIefn8VzRolcdUp3aNdlohEiMKC0Kuh1N9fHWbG/3x3IPuKSrnnjWU0TU1i\nwogu0S5LRCJAYYFaFkcjMcH4w6WD2XewhJ9PW0RaoyTGDuoQ7bJEpJbpT2mgtHzWWYVFjaQkJfD4\n94cxPCuDm19ayPvLtkS7JBGpZQoLoET3szhqjVMSeWpSNn07NOc//jZft2gVqWcUFmicRW1plprM\ns1eMoGtGE656NocFG/KjXZKI1BKFBVDqunS2tqSnpfDXK4+jVdNG/PCZuSz/Zne0SxKRWqCwQC2L\n2ta2eSp/u/I4GicnctmUOazdvi/aJYnIUVJY8K8+C106W3s6ZzThr1eOoMydy6bMZvOuA9EuSUSO\ngr4dCWlZ6H4Wtapnm2Y896MR7D5QzGVTZrN9b1G0SxKRGlJYoNuqRlL/ji14+orhbC44wA+emkPB\n/uJolyQiNXDEsDCzRDO7xsx+bWYnVnjt9nBvbmajzWyFmeWa2a2VvH6zmS01s6/M7H0z6xry2iQz\nWxV8TKrOQVVX+TiLBE2GFxHDszJ44gfZ5G7dwxV/mcO+opJolyQi1RSuZfEEcCqwA3jYzB4Iee27\nR9rRzBKBx4AxQF9ggpn1rbDZAiDb3QcCrwL3B/fNAO4EjgNGAHeaWXqVjqgG1LKIvFN7Z/LIhCEs\n3LiLa56fR2FxabRLEpFqCBcWI9x9ors/SOCLu6mZ/cPMGgHhvllHALnuvsbdDwJTgXGhG7j7h+6+\nP7j4BdAp+Pwc4F133+nu+cC7wOiqH1b1lJU5Zui+DBE2un977r94EJ/mbueGFxdQXFoW7ZJEpIrC\nhUVK+RN3L3H3q4GFwAdA0zD7dgQ2hiznBdcdzo+BN6uzr5ldbWY5Zpazbdu2MOUcXkmZq1VRRy4e\n1unQ3fb+69WvDk0PLyKxLVxY5JjZt/6id/e7gWeArDD7VvbtW+k3g5ldBmQDv6vOvu7+pLtnu3t2\nZmZmmHIOr7TMNcaiDk06IYufnt2baQs2ccf0xbgrMERi3RFnnXX3yw6zfgowJcx75wGdQ5Y7AZsr\nbmRmo4BfAKe6e1HIvqdV2PejMJ9XY4GWhS4Mq0vXn96TPUUlPPHxGpqlJvPfo4+NdkkicgRVmqLc\nzBLdvbo9knOBXmbWDdgEjAcmVnjfIQQ60Ue7+9aQl94GfhvSqX02cFs1P7/K1LKoe2bGraOPZW9h\nCY9/tJqmjZK4/vSe0S5LRA4jbFiYWTPgReD86ryxu5eY2WQCX/yJwNPuvsTM7gZy3H06gdNOTYFX\ngvdw3uDuY919p5n9mkDgANzt7jur8/nVUVJWprCIAjPj1+P6s7eohN+9vYJmqUlcPjIr2mWJSCWO\nGBZm1h74J3BPTd7c3d8A3qiw7o6Q56OOsO/TwNM1+dzqUssiehISjP+9ZBD7ikq547UlpKUkcdGw\nTuF3FJE6Fe5E/Uzg3mAroN4q1dVQUZWcmMCjE4dwYs9W/OzVL3lr8TfRLklEKggXFvkc+XLXeqFE\nLYuoS01O5MkfZDOoc0tufHEBM1fV/FJoEal94cLiNGCMmV1fB7VEjVoWsSGtURJ/+eEIumemcfVz\n85iVq7vticSKI4aFu+8DxgJD6qac6FDLIna0aJLM8z8+jo7pjbnsqdk8/tFqjcMQiQFhBxe4e6m7\nX1kXxURLaanGWcSSzGaNmHbdCYzu34773lrOVc/No+CAZqsViaYafUMGZ6P9fm0XEy0lZa55oWJM\ns9RkHps4lDvO78tHK7ZywSOfsnhTQbTLEmmwwk1R3tzMbjOzR83sbAu4AVgDfK9uSoy80rIy9VnE\nIDPjRyd146VrjudgSRnffXwWL83dEO2yRBqkcC2L54FjgEXAlcA7wMXAOHcfd6Qd40mp6/7bsWxY\n1wxm3HgSI7Iy+O+/L+Jnr3zJgYOa4lykLoUbwd3d3QcAmNkUYDvQxd33RLyyOqSWRexr3bQRz/5o\nBA+9t5KHP8hl0aYCHr9sGN1ap0W7NJEGIVzL4lCvYnBuqLX1LSgASkp1NVQ8SEwwbj77GJ65Yjjf\n7C5k7COf8tbir6NdlkiDEC4sBpnZ7uBjDzCw/LmZ7a6LAutCaZmTlKiwiBenH9OGGTecRPfMNK79\n63zueX2pbqQkEmHhxlkkunvz4KOZuyeFPG9eV0VGWmCchS6djSed0pvw8rUjuXxkV/48cy0T//wF\nW3YXRrsskXpL35AEJxJUwyLuNEpK5O5x/Xlo/GAWb9rNeQ/PZNZqjfoWiQSFBWpZxLtxgzsyffKJ\ntGiczGVTZvPYh7m6XatILdM3JFCmuaHiXq+2zXht8kmcO6A9v3t7BVc9l0PBfo36FqktCguCNz/S\neai417RREo9MGMJdY/vxyaptnPfITBbladS3SG1QWKBZZ+sTM2PSCVm8dM1Iysqcix6fxQuzN2gy\nQpGjpLBAs87WR0O7pDPjxpM5vkcrfj5tEbe8rFHfIkdDYYFaFvVVRloKz/xwOD8Z1YtpCzdx4WOf\nsWbb3miXJRKXFBaoZVGfJSYYPxnVm79cMYKtewoZ++hnvLFIo75FqkthQXCchcKiXju1dyav33gy\nvdo25bq/zefu/9Oob5HqiGhYmNloM1thZrlmdmslr59iZvPNrMTMLq7wWqmZLQw+pkeyzsBpKOVm\nfdehZWNeunokPzwhi6c/W8v4J7/g64ID0S5LJC5E7BvSzBKBx4AxQF9ggpn1rbDZBuCHwAuVvMUB\ndx8cfIyNVJ2glkVDkpKUwK/G9uORCUNY/vVuznv4U95buiXaZYnEvEj+OT0CyHX3Ne5+EJgKfOse\nGO6+zt2/AqJ6PqBEU5Q3OBcM6sBrk0+iXfNUrnwuhzteW0xhsa6WEjmcSIZFR2BjyHJecF1VpZpZ\njpl9YWYXVraBmV0d3CZn27ZtNS5ULYuGqWebpky7/gSuPKkbz32+nrGPfsryb+rNZMoitSqSYVHZ\nt291RkZ1cfdsYCLwoJn1+Lc3c3/S3bPdPTszM7OmdVKiS2cbrEZJidx+fl+e/dEIdu4rZuyjn/Hs\nrHUaxCdSQSTDIg/oHLLcCdhc1Z3dfXPw5xrgI2BIbRZXrqzMcYcEhUWDdmrvTN76ycmc1LM1d05f\nwo+fzWH73qJolyUSMyIZFnOBXmbWzcxSgPFAla5qMrN0M2sUfN4aOBFYGokiS4Kzk6plIa2bNuKp\nSdncNbYfn+ZuZ/SDM/l4Zc1Pb4rUJxELC3cvASYDbwPLgJfdfYmZ3W1mYwHMbLiZ5QGXAE+Y2ZLg\n7n2AHDP7EvgQuNfdIxIWZcHTDZqiXOBfc0tNn3wiGWnJTHp6Dr+ZsZSiEnV+S8OWFMk3d/c3gDcq\nrLsj5PlcAqenKu43CxgQydrKqWUhlTm2XXOmTz6J376xjCmfrmXW6h08PGEwPds0i3ZpIlHR4P+c\nLi0tb1koLOTbUpMDd+Kbcnk23+wu5PxHPtUMttJgNfiwKCkLDPFI0v0s5DBG9W3LWzedzPCsDH4+\nbRHX/nUe+fsORrsskTrV4MOicUoit5/Xh2Fd06NdisSwNs1TefaKEfzi3D58sHwrYx6ayaxc3e9b\nGo4GHxZNUpK48uTu9OvQItqlSIxLSDCuOqU70647kSYpiXz/qdnc99ZyTUgoDUKDDwuR6urfsQUz\nbjyJS7M78/hHq7n48Vms274v2mWJRJTCQqQGmqQkce9FA3n8+0NZt2M/5z48k1fn5anzW+othYXI\nURgzoD1v3nQyAzq24KevfMkNLy6g4EBxtMsSqXUKC5Gj1KFlY1646nh+ds4xvLn4G859aCZz1+2M\ndlkitUphIVILEhOM60/vyavXjiQxwbj0ic954N2VlKjzW+oJhYVILRrSJZ3XbzyJC4d05OH3V3Hp\nk1+wcef+aJclctQUFiK1rFlqMg98bzAPjR/Mym/2cO5DM5k6ZwNlZer8lvilsBCJkHGDO/LGTSfT\np0Nzbv3HIsY/+QW5W/dGuyyRGlFYiERQ54wmTL3qeO67aAArtgRaGX94d6VmsZW4o7AQibCEBOPS\n4V147+ZTGd2/HQ+9v4pzH5rJ7DU7ol2aSJUpLETqSGazRjw8YQh/uWI4RSVlXPrkF9z6968o2K9x\nGRL7FBYidey0Y9rwzn+ewjWndOeVeXmc+cBHTP9ys0Z/S0xTWIhEQZOUJG47tw/TJ59Ih5aNufHF\nBfzwmbm6zFZilsJCJIr6dWjBtOtO5I7z+zJ33U7O/sMnPPnJag3mk5ijsBCJssQE40cndePdm0/l\nxJ6t+O0byxn32Gd8lbcr2qWJHKKwEIkRHVs25s+XZ/P494eybU8RFz72GXf/31L2FZVEuzQRhYVI\nLDEzxgxoz3u3nMrE47rwzKy1nPXAx7y3dEu0S5MGLqJhYWajzWyFmeWa2a2VvH6Kmc03sxIzu7jC\na5PMbFXwMSmSdYrEmuapyfzmwgG8eu1ImqYmceVzOVz3t3ls3V0Y7dKkgYpYWJhZIvAYMAboC0ww\ns74VNtsA/BB4ocK+GcCdwHHACOBOM9NNsqXBGdY1gxk3nMzPzjmG95Zt5czff8xfv1iveaakzkWy\nZTECyHX3Ne5+EJgKjAvdwN3XuftXQMVLP84B3nX3ne6eD7wLjI5grSIxKyUpgetP78nbPzmFAZ1a\ncPs/F3PJE5+zcsueaJcmDUgkw6IjsDFkOS+4rtb2NbOrzSzHzHK2bdtW40JF4kG31mn87crj+P0l\ng1izbS/nPTyT37+zgsJizTMlkRfJsLBK1lW17Vylfd39SXfPdvfszMzMahUnEo/MjIuGdeL9W07j\ngkEdeOSDXMY8NJNZq7dHuzSp5yIZFnlA55DlTsDmOthXpN7LSEvhge8N5q8/Po4ydyb+eTY3vLiA\n3K06NSWREcmwmAv0MrNuZpbVp9ngAAAM7UlEQVQCjAemV3Hft4GzzSw92LF9dnCdiIQ4qVdr3v7J\nKUw+vSfvLd3CWX/4hOv+No+lm3dHuzSpZyySk5eZ2bnAg0Ai8LS732NmdwM57j7dzIYD04B0oBD4\nxt37Bff9EfDz4Fvd4+7PHOmzsrOzPScnJ1KHIhLzduwt4unP1vLsrPXsLSphVJ+23HhmTwZ2ahnt\n0iSGmdk8d88Ou119melSYSESULC/mGdmreXpT9eyu7CEU3tncsMZPcnOyoh2aRKDFBYiDdyewmKe\n/2I9U2auZee+g4zs3oobzuzJyO6tMKvsGhJpiBQWIgLA/oMlvDB7A098soZte4rI7prO5DN6cmrv\nTIWGKCxE5NsKi0t5OWcjf/poNZsLChnUqQWTz+jFqD5tFBoNmMJCRCp1sKSMv8/P448f5bJx5wGO\nbdeMG87oxZj+7UhIUGg0NAoLETmi4tIypi/czGMf5rJm+z56tmnK5NN7cv7A9iQlakLqhkJhISJV\nUlrmvL7oax79YBUrt+wlq1UTrju9J98Z0pFkhUa9p7AQkWopK3PeWbqFRz5YxZLNu+nYsjH/cVoP\nLsnuRKOkxGiXJxGisBCRGnF3PlyxlYffz2Xhxl20bd6Ia07pwYQRXWicotCobxQWInJU3J3Pcnfw\n8AermLN2J62bpnDlyd0ZP7wzLZukRLs8qSUKCxGpNbPX7OCRD3L5NHc7KUkJnDegPRNGdGF4Vrou\nu41zVQ2LpLooRkTi23HdW3Fc91Ys3bybF+ds4J8LNjFtwSZ6tmnKhBFduGhoR7U26jm1LESk2vYf\nLGHGV1/zwuwNLNy4i5SkBM7t346Jx3VVayPO6DSUiNSJpZt3M3XuBqbN38SeohJ6ZKYFWxudSE9T\nayPWKSxEpE6VtzZenLOBBRv+1dqYMKILI7plqLURoxQWIhI1am3ED4WFiESdWhuxT2EhIjFl2de7\nmTpnA/9YsIk9hWptxAqFhYjEpAMHS5nx1WZeKG9tJCYwZkA7Jqq1ERUKCxGJeRVbG90z05g4oguj\n+7ejU3qTaJfXICgsRCRulLc2XpyzgfkbdgHQoUUq2VkZDO+WwfCsdHq3aab7bUSAwkJE4lLu1j18\nlruDuet2MnfdTrbsLgKgeWpSIDyyAuExoFMLzYZbC2Jiug8zGw08BCQCU9z93gqvNwKeA4YBO4BL\n3X2dmWUBy4AVwU2/cPdrI1mriMSGnm2a0bNNMyadkIW7s3HngUPBMXfdTj5YvhWAlKQEBndqyfBu\n6WRnZTCsazrNU5OjXH39FbGWhZklAiuBs4A8YC4wwd2XhmxzHTDQ3a81s/HAd9z90mBYzHD3/lX9\nPLUsRBqGHXuLyFmfz9y1O5m7Pp8lmwooKXPM4Nh2zRmRlR48dZVB2+ap0S435sVCy2IEkOvua4IF\nTQXGAUtDthkH/Cr4/FXgUdOlECJyBK2aNuKcfu04p187IDCWY+GGXcxZt5Ocdfm8Mi+PZz9fD0CX\njCZkZ6UzIiuD7KwMemSm6WqrGopkWHQENoYs5wHHHW4bdy8xswKgVfC1bma2ANgN3O7uMyt+gJld\nDVwN0KVLl9qtXkTiQpOUJE7o2ZoTerYGAvcWX/b1buasDYTHxyu28Y/5mwDISEshu2s6I7oFwqNv\n++akJOnWsVURybCoLL4rnvM63DZfA13cfYeZDQP+aWb93H33tzZ0fxJ4EgKnoWqhZhGJc8mJCQzs\n1JKBnVpy5cmBmzit2b6PnHU7mbM2n5z1O3ln6RYAGiUlMLBTC4Z2SWdo13SGdkkns1mjKB9BbIpk\nWOQBnUOWOwGbD7NNnpklAS2AnR7oSCkCcPd5ZrYa6A2oU0JEqsXM6JHZlB6ZTbl0eOAMxNbdheSs\nz2f++nzmbcjnmc/W8cQna4DAqathXdMZ2qUlQ7umc0zbZiQlqvURybCYC/Qys27AJmA8MLHCNtOB\nScDnwMXAB+7uZpZJIDRKzaw70AtYE8FaRaQBadM8lXMHtOfcAe0BKCwuZcnmAuav38W89fl8mrud\naQsCp66apCQyuHNLhnZJZ1jXdIZ0adkgb/QUsbAI9kFMBt4mcOns0+6+xMzuBnLcfTrwFPC8meUC\nOwkECsApwN1mVgKUAte6+85I1SoiDVtqciLDumYwrGsGVxE4dZWXf4D5G/7V+nj849WUlgXOdvfI\nTDsUHkO7ptMzs2m9HzCoQXkiIlWw/2AJX24sOBQg8zfkk7+/GIBmqUkM6ZLOsC7pDO3aksGdW9Is\nTsZ8xMKlsyIi9UaTlCRG9mjFyB6BCzbdnbXb9zF/Q+DU1fz1+Tz4/krcwQyOaduMoV3LAySdrFZN\n4vqyXbUsRERqye7CYhZu2MX8DfnMW5/Pwg272FNUAgQu2y3vNB/aJZ1BnVrSOCX605WoZSEiUsea\npyZzSu9MTumdCUBZmbNq695D4TF/Qz7vLQtMV5KUYPRp3/xQp/mwrul0bNk4ZlsfalmIiNSh/H0H\nWbAxEB7z1ufz5cYCDhSXAtCmWaPgZbuBU1f9OzaP+GSJalmIiMSg9LQUzji2LWcc2xaAktIyln+z\n51utjzcXfwNASmIC/Ts2/9aVV9Ga70otCxGRGLN1TyHz1+86dOXVV5sKOFhSBkDHlo0PDRoc1jWD\nY9s3I/koBg3qfhYiIvVEUUkpSzfvPtTymLc+/9B9PlKTExjVpy2PThxao/fWaSgRkXqiUVIiQ7qk\nM6RLOhC4bHdzQWFgwOD6fNIaRf6qKoWFiEicMTM6tmxMx5aNuWBQhzr5TM2OJSIiYSksREQkLIWF\niIiEpbAQEZGwFBYiIhKWwkJERMJSWIiISFgKCxERCaveTPdhZtuA9TXcvTWwvRbLiSYdS2yqL8dS\nX44DdCzlurp7ZriN6k1YHA0zy6nK3CjxQMcSm+rLsdSX4wAdS3XpNJSIiISlsBARkbAUFgFPRruA\nWqRjiU315Vjqy3GAjqVa1GchIiJhqWUhIiJhKSxERCSsBh8WZjbazFaYWa6Z3RrteqrDzJ42s61m\ntjhkXYaZvWtmq4I/06NZY1WYWWcz+9DMlpnZEjO7Kbg+Ho8l1czmmNmXwWO5K7i+m5nNDh7LS2aW\nEu1aq8rMEs1sgZnNCC7H5bGY2TozW2RmC80sJ7guHn/HWprZq2a2PPhvZmRdHEeDDgszSwQeA8YA\nfYEJZtY3ulVVy1+A0RXW3Qq87+69gPeDy7GuBLjF3fsAxwPXB/8/xOOxFAFnuPsgYDAw2syOB+4D\n/hA8lnzgx1GssbpuApaFLMfzsZzu7oNDxiTE4+/YQ8Bb7n4sMIjA/5vIH4e7N9gHMBJ4O2T5NuC2\naNdVzWPIAhaHLK8A2geftwdWRLvGGhzTa8BZ8X4sQBNgPnAcgdG1ScH13/q9i+UH0Cn45XMGMAOw\nOD6WdUDrCuvi6ncMaA6sJXhxUl0eR4NuWQAdgY0hy3nBdfGsrbt/DRD82SbK9VSLmWUBQ4DZxOmx\nBE/bLAS2Au8Cq4Fd7l4S3CSefs8eBP4LKAsutyJ+j8WBd8xsnpldHVwXb79j3YFtwDPBU4NTzCyN\nOjiOhh4WVsk6XUscJWbWFPg78BN33x3temrK3UvdfTCBv8pHAH0q26xuq6o+Mzsf2Oru80JXV7Jp\nzB9L0InuPpTAaefrzeyUaBdUA0nAUOBxdx8C7KOOTp019LDIAzqHLHcCNkepltqyxczaAwR/bo1y\nPVViZskEguJv7v6P4Oq4PJZy7r4L+IhAP0xLM0sKvhQvv2cnAmPNbB0wlcCpqAeJz2PB3TcHf24F\nphEI8nj7HcsD8tx9dnD5VQLhEfHjaOhhMRfoFby6IwUYD0yPck1HazowKfh8EoHz/zHNzAx4Cljm\n7g+EvBSPx5JpZi2DzxsDowh0QH4IXBzcLC6Oxd1vc/dO7p5F4N/GB+7+feLwWMwszcyalT8HzgYW\nE2e/Y+7+DbDRzI4JrjoTWEpdHEe0O2yi/QDOBVYSOK/8i2jXU83aXwS+BooJ/MXxYwLnlN8HVgV/\nZkS7ziocx0kETmV8BSwMPs6N02MZCCwIHsti4I7g+u7AHCAXeAVoFO1aq3lcpwEz4vVYgjV/GXws\nKf+3Hqe/Y4OBnODv2D+B9Lo4Dk33ISIiYTX001AiIlIFCgsREQlLYSEiImEpLEREJCyFhYiIhKWw\nEIkgM2tnZlPNbLWZLTWzN8ysd7TrEqkuhYVIhAQHG04DPnL3Hu7eF/g50Da6lYlUX1L4TUSkhk4H\nit39T+Ur3H1hFOsRqTG1LEQipz8wL+xWInFAYSEiImEpLEQiZwkwLNpFiNQGhYVI5HwANDKzq8pX\nmNlwMzs1ijWJ1IgmEhSJIDPrQOAeEMOAQgK39vyJu6+KZl0i1aWwEBGRsHQaSkREwlJYiIhIWAoL\nEREJS2EhIiJhKSxERCQshYWIiISlsBARkbD+HyUfdkmnmIJzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x27dbb2a0588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maximum R^2:  0.356978315261\n",
      "Best C:  3\n",
      "Pearson score = 0.602169061256\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVR\n",
    "\n",
    "Cs = [0.1,0.3,0.5,0.75,1,2,3,4,5,8,10,15,20,25,30,35,40,45,50,60]\n",
    "r2 = []\n",
    "max_r2, best_C = 0, 0\n",
    "for c in Cs:\n",
    "    model = SVR(C=c)\n",
    "    model.fit(X_train, y_train) # Train the model using the training sets\n",
    "    r = model.score(X_test, np.array(y_test))\n",
    "    if r > max_r2: max_r2, best_C = r, c\n",
    "    r2.append(r)\n",
    "\n",
    "plt.plot(Cs,r2)\n",
    "plt.xlabel('C')\n",
    "plt.ylabel('R^2')\n",
    "plt.title('SVR')\n",
    "plt.show()\n",
    "\n",
    "print('maximum R^2: ', max_r2)\n",
    "print('Best C: ', best_C)\n",
    "\n",
    "best_svr_model = SVR(C=best_C)\n",
    "best_svr_model.fit(X_train, y_train) # Train the model using the training sets\n",
    "y_pred = best_svr_model.predict(X_test) # Make predictions using the testing set\n",
    "pearson = pearsonr(y_test, y_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R^2:  0.259401439798\n",
      "Pearson score = 0.538521207861\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "\n",
    "AdaBoost_model = AdaBoostRegressor(base_estimator=SVR(C=3), random_state=seed)\n",
    "AdaBoost_model.fit(X_train, y_train) # Train the model using the training sets\n",
    "r2 = AdaBoost_model.score(X_test, np.array(y_test))\n",
    "print('R^2: ', r2)\n",
    "y_pred = AdaBoost_model.predict(X_test) # Make predictions using the testing set\n",
    "pearson = pearsonr(y_test, y_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pearson score = 0.518954183872\n"
     ]
    }
   ],
   "source": [
    "import xgboost\n",
    "\n",
    "xg_reg = xgboost.XGBRegressor(objective='reg:linear', max_depth=5, n_estimators=100, seed=seed)\n",
    "xg_reg.fit(X_train, y_train)\n",
    "y_pred = xg_reg.predict(X_test)\n",
    "pearson = pearsonr(y_test, y_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Manual Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R^2:  0.358935245862\n",
      "Pearson score = 0.627178385388\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "\n",
    "ensemble_models = [best_ridge_model, best_svr_model, best_random_forest_model]\n",
    "avg_pred = np.zeros(X_test.shape[0])\n",
    "for model in ensemble_models:\n",
    "    pred = model.predict(X_test)\n",
    "    avg_pred = np.add(avg_pred, pred)\n",
    "avg_pred = avg_pred / len(ensemble_models)\n",
    "#ridge_pred = best_ridge_model.predict(X_test)\n",
    "#svr_pred = best_svr_model.predict(X_test)\n",
    "#random_forest_pred = best_random_forest_model.predict(X_test)\n",
    "#avg_pred = np.add(np.add(ridge_pred,svr_pred), random_forest_pred) / 3\n",
    "r2 = r2_score(y_test, avg_pred)\n",
    "print('R^2: ', r2)\n",
    "pearson = pearsonr(y_test, avg_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 600 samples, validate on 199 samples\n",
      "Epoch 1/200\n",
      "600/600 [==============================] - 0s - loss: 0.0625 - mean_squared_error: 0.0625 - val_loss: 0.0309 - val_mean_squared_error: 0.0309\n",
      "Epoch 2/200\n",
      "600/600 [==============================] - 0s - loss: 0.0346 - mean_squared_error: 0.0346 - val_loss: 0.0361 - val_mean_squared_error: 0.0361\n",
      "Epoch 3/200\n",
      "600/600 [==============================] - 0s - loss: 0.0351 - mean_squared_error: 0.0351 - val_loss: 0.0329 - val_mean_squared_error: 0.0329\n",
      "Epoch 4/200\n",
      "600/600 [==============================] - 0s - loss: 0.0338 - mean_squared_error: 0.0338 - val_loss: 0.0337 - val_mean_squared_error: 0.0337\n",
      "Epoch 5/200\n",
      "600/600 [==============================] - 0s - loss: 0.0341 - mean_squared_error: 0.0341 - val_loss: 0.0273 - val_mean_squared_error: 0.0273\n",
      "Epoch 6/200\n",
      "600/600 [==============================] - 0s - loss: 0.0308 - mean_squared_error: 0.0308 - val_loss: 0.0264 - val_mean_squared_error: 0.0264\n",
      "Epoch 7/200\n",
      "600/600 [==============================] - 0s - loss: 0.0310 - mean_squared_error: 0.0310 - val_loss: 0.0258 - val_mean_squared_error: 0.0258\n",
      "Epoch 8/200\n",
      "600/600 [==============================] - 0s - loss: 0.0307 - mean_squared_error: 0.0307 - val_loss: 0.0258 - val_mean_squared_error: 0.0258\n",
      "Epoch 9/200\n",
      "600/600 [==============================] - 0s - loss: 0.0305 - mean_squared_error: 0.0305 - val_loss: 0.0245 - val_mean_squared_error: 0.0245\n",
      "Epoch 10/200\n",
      "600/600 [==============================] - 0s - loss: 0.0296 - mean_squared_error: 0.0296 - val_loss: 0.0257 - val_mean_squared_error: 0.0257\n",
      "Epoch 11/200\n",
      "600/600 [==============================] - 0s - loss: 0.0279 - mean_squared_error: 0.0279 - val_loss: 0.0364 - val_mean_squared_error: 0.0364\n",
      "Epoch 12/200\n",
      "600/600 [==============================] - 0s - loss: 0.0278 - mean_squared_error: 0.0278 - val_loss: 0.0304 - val_mean_squared_error: 0.0304\n",
      "Epoch 13/200\n",
      "600/600 [==============================] - 0s - loss: 0.0295 - mean_squared_error: 0.0295 - val_loss: 0.0233 - val_mean_squared_error: 0.0233\n",
      "Epoch 14/200\n",
      "600/600 [==============================] - 0s - loss: 0.0280 - mean_squared_error: 0.0280 - val_loss: 0.0226 - val_mean_squared_error: 0.0226\n",
      "Epoch 15/200\n",
      "600/600 [==============================] - 0s - loss: 0.0263 - mean_squared_error: 0.0263 - val_loss: 0.0220 - val_mean_squared_error: 0.0220\n",
      "Epoch 16/200\n",
      "600/600 [==============================] - 0s - loss: 0.0261 - mean_squared_error: 0.0261 - val_loss: 0.0335 - val_mean_squared_error: 0.0335\n",
      "Epoch 17/200\n",
      "600/600 [==============================] - 0s - loss: 0.0285 - mean_squared_error: 0.0285 - val_loss: 0.0232 - val_mean_squared_error: 0.0232\n",
      "Epoch 18/200\n",
      "600/600 [==============================] - 0s - loss: 0.0244 - mean_squared_error: 0.0244 - val_loss: 0.0220 - val_mean_squared_error: 0.0220\n",
      "Epoch 19/200\n",
      "600/600 [==============================] - 0s - loss: 0.0257 - mean_squared_error: 0.0257 - val_loss: 0.0228 - val_mean_squared_error: 0.0228\n",
      "Epoch 20/200\n",
      "600/600 [==============================] - 0s - loss: 0.0242 - mean_squared_error: 0.0242 - val_loss: 0.0204 - val_mean_squared_error: 0.0204\n",
      "Epoch 21/200\n",
      "600/600 [==============================] - 0s - loss: 0.0254 - mean_squared_error: 0.0254 - val_loss: 0.0247 - val_mean_squared_error: 0.0247\n",
      "Epoch 22/200\n",
      "600/600 [==============================] - 0s - loss: 0.0259 - mean_squared_error: 0.0259 - val_loss: 0.0341 - val_mean_squared_error: 0.0341\n",
      "Epoch 23/200\n",
      "600/600 [==============================] - 0s - loss: 0.0252 - mean_squared_error: 0.0252 - val_loss: 0.0209 - val_mean_squared_error: 0.0209\n",
      "Epoch 24/200\n",
      "600/600 [==============================] - 0s - loss: 0.0257 - mean_squared_error: 0.0257 - val_loss: 0.0214 - val_mean_squared_error: 0.0214\n",
      "Epoch 25/200\n",
      "600/600 [==============================] - 0s - loss: 0.0240 - mean_squared_error: 0.0240 - val_loss: 0.0507 - val_mean_squared_error: 0.0507\n",
      "Epoch 26/200\n",
      "600/600 [==============================] - 0s - loss: 0.0256 - mean_squared_error: 0.0256 - val_loss: 0.0216 - val_mean_squared_error: 0.0216\n",
      "Epoch 27/200\n",
      "600/600 [==============================] - 0s - loss: 0.0239 - mean_squared_error: 0.0239 - val_loss: 0.0315 - val_mean_squared_error: 0.0315\n",
      "Epoch 28/200\n",
      "600/600 [==============================] - 0s - loss: 0.0230 - mean_squared_error: 0.0230 - val_loss: 0.0370 - val_mean_squared_error: 0.0370\n",
      "Epoch 29/200\n",
      "600/600 [==============================] - 0s - loss: 0.0237 - mean_squared_error: 0.0237 - val_loss: 0.0210 - val_mean_squared_error: 0.0210\n",
      "Epoch 30/200\n",
      "600/600 [==============================] - 0s - loss: 0.0233 - mean_squared_error: 0.0233 - val_loss: 0.0203 - val_mean_squared_error: 0.0203\n",
      "Epoch 31/200\n",
      "600/600 [==============================] - 0s - loss: 0.0227 - mean_squared_error: 0.0227 - val_loss: 0.0271 - val_mean_squared_error: 0.0271\n",
      "Epoch 32/200\n",
      "600/600 [==============================] - 0s - loss: 0.0237 - mean_squared_error: 0.0237 - val_loss: 0.0198 - val_mean_squared_error: 0.0198\n",
      "Epoch 33/200\n",
      "600/600 [==============================] - 0s - loss: 0.0207 - mean_squared_error: 0.0207 - val_loss: 0.0201 - val_mean_squared_error: 0.0201\n",
      "Epoch 34/200\n",
      "600/600 [==============================] - 0s - loss: 0.0224 - mean_squared_error: 0.0224 - val_loss: 0.0203 - val_mean_squared_error: 0.0203\n",
      "Epoch 35/200\n",
      "600/600 [==============================] - 0s - loss: 0.0222 - mean_squared_error: 0.0222 - val_loss: 0.0191 - val_mean_squared_error: 0.0191\n",
      "Epoch 36/200\n",
      "600/600 [==============================] - 0s - loss: 0.0232 - mean_squared_error: 0.0232 - val_loss: 0.0223 - val_mean_squared_error: 0.0223\n",
      "Epoch 37/200\n",
      "600/600 [==============================] - 0s - loss: 0.0218 - mean_squared_error: 0.0218 - val_loss: 0.0386 - val_mean_squared_error: 0.0386\n",
      "Epoch 38/200\n",
      "600/600 [==============================] - 0s - loss: 0.0226 - mean_squared_error: 0.0226 - val_loss: 0.0205 - val_mean_squared_error: 0.0205\n",
      "Epoch 39/200\n",
      "600/600 [==============================] - 0s - loss: 0.0204 - mean_squared_error: 0.0204 - val_loss: 0.0246 - val_mean_squared_error: 0.0246\n",
      "Epoch 40/200\n",
      "600/600 [==============================] - 0s - loss: 0.0223 - mean_squared_error: 0.0223 - val_loss: 0.0272 - val_mean_squared_error: 0.0272\n",
      "Epoch 41/200\n",
      "600/600 [==============================] - 0s - loss: 0.0213 - mean_squared_error: 0.0213 - val_loss: 0.0217 - val_mean_squared_error: 0.0217\n",
      "Epoch 42/200\n",
      "600/600 [==============================] - 0s - loss: 0.0240 - mean_squared_error: 0.0240 - val_loss: 0.0200 - val_mean_squared_error: 0.0200.02\n",
      "Epoch 43/200\n",
      "600/600 [==============================] - 0s - loss: 0.0204 - mean_squared_error: 0.0204 - val_loss: 0.0463 - val_mean_squared_error: 0.0463\n",
      "Epoch 44/200\n",
      "600/600 [==============================] - 0s - loss: 0.0228 - mean_squared_error: 0.0228 - val_loss: 0.0200 - val_mean_squared_error: 0.0200\n",
      "Epoch 45/200\n",
      "600/600 [==============================] - 0s - loss: 0.0197 - mean_squared_error: 0.0197 - val_loss: 0.0190 - val_mean_squared_error: 0.0190\n",
      "Epoch 46/200\n",
      "600/600 [==============================] - 0s - loss: 0.0221 - mean_squared_error: 0.0221 - val_loss: 0.0215 - val_mean_squared_error: 0.0215\n",
      "Epoch 47/200\n",
      "600/600 [==============================] - 0s - loss: 0.0216 - mean_squared_error: 0.0216 - val_loss: 0.0324 - val_mean_squared_error: 0.0324\n",
      "Epoch 48/200\n",
      "600/600 [==============================] - 0s - loss: 0.0201 - mean_squared_error: 0.0201 - val_loss: 0.0254 - val_mean_squared_error: 0.0254\n",
      "Epoch 49/200\n",
      "600/600 [==============================] - 0s - loss: 0.0208 - mean_squared_error: 0.0208 - val_loss: 0.0200 - val_mean_squared_error: 0.0200\n",
      "Epoch 50/200\n",
      "600/600 [==============================] - 0s - loss: 0.0203 - mean_squared_error: 0.0203 - val_loss: 0.0193 - val_mean_squared_error: 0.0193\n",
      "Epoch 51/200\n",
      "600/600 [==============================] - 0s - loss: 0.0204 - mean_squared_error: 0.0204 - val_loss: 0.0189 - val_mean_squared_error: 0.0189\n",
      "Epoch 52/200\n",
      "600/600 [==============================] - 0s - loss: 0.0200 - mean_squared_error: 0.0200 - val_loss: 0.0383 - val_mean_squared_error: 0.0383\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers.convolutional import Convolution1D, MaxPooling1D\n",
    "from keras.models import Sequential\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.layers import Embedding, LSTM\n",
    "\n",
    "def baseline_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(400, input_dim=300, activation='relu'))\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Dense(200, activation='relu'))\n",
    "    #model.add(Dense(1, activation='sigmoid'))\n",
    "    model.add(Dense(1))\n",
    "    model.add(Activation('sigmoid'))\n",
    "    # Compile model\n",
    "    model.compile(loss='mse', optimizer='RMSprop', metrics=['mse'])\n",
    "    return model\n",
    "\n",
    "early_stop = EarlyStopping(monitor='mean_squared_error', patience=6)\n",
    "callbacks = [early_stop]\n",
    "estimator = KerasRegressor(build_fn=baseline_model, epochs=200, batch_size=32, verbose=True)\n",
    "history = estimator.fit(X_train, y_train, callbacks=callbacks, validation_data=(X_test,y_test)) #callbacks=callbacks\n",
    "#print(estimator.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 32/199 [===>..........................] - ETA: 0s0.031834627022\n"
     ]
    }
   ],
   "source": [
    "print(estimator.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Neural Networ to predict word joy score given its embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Dense\n",
    "from keras.models import Sequential\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from keras.callbacks import EarlyStopping\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0057 - mean_squared_error: 0.0057     \n",
      "Epoch 2/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0048 - mean_squared_error: 0.0048     \n",
      "Epoch 3/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0046 - mean_squared_error: 0.0046     \n",
      "Epoch 4/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0044 - mean_squared_error: 0.0044     \n",
      "Epoch 5/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0043 - mean_squared_error: 0.0043     \n",
      "Epoch 6/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0041 - mean_squared_error: 0.0041     \n",
      "Epoch 7/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0041 - mean_squared_error: 0.0041     \n",
      "Epoch 8/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0040 - mean_squared_error: 0.0040     \n",
      "Epoch 9/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0039 - mean_squared_error: 0.0039     \n",
      "Epoch 10/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0038 - mean_squared_error: 0.0038     \n",
      "Epoch 11/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0038 - mean_squared_error: 0.0038     \n",
      "Epoch 12/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0037 - mean_squared_error: 0.0037     \n",
      "Epoch 13/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0037 - mean_squared_error: 0.0037     \n",
      "Epoch 14/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0037 - mean_squared_error: 0.0037     \n",
      "Epoch 15/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0037 - mean_squared_error: 0.0037     \n",
      "Epoch 16/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0036 - mean_squared_error: 0.0036     \n",
      "Epoch 17/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0036 - mean_squared_error: 0.0036     \n",
      "Epoch 18/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0036 - mean_squared_error: 0.0036     \n",
      "Epoch 19/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0036 - mean_squared_error: 0.0036     \n",
      "Epoch 20/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0036 - mean_squared_error: 0.0036     \n",
      "Epoch 21/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0035 - mean_squared_error: 0.0035     \n",
      "Epoch 22/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0035 - mean_squared_error: 0.0035     \n",
      "Epoch 23/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0035 - mean_squared_error: 0.0035     \n",
      "Epoch 24/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0035 - mean_squared_error: 0.0035     \n",
      "Epoch 25/60\n",
      "76719/76719 [==============================] - 10s - loss: 0.0035 - mean_squared_error: 0.0035    \n",
      "Epoch 26/60\n",
      "76719/76719 [==============================] - 10s - loss: 0.0035 - mean_squared_error: 0.0035    \n",
      "Epoch 27/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0035 - mean_squared_error: 0.0035     \n",
      "Epoch 28/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0035 - mean_squared_error: 0.0035     \n",
      "Epoch 29/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 30/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 31/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 32/60\n",
      "76719/76719 [==============================] - 10s - loss: 0.0034 - mean_squared_error: 0.0034    \n",
      "Epoch 33/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 34/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 35/60\n",
      "76719/76719 [==============================] - 10s - loss: 0.0034 - mean_squared_error: 0.0034    \n",
      "Epoch 36/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 37/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 38/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 39/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 40/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 41/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 42/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 43/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 44/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0034 - mean_squared_error: 0.0034     \n",
      "Epoch 45/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 46/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 47/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 48/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 49/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 50/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 51/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 52/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 53/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 54/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 55/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 56/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 57/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 58/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 59/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n",
      "Epoch 60/60\n",
      "76719/76719 [==============================] - 9s - loss: 0.0033 - mean_squared_error: 0.0033     \n"
     ]
    }
   ],
   "source": [
    "def baseline_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(200, input_dim=300, activation='relu'))\n",
    "    model.add(Dense(200, activation='relu'))\n",
    "    model.add(Dense(1))\n",
    "    # Compile model\n",
    "    model.compile(loss='mse', optimizer='adam', metrics=['mse'])\n",
    "    return model\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(np.array(X), np.array(y), test_size=0.33, random_state=42)\n",
    "#print(all([isinstance(l,float) for l in y_train]))\n",
    "#print(all([len(l)==300 for l in X_train]))\n",
    "#print(len(X_train[0]))\n",
    "#X_small, y_small = np.array(X[:5]), np.array(y[:5])\n",
    "early_stop = EarlyStopping(monitor='mean_squared_error', patience=3)\n",
    "callbacks = [early_stop]\n",
    "estimator = KerasRegressor(build_fn=baseline_model, epochs=60, batch_size=32, verbose=True)\n",
    "history = estimator.fit(X_train, y_train, callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37787/37787 [==============================] - 1s     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.0047048339411413925"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimator.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.018010324184401072"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(np.array(X), np.array(y), test_size=0.2, random_state=42)\n",
    "model = linear_model.Ridge(alpha=50, random_state=42)\n",
    "model.fit(X_train, y_train) # Train the model using the training sets\n",
    "model.score(X_test, y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing CNN+LSTM (2nd WASSA 2017 team model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word embeddings found for unprocessed tweets:  65.15487895126336\n",
      "word embeddings found for preprocessed tweets:  90.83756345177665\n",
      "word embeddings found for preprocessed tweets (without stemming):  90.36802030456853\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "import pandas as pd\n",
    "from resources.arabic_preprocessing import Arabic_preprocessing\n",
    "\n",
    "def embeddings_found_percentage(tweets, wv): #tweets: list of str, wv: word vectors\n",
    "    total_words, embedding_found = 0, 0\n",
    "    for tweet in tweets:\n",
    "        for word in tweet.split():\n",
    "            total_words += 1\n",
    "            if word in wv.wv.vocab:\n",
    "                embedding_found += 1\n",
    "    return embedding_found/total_words *100\n",
    "\n",
    "# load the embeddings model\n",
    "model = gensim.models.Word2Vec.load('C:\\\\Users\\\\oae15\\\\Downloads\\\\Compressed\\\\Twt-SG\\\\Twt-SG')\n",
    "\n",
    "sample=pd.read_csv('data/joy_train.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "prep = Arabic_preprocessing()\n",
    "sample['tweet_preprocessed'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x))\n",
    "sample['tweet_preprocessed_without_stemming'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False))\n",
    "tweets = sample['tweet'].tolist()\n",
    "tweets_preprocessed = sample['tweet_preprocessed'].tolist()\n",
    "tweets_preprocessed_without_stemming = sample['tweet_preprocessed_without_stemming'].tolist()\n",
    "print('word embeddings found for unprocessed tweets: ', embeddings_found_percentage(tweets,model))\n",
    "print('word embeddings found for preprocessed tweets: ', embeddings_found_percentage(tweets_preprocessed,model))\n",
    "print('word embeddings found for preprocessed tweets (without stemming): ', embeddings_found_percentage(tweets_preprocessed_without_stemming,model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preparing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from resources.arabic_preprocessing import Arabic_preprocessing\n",
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "# size of the word embeddings\n",
    "embeddings_dim = 300\n",
    "# maximum number of words to consider in the representations\n",
    "max_features = 4000\n",
    "# maximum length of a sentence\n",
    "max_sent_len = 30\n",
    "## Random Seed\n",
    "seed = 27\n",
    "#number of dimensions in regression problem\n",
    "reg_dimensions = 1\n",
    "\n",
    "np.random.seed(seed)\n",
    "prep = Arabic_preprocessing()\n",
    "\n",
    "# load the embeddings model\n",
    "word_embeddings_model = gensim.models.Word2Vec.load('C:\\\\Users\\\\oae15\\\\Downloads\\\\Compressed\\\\Twt-SG\\\\Twt-SG')\n",
    "\n",
    "# prepare traing data\n",
    "sample=pd.read_csv('data/joy_train.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "sample['tweet'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x))\n",
    "tweets = sample['tweet'].tolist()\n",
    "# prepare test data\n",
    "sample_test=pd.read_csv('data/joy_test.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "sample_test['tweet'] = sample_test['tweet'].apply(lambda x : prep.preprocess_arabic_text(x))\n",
    "tweets_test = sample_test['tweet'].tolist()\n",
    "\n",
    "tokenizer = Tokenizer(num_words=max_features, filters='%&()*+,-./:;<=>[\\\\]^_`{|}~\\t\\n',lower=False, split=\" \")\n",
    "tokenizer.fit_on_texts(tweets + tweets_test) # <-- Tokenizer based on all TEXTS\n",
    "\n",
    "train_sequences = sequence.pad_sequences( tokenizer.texts_to_sequences( tweets ) , maxlen=max_sent_len )\n",
    "y_train = sample['score'].tolist()\n",
    "test_sequences = sequence.pad_sequences( tokenizer.texts_to_sequences( tweets_test ) , maxlen=max_sent_len )\n",
    "y_test = sample_test['score'].tolist()\n",
    "\n",
    "embedding_weights = np.zeros( ( max_features , embeddings_dim ) )\n",
    "for word,index in tokenizer.word_index.items():\n",
    "    if index < max_features:\n",
    "        try: embedding_weights[index,:] = word_embeddings_model.wv[word]\n",
    "        except: embedding_weights[index,:] = np.random.rand( 1 , embeddings_dim )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\oae15\\AppData\\Local\\Continuum\\anaconda3\\envs\\ML\\lib\\site-packages\\ipykernel_launcher.py:12: UserWarning: Update your `Conv1D` call to the Keras 2 API: `Conv1D(padding=\"valid\", activation=\"relu\", filters=300, strides=1, kernel_size=3)`\n",
      "  if sys.path[0] == '':\n",
      "C:\\Users\\oae15\\AppData\\Local\\Continuum\\anaconda3\\envs\\ML\\lib\\site-packages\\ipykernel_launcher.py:13: UserWarning: Update your `MaxPooling1D` call to the Keras 2 API: `MaxPooling1D(pool_size=2)`\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "600/600 [==============================] - 9s - loss: 0.1532 - mean_squared_error: 0.0355     \n",
      "Epoch 2/20\n",
      "600/600 [==============================] - 6s - loss: 0.1302 - mean_squared_error: 0.0269     \n",
      "Epoch 3/20\n",
      "600/600 [==============================] - 7s - loss: 0.1015 - mean_squared_error: 0.0171     \n",
      "Epoch 4/20\n",
      "600/600 [==============================] - 6s - loss: 0.0942 - mean_squared_error: 0.0148     \n",
      "Epoch 5/20\n",
      "600/600 [==============================] - 5s - loss: 0.0849 - mean_squared_error: 0.0123     \n",
      "Epoch 6/20\n",
      "600/600 [==============================] - 6s - loss: 0.0742 - mean_squared_error: 0.0093     \n",
      "Epoch 7/20\n",
      "600/600 [==============================] - 5s - loss: 0.0625 - mean_squared_error: 0.0068     \n",
      "Epoch 8/20\n",
      "600/600 [==============================] - 5s - loss: 0.0648 - mean_squared_error: 0.0071     \n",
      "Epoch 9/20\n",
      "600/600 [==============================] - 5s - loss: 0.0636 - mean_squared_error: 0.0067     \n",
      "Epoch 10/20\n",
      "600/600 [==============================] - 6s - loss: 0.0636 - mean_squared_error: 0.0071     \n",
      "Epoch 11/20\n",
      "600/600 [==============================] - 6s - loss: 0.0512 - mean_squared_error: 0.0043     \n",
      "Epoch 12/20\n",
      "600/600 [==============================] - 6s - loss: 0.0509 - mean_squared_error: 0.0043     \n",
      "Epoch 13/20\n",
      "600/600 [==============================] - 6s - loss: 0.0475 - mean_squared_error: 0.0037     \n",
      "Epoch 14/20\n",
      "600/600 [==============================] - 6s - loss: 0.0416 - mean_squared_error: 0.0029     \n",
      "Epoch 15/20\n",
      "600/600 [==============================] - 6s - loss: 0.0445 - mean_squared_error: 0.0031     \n",
      "Epoch 16/20\n",
      "600/600 [==============================] - 6s - loss: 0.0446 - mean_squared_error: 0.0035     \n",
      "Epoch 17/20\n",
      "600/600 [==============================] - 6s - loss: 0.0424 - mean_squared_error: 0.0030     \n",
      "Epoch 18/20\n",
      "600/600 [==============================] - 6s - loss: 0.0423 - mean_squared_error: 0.0030     \n",
      "Epoch 19/20\n",
      "600/600 [==============================] - 5s - loss: 0.0372 - mean_squared_error: 0.0023     \n",
      "Epoch 20/20\n",
      "600/600 [==============================] - 6s - loss: 0.0353 - mean_squared_error: 0.0021     \n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.layers.convolutional import Convolution1D, MaxPooling1D\n",
    "from keras.layers import Embedding, LSTM\n",
    "\n",
    "filter_length = 3\n",
    "nb_filter = embeddings_dim\n",
    "pool_length = 2\n",
    "model = Sequential()\n",
    "model.add(Embedding(max_features, embeddings_dim, input_length=max_sent_len, weights=[embedding_weights]))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', activation='relu', subsample_length=1))\n",
    "model.add(MaxPooling1D(pool_length=pool_length))\n",
    "model.add(LSTM(embeddings_dim))\n",
    "model.add(Dense(reg_dimensions))\n",
    "model.add(Activation('sigmoid'))\n",
    "model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mse'])\n",
    "model.fit( train_sequences , y_train , epochs=20, batch_size=16)\n",
    "model.save(\"models/CNN_LSTM_Embeddings_20_epochs.h5\")  # creates a HDF5 file\n",
    "#results = model.predict( test_sequences )\n",
    "#np.savetxt(\"models/predictions_CNN_LSTM_Embeddings.txt\", results, newline='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: R2 : 0.968675, RMSE : 0.001064\n",
      "Test: R2 : 0.283799, RMSE : 0.022783\n",
      "Pearson score = 0.584550728745\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from scipy.stats import pearsonr\n",
    "\n",
    "#results = model.predict( test_sequences )\n",
    "#np.savetxt(\"models/predictions_CNN_LSTM_Embeddings.txt\", results, newline='\\n')\n",
    "\n",
    "y_pred = model.predict(test_sequences)\n",
    "y_pred_train = model.predict(train_sequences)\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "r2_train = r2_score(y_train, y_pred_train)\n",
    "rmse = mean_squared_error(y_test, y_pred)\n",
    "rmse_train = mean_squared_error(y_train, y_pred_train)\n",
    "print(\"Train: R2 : {0:f}, RMSE : {1:f}\".format( r2_train, rmse_train ) )\n",
    "print( \"Test: R2 : {0:f}, RMSE : {1:f}\".format( r2, rmse ) )\n",
    "\n",
    "pearson = pearsonr(y_test, y_pred[:,0])\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature-Based: Lexicon (ArSeL) scores\n",
    "#### 7 features:\n",
    "1. Average rating score across all words\n",
    "2. Average rating score across all nouns\n",
    "3. Average rating score across all adjectives\n",
    "4. Average rating score across all verbs\n",
    "5. Average rating score across all hashtags\n",
    "6. Maximum rating score\n",
    "7. Standard deviation of all rating scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(600, 4)\n",
      "(200, 4)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from resources.arabic_preprocessing import Arabic_preprocessing\n",
    "\n",
    "def remove_emojis(text):\n",
    "    emojis_unicode = r'([\\U0001F600-\\U0001F64F\\U00002000-\\U00003000]+)'\n",
    "    return re.sub(emojis_unicode, '', text)\n",
    "\n",
    "prep = Arabic_preprocessing()\n",
    "sample=pd.read_csv('data/joy_train.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "sample['tweet'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False, replace_emojis=False, normalize_arabic=False))\n",
    "sample['tweet'] = sample['tweet'].apply(lambda x : remove_emojis(x))\n",
    "#sample=sample.loc[sample['tweet'].str.len() > 10]\n",
    "#sample = sample.drop(sample[sample['tweet'].str.len() > 10].index)\n",
    "print(sample.shape)\n",
    "tweets = sample['tweet'].tolist()\n",
    "# prepare test data\n",
    "sample_test=pd.read_csv('data/joy_test.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "sample_test['tweet'] = sample_test['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False, replace_emojis=False, normalize_arabic=False))\n",
    "sample_test['tweet'] = sample_test['tweet'].apply(lambda x : remove_emojis(x))\n",
    "#sample_test=sample_test.loc[sample_test['tweet'].str.len() > 10]\n",
    "#sample_test = sample_test.drop(sample_test[sample_test['tweet'].str.len() > 10].index)\n",
    "print(sample_test.shape)\n",
    "tweets_test = sample_test['tweet'].tolist()\n",
    "\n",
    "all_tweets = tweets + tweets_test\n",
    "with open('all_tweets.txt', 'w', encoding='utf-8') as file:\n",
    "        file.write('\\n'.join(all_tweets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample=pd.read_csv('data/joy_train.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "y1 = sample['score'].tolist()\n",
    "\n",
    "sample_test=pd.read_csv('data/joy_test.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "y2 = sample_test['score'].tolist()\n",
    "\n",
    "y = y1 + y2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 13  26 141 191 318 529]\n",
      "(794, 2)\n"
     ]
    }
   ],
   "source": [
    "all_tweets = []\n",
    "with open('resources/output_bw.txt', 'r', encoding='utf-8') as file:\n",
    "    all_tweets = [line.strip() for line in file.readlines()]\n",
    "\n",
    "df = pd.DataFrame()\n",
    "df['tweet'] = all_tweets\n",
    "df['score'] = y\n",
    "indexes_to_drop = df.loc[df['tweet'].str.len() <= 10].index.values\n",
    "print(indexes_to_drop)\n",
    "df=df.loc[df['tweet'].str.len() > 10]\n",
    "print(df.shape)\n",
    "\n",
    "X = df['tweet'].tolist()\n",
    "y = df['score'].tolist()\n",
    "\n",
    "arsel = pd.read_csv('resources/ArSEL.txt', header=0, delimiter=';')\n",
    "arsel_bw = {bw: score for (bw,score) in zip(arsel['SAMA_Lemma'].tolist(),arsel['HAPPY'].tolist())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "794\n"
     ]
    }
   ],
   "source": [
    "arsel_scores = []\n",
    "for tweet in X:\n",
    "    scores = []\n",
    "    for word in tweet.split():\n",
    "        if word in arsel_bw: scores.append(arsel_bw[word])\n",
    "    arsel_scores.append(scores)\n",
    "    #feature_arsel_score_avg.append(sum/len(tweet.split()) if len(tweet.split()) != 0 else 0)\n",
    "feature_arsel_score_avg = [float(np.mean(tweet_score)) if len(tweet_score)>0 else 0 for tweet_score in arsel_scores]\n",
    "feature_arsel_score_std = [float(np.std(tweet_score)) if len(tweet_score)>0 else 0 for tweet_score in arsel_scores]\n",
    "feature_arsel_score_max = [float(np.max(tweet_score)) if len(tweet_score)>0 else 0 for tweet_score in arsel_scores]\n",
    "print(len(feature_arsel_score_avg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#nouns:  5116\n",
      "#verbs:  1652\n",
      "#adjs:  627\n"
     ]
    }
   ],
   "source": [
    "all_tweets_pos = []\n",
    "with open('resources/output_bw_pos.txt', 'r', encoding='utf-8') as file:\n",
    "    all_tweets_pos = [line.strip() for line in file.readlines()]\n",
    "\n",
    "#remove drop items\n",
    "for i in indexes_to_drop:\n",
    "    del all_tweets_pos[i]\n",
    "\n",
    "pos_dict = {'noun':[], 'verb':[],'adj':[]}\n",
    "for line in all_tweets_pos:\n",
    "    for word in line.split(' '):\n",
    "        w, pos = word.split('=>')[0], word.split('=>')[-1]\n",
    "        if pos == 'noun': pos_dict['noun'].append(w)\n",
    "        if pos == 'verb': pos_dict['verb'].append(w)\n",
    "        if pos == 'adj': pos_dict['adj'].append(w)\n",
    "print('#nouns: ', len(pos_dict['noun']))\n",
    "print('#verbs: ', len(pos_dict['verb']))\n",
    "print('#adjs: ', len(pos_dict['adj']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_noun_scores, all_verb_scores, all_adj_scores = [], [], []\n",
    "for tweet in all_tweets_pos:\n",
    "    noun_scores, verb_scores, adj_scores = [], [], []\n",
    "    for word in tweet.split():\n",
    "        w = word.split('=>')[0]\n",
    "        if w in arsel_bw:\n",
    "            if w in pos_dict['noun']: noun_scores.append(arsel_bw[w])\n",
    "            if w in pos_dict['verb']: verb_scores.append(arsel_bw[w])\n",
    "            if w in pos_dict['adj']: adj_scores.append(arsel_bw[w])\n",
    "    all_noun_scores.append(noun_scores)\n",
    "    all_verb_scores.append(verb_scores)\n",
    "    all_adj_scores.append(adj_scores)\n",
    "feature_arsel_score_noun = [float(np.mean(noun_scores)) if len(noun_scores)>0 else 0 for noun_scores in all_noun_scores]\n",
    "feature_arsel_score_verb = [float(np.mean(verb_scores)) if len(verb_scores)>0 else 0 for verb_scores in all_verb_scores]\n",
    "feature_arsel_score_adj = [float(np.mean(adj_scores)) if len(adj_scores)>0 else 0 for adj_scores in all_adj_scores]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 1, 4, 1, 0, 0, 0, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "#happy emojies\n",
    "# https://www.insight-centre.org/sites/default/files/publications/ianwood-emotionworkshop.pdf\n",
    "\n",
    "prep = Arabic_preprocessing()\n",
    "sample=pd.read_csv('data/joy_train.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "sample['tweet'] = sample['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False, replace_emojis=False, normalize_arabic=False))\n",
    "tweets = sample['tweet'].tolist()\n",
    "sample_test=pd.read_csv('data/joy_test.csv',header=None, names=['index', 'tweet','emotion','score'])\n",
    "sample_test['tweet'] = sample_test['tweet'].apply(lambda x : prep.preprocess_arabic_text(x, stem=False, replace_emojis=False, normalize_arabic=False))\n",
    "tweets_test = sample_test['tweet'].tolist()\n",
    "\n",
    "all_tweets = tweets + tweets_test\n",
    "\n",
    "joy_emojies = ['\\U0001F600','\\U0001F602','\\U0001F603','\\U0001F604','\\U0001F606','\\U0001F607','\\U0001F609','\\U0001F60A','\\U0001F60B','\\U0001F60C','\\U0001F60D','\\U0001F60E','\\U0001F60F','\\U0001F31E','\\U0000263A','\\U0001F618','\\U0001F61C','\\U0001F61D','\\U0001F61B','\\U0001F63A','\\U0001F638','\\U0001F639','\\U0001F63B','\\U0001F63C','\\U00002764','\\U0001F496','\\U0001F495','\\U0001F601','\\U00002665']\n",
    "feature_joy_emojies = []\n",
    "for tweet in all_tweets:\n",
    "    count = 0\n",
    "    for word in tweet.split():\n",
    "        if word in joy_emojies: count += 1\n",
    "    feature_joy_emojies.append(count)\n",
    "print(feature_joy_emojies[:22])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(794, 6)\n"
     ]
    }
   ],
   "source": [
    "X = np.zeros((794,6))\n",
    "X[:,0] = feature_arsel_score_avg\n",
    "X[:,1] = feature_arsel_score_max\n",
    "X[:,2] = feature_arsel_score_std\n",
    "X[:,3] = feature_arsel_score_noun\n",
    "X[:,4] = feature_arsel_score_verb\n",
    "X[:,5] = feature_arsel_score_adj\n",
    "#X[:,3] = feature_joy_emojies\n",
    "print(X.shape)\n",
    "X_train, X_test = X[:600,:], X[600:,:]\n",
    "\n",
    "#y = sample['score'].tolist() + sample_test['score'].tolist()\n",
    "y_train, y_test = y[:600], y[600:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[[ 0.14965305  0.19949312  0.03075276  0.14965305  0.17750023  0.16650379]\n",
      " [ 0.14853095  0.22253058  0.04509771  0.15507508  0.          0.12889857]\n",
      " [ 0.11190085  0.12354512  0.01164426  0.11190085  0.          0.        ]\n",
      " [ 0.16468161  0.19956698  0.03427047  0.14723893  0.          0.19956698]\n",
      " [ 0.14239969  0.17642479  0.02185232  0.13478484  0.17642479  0.16143681]]\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(X)\n",
    "print(df.isna().sum().sum())\n",
    "\n",
    "print(X_train[:5,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1 -0.00324127625113\n",
      "0.3 -0.00655329858866\n",
      "0.5 -0.00797276265571\n",
      "0.75 -0.00887400716181\n",
      "1 -0.00939798462402\n",
      "2 -0.0104324493076\n",
      "3 -0.0110021687364\n",
      "4 -0.0114222756106\n",
      "5 -0.0117595791394\n",
      "8 -0.0124872883373\n",
      "10 -0.012828327841\n",
      "15 -0.0134117595277\n",
      "20 -0.0137813708168\n",
      "25 -0.0140366755181\n",
      "30 -0.014223619723\n",
      "35 -0.0143664196813\n",
      "40 -0.0144790593075\n",
      "45 -0.0145701799801\n",
      "50 -0.0146454097978\n",
      "60 -0.0147623535872\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAEWCAYAAABfdFHAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcXXV9//HXe+7sS0Imu1kISwgI\nsg4hqK2RJcbWEqq4UNHY6i+tda2lilWLBW0Rd39VWh4Ujf4qKFaB6gNjiGIVQRMEWZQQCJAZspKZ\nkGUmme3z++OemdwMd5bkzp0zM3k/H4/7uOd87/ee+X7DkHe+3/M95ygiMDMzK5aStBtgZmbjm4PG\nzMyKykFjZmZF5aAxM7OictCYmVlROWjMzKyoHDRmKZH075I+McDnIenEkWyTWTHI19GYFY+kZ4Dp\nQBewF/gx8N6I2DuE7wYwPyKeLGojzYrMIxqz4vuziKgFzgTOAj6acnvMRpSDxmyERMRWYBXZwEHS\nNyR9qudzSf8gaYukzZL+Kve7kiZL+h9JuyWtlfQpSb/M+fxkSaslNUtaL+lNI9Uvs8E4aMxGiKTZ\nwGuBF02FSVoKXAlcDMwHLupT5avAPmAGsDx59Xy3BlgNfBuYBlwOfE3SqcPfC7PD56AxK77bJe0B\nGoHtwNV56rwJ+HpEPBoR+4BP9nwgKQO8Abg6Iloj4vfAypzvvg54JiK+HhGdEfFb4L+By4rTHbPD\n46AxK75LI6IOWAycDEzJU+clZIOox7M521OB0j6f524fC5wnaVfPC3gr2dGPWeocNGYjJCJ+DnwD\n+Fyej7cAc3L25+Zs7wA6gdk5Zbl1G4GfR8QxOa/aiHj38LTcrDAOGrOR9SXgYkln9in/LvAOSS+V\nVE3O9FpEdAHfBz4pqVrSycDbc777Q+AkSW+TVJa8zpV0SpH7YjYkDhqzERQRO4BvAp/oU34X2RD6\nKdnFAj/t89X3AhOBrcC3gFuAA8l39wBLgLcAm5M6nwEqitUPs8PhCzbNxiBJnwFmRMTyQSubpcwj\nGrMxILlO5nRlLQTeCfwg7XaZDUVp2g0wsyGpIztd9hKyS6Q/D9yRaovMhshTZ2ZmVlSeOjMzs6Ly\n1BkwZcqUmDdvXtrNMDMbUx544IHnI2LqYPUcNMC8efNYt25d2s0wMxtTJD07eC1PnZmZWZE5aMzM\nrKgcNGZmVlQOGjMzKyoHjZmZFZWDxszMispBY2ZmReWgKcDaZ5r5/E/W09HVnXZTzMxGLQdNAR7c\n1ML//emTtHc6aMzM+uOgKUCmJPvH19nlG5OamfXHQVOAsowA6Oj2iMbMrD8OmgJkSrJB09XtEY2Z\nWX8cNAUoS6bOvBjAzKx/DpoClGY8ojEzG4yDpgA9U2cdXgxgZtYvB00ByjLZPz6PaMzM+uegKcDB\nEY3P0ZiZ9cdBU4Ce5c2dHtGYmfUrtaCRVC9ptaQNyfukfuotT+pskLQ8p/wcSY9IelLSVySpz/eu\nlBSSphSrDz0XbHb5Ohozs36lOaK5ClgTEfOBNcn+ISTVA1cD5wELgatzAukGYAUwP3ktzfneHOBi\nYFMxO1DmxQBmZoNKM2iWASuT7ZXApXnqvAZYHRHNEdECrAaWSpoJTIiI+yIigG/2+f4XgQ8DRU2A\nUi8GMDMbVJpBMz0itgAk79Py1JkFNObsNyVls5LtvuVIugR4LiJ+N9APl7RC0jpJ63bs2HFEHfBi\nADOzwZUW8+CS7gZm5PnoY0M9RJ6y6K9cUnVy7CWDHTgibgRuBGhoaDiiIUmZL9g0MxtUUYMmIi7q\n7zNJ2yTNjIgtyVTY9jzVmoDFOfuzgXuS8tl9yjcDJwDHAb9L1gbMBn4raWFEbC2gK3n5gk0zs8Gl\nOXV2J9Czimw5cEeeOquAJZImJYsAlgCrkqm2PZIWJavN3g7cERGPRMS0iJgXEfPIBtLZxQgZOHjB\nZqdXnZmZ9SvNoLkOuFjSBrIrxK4DkNQg6SaAiGgGrgXWJq9rkjKAdwM3AU8CTwF3jWzzffdmM7Oh\nKOrU2UAiYidwYZ7ydcC7cvZvBm7up95pg/yMeQU3dAAH797soDEz64/vDFCAg3dv9tSZmVl/HDQF\nKPViADOzQTloCuALNs3MBuegKYAv2DQzG5yDpgC+e7OZ2eAcNAXw8mYzs8E5aApwcHmzp87MzPrj\noClASYmQPKIxMxuIg6ZAZSUlXt5sZjYAB02BSjPyBZtmZgNw0BQoUyKPaMzMBuCgKVBZpsR3bzYz\nG4CDpkCZEnkxgJnZABw0BSrz1JmZ2YAcNAXKZDyiMTMbiIOmQNnlzT5HY2bWHwdNgUo9ojEzG5CD\npkAZX7BpZjYgB02BynzBppnZgBw0BcqUyI8JMDMbgIOmQF4MYGY2MAdNgXzBppnZwBw0BSrN+IJN\nM7OBOGgKVJYp8YjGzGwADpoCZe/e7HM0Zmb9SSVoJNVLWi1pQ/I+qZ96y5M6GyQtzyk/R9Ijkp6U\n9BVJyvnsfZLWS3pM0vXF7kuZL9g0MxtQWiOaq4A1ETEfWJPsH0JSPXA1cB6wELg6J5BuAFYA85PX\n0uQ7rwaWAadHxKnA54rcDzIlJV7ebGY2gLSCZhmwMtleCVyap85rgNUR0RwRLcBqYKmkmcCEiLgv\nIgL4Zs733w1cFxEHACJiezE7AT13b/bUmZlZf9IKmukRsQUgeZ+Wp84soDFnvykpm5Vs9y0HOAn4\nI0m/lvRzSecOe8v78PJmM7OBlRbrwJLuBmbk+ehjQz1EnrIYoByy/ZkELALOBb4r6fhk5NO3fSvI\nTr8xd+7cITbpxUozvteZmdlAihY0EXFRf59J2iZpZkRsSabC8k1xNQGLc/ZnA/ck5bP7lG/O+c73\nk2D5jaRuYAqwI0/7bgRuBGhoaDjipPC9zszMBpbW1NmdQM8qsuXAHXnqrAKWSJqULAJYAqxKptr2\nSFqUrDZ7e873bwcuAJB0ElAOPF+8biT3OvOIxsysX2kFzXXAxZI2ABcn+0hqkHQTQEQ0A9cCa5PX\nNUkZZE/63wQ8CTwF3JWU3wwcL+lR4FZgeb5ps+FUlvGqMzOzgRRt6mwgEbETuDBP+TrgXTn7N5MN\nj3z1TstT3g5cMayNHUT27s2eOjMz64/vDFCg7PLmoMgDJzOzMctBU6BMSfaP0LNnZmb5OWgKVJrJ\nrrb2RZtmZvk5aApUlgSNL9o0M8vPQVOg8kz2j/BAp0c0Zmb5OGgKVFWeAaCtoyvllpiZjU4OmgJV\nlWdXiLe1d6bcEjOz0clBU6DqsuyIprXdIxozs3wcNAXqnTpz0JiZ5eWgKVBP0LT6HI2ZWV4OmgJV\ne0RjZjYgB02BqsocNGZmA3HQFMhTZ2ZmA3PQFKjay5vNzAbkoClQlZc3m5kNyEFToEyJKC8t8Z0B\nzMz64aAZBtXlGS8GMDPrh4NmGFSXZTx1ZmbWDwfNMKgsz3jqzMysHw6aYeCpMzOz/jlohkF1WSmt\nXt5sZpaXg2YYVJVnaOvwg8/MzPJx0AyDqrKML9g0M+uHg2YYVJd71ZmZWX8cNMOgqjzDfq86MzPL\ny0EzDKp8HY2ZWb9SCxpJ9ZJWS9qQvE/qp97ypM4GSctzys+R9IikJyV9RZKS8jMl3S/pIUnrJC0s\ndl+qk+toIqLYP8rMbMxJc0RzFbAmIuYDa5L9Q0iqB64GzgMWAlfnBNINwApgfvJampRfD/xzRJwJ\n/FOyX1RV5aVEwIFOrzwzM+srzaBZBqxMtlcCl+ap8xpgdUQ0R0QLsBpYKmkmMCEi7ovsMOKbOd8P\nYEKyPRHYXKwO9KipyN7Bec9+rzwzM+urNMWfPT0itgBExBZJ0/LUmQU05uw3JWWzku2+5QAfBFZJ\n+hzZIH15vh8uaQXZERFz584toBswqbocgJbWdqbWVRR0LDOz8aaoIxpJd0t6NM9r2VAPkacsBigH\neDfwdxExB/g74D/zHTgiboyIhohomDp16hCbk9/kmmzQ7NzbXtBxzMzGo6KOaCLiov4+k7RN0sxk\nNDMT2J6nWhOwOGd/NnBPUj67T3nPFNly4APJ9m3ATUfU+MNQX5sNmuZ9Dhozs77SPEdzJ9lQIHm/\nI0+dVcASSZOSRQBLgFXJlNseSYuS1WZvz/n+ZuBVyfYFwIZidaBHfU1P0Bwo9o8yMxtz0jxHcx3w\nXUnvBDYBbwSQ1AD8TUS8KyKaJV0LrE2+c01ENCfb7wa+AVQBdyUvgP8DfFlSKbCf5DxMMfWco9np\nEY2Z2YukFjQRsRO4ME/5OuBdOfs3Azf3U++0POW/BM4Z1sYOoixTwsSqMk+dmZnl4TsDDJPJNeUe\n0ZiZ5eGgGSb1NeU0e9WZmdmLOGiGSX1NuafOzMzycNAMk8m1njozM8tnwKCRlJH015KulfSKPp99\nvLhNG1vqa8ppaW2nu9s31jQzyzXYiOY/yF6TshP4iqQv5Hz2+qK1agyqr6mgqzvYvb8j7aaYmY0q\ngwXNwoj4i4j4Etk7KNdK+r6kCvLfBuao1XsbGk+fmZkdYrCgKe/ZiIjOiFgBPAT8FKgtZsPGmoN3\nB3DQmJnlGixo1klamlsQEdcAXwfmFatRY1G9b6xpZpbXgEETEVdExI/zlN8UEWXFa9bYM+uYKgCe\n29WWckvMzEaXIS1vlpQpdkPGumOqy5hQWcqzO/el3RQzs1Fl0KCRVEf+OytbDknMm1LDMztb026K\nmdmoMth1NDOBu4EbR6Y5Y9uxk2s8ojEz62OwEc0vgOsi4s6RaMxYN29yNU0tbXR0dafdFDOzUWOw\noGkBZo1EQ8aDeZNr6OoOmlq8IMDMrMdgQbMYeK2k94xAW8a8eVOqAXjG02dmZr0GW968D7gEOGtk\nmjO2HTu5BoBnn3fQmJn1GPQJmxHRRc4TL61/k2vKqa0oZaODxsys1xE9JiC5q/Nbh7sxY50kTp89\nkV9vbE67KWZmo8Zgy5snSPqopH+TtERZ7wM2Am8amSaOLYsXTGX9tj1s9h0CzMyAwUc03wIWAI+Q\nnT77CXAZsCwilhW5bWPS4gXTAPj5EztSbomZ2egw2Dma4yPiZQCSbgKeB+ZGxJ6it2yMmj+tllnH\nVPGzx7dz+cK5aTfHzCx1g41oep/ilSwKeNohMzBJvGrBVO598nnaO33hppnZYEFzhqTdyWsPcHrP\ntqTdI9HAsWjxSVPZ197Fume9KMDMbLDraDIRMSF51UVEac72hJFq5FjzihOnUJYR96z3eRozsyNa\n3lwoSfWSVkvakLxP6qfe8qTOBknLc8o/LalR0t4+9SskfUfSk5J+LWlecXuSX01FKQuPq+ee9dvT\n+PFmZqNKKkEDXAWsiYj5wJpk/xCS6oGrgfOAhcDVOYH0P0lZX+8EWiLiROCLwGeK0PYhWXzSNJ7Y\nttcPQjOzo15aQbMMWJlsrwQuzVPnNcDqiGiOiBZgNbAUICLuj4gtgxz3e8CFkjSsLR+ixQumAnhU\nY2ZHvbSCZnpPUCTv0/LUmQU05uw3MfidpHu/ExGdwAvA5HwVJa2QtE7Suh07hv9cyonJMmefpzGz\no92g9zo7UpLuBmbk+ehjQz1EnrIYru9ExI0kD3RraGgY7LiHTRKLF0zl9gefo72zm/LStDLdzCxd\nRfvbLyIuiojT8rzuALYlT+/seYpnvvmlJmBOzv5sYPMgP7b3O5JKgYlAamuMFy+Yll3m/IyXOZvZ\n0Sutf2bfCfSsIlsO3JGnzipgiaRJySKAJUnZUI97GfDTiBj20cpQvfyEyZRnSviZz9OY2VEsraC5\nDrhY0gbg4mQfSQ3JrW6IiGbgWmBt8romKUPS9ZKagGpJTZI+mRz3P4HJkp4EPkSe1Wwj6eAyZ5+n\nMbOjl1L8B/+o0dDQEOvWrSvKsW/6xUY+9aM/8MuPvJrZk6qL8jPMzNIg6YGIaBisns9QF1nP3Zw9\nqjGzo5WDpshOmFrDnPoqfvRwvst+zMzGPwdNkUni7Yvmcd/Gndy/cWfazTEzG3EOmhHwtvOPZfqE\nCj63aj0+J2ZmRxsHzQioLMvw3gvms+7ZFu7xkzfN7CjjoBkhb26Yw5z6Kj63aj3d3R7VmNnRw0Ez\nQspLS/jghSfx2Obd/PixrWk3x8xsxDhoRtClZ83ixGm1fGH1E3R5VGNmRwkHzQjKlIgPXXwST27f\ny+0PPpd2c8zMRoSDZoQtPXUGp75kAl9a8wTtnd1pN8fMrOgcNCOspERcuWQBjc1tfHdd4+BfMDMb\n4xw0KVi8YCrnzpvE53+ynu2796fdHDOzonLQpEAS//r602nr6OLK7z3sizjNbFxz0KTkxGm1fOxP\nTuF/n9jByl89k3ZzzMyKxkGToisWHcurF0zlX+96nCe27Um7OWZmReGgSZEkrr/sDGorSvnArQ9x\noLMr7SaZmQ07B03KptZV8Jk3nM4ftuzmCz95Iu3mmJkNOwfNKHDRS6fzF+fN5cZfbORXTz2fdnPM\nzIaVg2aU+PifnsJxk2v44K0P8dyutrSbY2Y2bBw0o0R1eSk3XHEObR1d/OXXf8MLbR1pN8nMbFg4\naEaRBTPq+I+3ncPTz+/jr7+1zosDzGxccNCMMi8/YQqfvewM7t/YzId9MaeZjQOlaTfAXuzSs2bx\n3K42PrtqPbOOqeLDS09Ou0lmZkfMQTNK/e3iE2hqaeNr9zzFzGOqeNuiY9NukpnZEXHQjFKSuHbZ\nqWzbvZ9P3P4oO/Yc4IMXzqekRGk3zczssKRyjkZSvaTVkjYk75P6qbc8qbNB0vKc8k9LapS0t0/9\nD0n6vaSHJa2RNKaHAaWZEm644mwuO2c2X1mzgXf/1wPsO9CZdrPMzA5LWosBrgLWRMR8YE2yfwhJ\n9cDVwHnAQuDqnED6n6SsrweBhog4HfgecH0R2j6iKkozfPay0/n4n57C6t9v4w03/IrG5ta0m2Vm\nNmRpBc0yYGWyvRK4NE+d1wCrI6I5IlqA1cBSgIi4PyK29P1CRPwsInr+Fr4fmD3sLU+BJN71R8fz\njb9cyOZdbSz76r3cv3Fn2s0yMxuStIJmek9QJO/T8tSZBeQ+grIpKRuqdwJ3HXELR6E/Pmkqd7z3\nlUyqLuOKm37Nt+5/Nu0mmZkNqmhBI+luSY/meS0b6iHylA3pohJJVwANwGcHqLNC0jpJ63bs2DHE\nJqXvuCk1/OA9r+CP5k/hE7c/ysd+8AgdXd1pN8vMrF9FW3UWERf195mkbZJmRsQWSTOB7XmqNQGL\nc/ZnA/cM9nMlXQR8DHhVRBwYoH03AjcCNDQ0jKmrIidUlnHT8nP57Kr1/PvPn+LJ7Xv52lvPZnJt\nRdpNMzN7kbSmzu4EelaRLQfuyFNnFbBE0qRkEcCSpKxfks4C/gO4JCLyhde4kSkRV732ZL705jN5\nsHEXy756L3/YsjvtZpmZvUhaQXMdcLGkDcDFyT6SGiTdBBARzcC1wNrkdU1ShqTrJTUB1ZKaJH0y\nOe5ngVrgNkkPSbpzJDuVhkvPmsVtf30+HV3dvOGGX/HjR1+0RsLMLFXyvbSyU2fr1q1LuxkF2b57\nPyu+9QAPNe7igxfN5/0X+OJOMysuSQ9ERMNg9XxTzXFi2oRKbl2xiNefPYsv3b2B93z7t7S2++JO\nM0ufg2YcqSzL8Pk3nsHH//QUVj22lTfccB9NLb6408zS5aAZZ3ou7rz5HefS1NLKJf92L7eta6S7\n21OkZpYOB804tXjBNG5/zyuYW1/NP3zvYS792r088Gxz2s0ys6OQg2YcO2FqLd9/98v54pvPYNvu\n/bzhhvv4wK0PsnlXW9pNM7OjiINmnCspEX9+1mx++veLed8FJ3LXo1u54PP38OW7N9DW7kdFm1nx\nOWiOEjUVpfz9kgWs+dCruPDk6Xzx7ie48PP3cOfvNvtx0WZWVA6ao8yc+mq++taz+c6KRRxTXc77\nb3mQN/77fTzS9ELaTTOzccoXbDI+Ltg8El3dwW3rGvnsqvU0t7bzyhOncPnCuVx0ynTKS/1vEDMb\n2FAv2HTQcPQGTY/d+zu4+ZdP8521jWx5YT9Tast5w9mzefO5czh+am3azTOzUcpBcxiO9qDp0dUd\n/O8TO7jlN5tY8/h2urqD846r5/KFc1l62gwqyzJpN9HMRhEHzWFw0LzY9t37ue2BJr6ztpFNza1M\nrCrjz8+axVsWzuHkGRPSbp6ZjQIOmsPgoOlfd3dw/8ad3LK2kVWPbqW9q5sz5xzD5Qvn8LrTX0JN\nRdEeaWRmo5yD5jA4aIameV873/9tE7eubeTJ7XuprSjlz854CZcvnMPLZk1E8t2izY4mDprD4KA5\nPBHBA8+2cMtvGvnRI5vZ39HNS2dO4PKFc7jkzFlMrCpLu4lmNgIcNIfBQXPkXmjr4M7fbebW32zi\nsc27qSwr4U9eNpPLF86l4dhJHuWYjWMOmsPgoBkejzS9wC1rN3HnQ5vZe6CTE6fV8pZz5/D6s2dT\nX1OedvPMbJg5aA6Dg2Z47TvQyY8e3sItazfx4KZdlGdKWHLqdC5fOJfzj5/sJ3+ajRMOmsPgoCme\n9Vv3cOvaTXz/t8/xQlsHU2orWHR8PYuOn8yi4ydzwtQaT6+ZjVEOmsPgoCm+/R1drHpsKz97fDv3\nbdzJtt0HAJhSW8F5SfCcf3w9J0ytdfCYjRFDDRpfBGEjorIsw7IzZ7HszFlEBM/ubOX+jTuTVzM/\nengLAFNqyzkvGe04eMzGBweNjThJzJtSw7wpNbxl4Vwigk3Nrb2hc99TOx08ZuOIg8ZSJ4ljJ9dw\n7OQa3nzui4Pn/o19gue4yb3neU6c5uAxG+0cNDbq5Auexua23qm2+zbu5EePZINnck15srDAwWM2\nWjlobNSTxNzJ1cydXM2bzp0zaPAsPK6eU2ZO4KTpdSyYUcfc+moyXlJtlhoHjY05AwbP0ztZ90wL\ndz26tbd+RWkJ86fXctL0Ok6eUdcbQDMmVHr0YzYCUlneLKke+A4wD3gGeFNEtOSptxz4eLL7qYhY\nmZR/Gng7MCkiXvRkLkmXAbcB50bEoOuWvbx5/Glt72TDtr2s37aHJ7buyb5v29O7rBqgrrKUBdPr\nOGlGXfY9CSDfxcBsaEb1dTSSrgeaI+I6SVeRDYyP9KlTD6wDGoAAHgDOiYgWSYuAZ4ENfYNGUh3w\nI6AceK+DxnLtam3niW17Wb91dxJCe3l862527+/srTO1riIneLIjofnT66j1IxHMDjHar6NZBixO\ntlcC9wAf6VPnNcDqiGgGkLQaWArcEhH3J2X5jn0tcD1w5XA32sa+Y6qz53AWHlffWxYRbN9zgPVb\ns6Oex5P3b//mWfZ3dPfWmz2pigXJqGfBjDrmTa5h9qQq6mvKPQVnNoC0gmZ6RGwBiIgtkqblqTML\naMzZb0rK+iXpLGBORPxQ0oBBI2kFsAJg7ty5h9N2G2ckMX1CJdMnVPLHJ03tLe/uDhpbWnsDaP22\nvTyxdQ8/f2IHnd0HZwKqyzPMnlTF7EnVzEnee/frq5hYVeYgsqNa0YJG0t3AjDwffWyoh8hT1u88\nn6QS4IvAO4Zy8Ii4EbgRslNnQ2yTHUVKSg4us15y6sFf5fbObp5+fh+bmltpbG6lqaWNppZWGlva\nWPt0M3sOdB5ynNqK0t7gyb5XMaf+YBj5+T023hUtaCLiov4+k7RN0sxkNDMT2J6nWhMHp9cAZpOd\nYutPHXAacE/yr8cZwJ2SLhnKeRqzoSovLemdPsvnhbaOQwLo4Hsr9z31PPvauw6pP6Gy9EWjoJ79\nWZOqqKso9YjIxrS0ps7uBJYD1yXvd+Spswr4F0mTkv0lwEf7O2BEvABM6dmXdA9wpUPGRtrEqjIm\nzprIabMmvuiziGBXawdNLW00JuHT1NJGY3MrTz+/j19seJ62jkODqLKshKl1FUyrq2RaXQXT6ip6\n96dOqGBqbQXTJlQwuabC1wvZqJRW0FwHfFfSO4FNwBsBJDUAfxMR74qIZknXAmuT71yTszDgeuAv\ngGpJTcBNEfHJke6E2eGSxKSacibVlPOy2fmDqHlfO43JKGjzrja27z7Ajr0H2L77AE9s28O9Tz5/\nyCq5HiWCybUVvWE0ra4yG0gT+oRTXQWVZZmR6K4Z4McEAF7ebGPP/o4uduw5wPY9B9ixZz/b92SD\nKFu2Pyk/wPN7D9Cd53/xusrSQ8Kovqac+ppyJlWXcUx1dvuY6rKkrNzBZHmN9uXNZlaAyrIMc+qr\nmVNfPWC9ru5g574DB0NpdzaIeva37znAQ427aNnX/qJFDLmqyjJMqi7Ljsaqy5P3sux2TnluQFWV\nZXxuyQAHjdm4lilRcm6nklMHqdve2c2utnZ2tXbQvK+dXa3tNO/roKW1nZZ97bS0Jtut7TS1tNLS\n2sELbR39Hq+8tIT66oPBc0x1GRMqy6irLGVCZRkTqsqYUFVKXcXB7Z7Pa70AYlxx0JgZkA2GnlAa\nqs6ubl5o6wmgQwNqVxJKPWH1xLa97G7rYM/+zhcteOirRFBXmRtEB8PpkKCqLO2tN6EyG2Q1FRlq\nKkqpKC1xWI0SDhozO2KlmRIm11YwubbisL7X3tnNnv0d7N7fmX1v62T3/o5DtntCaXdStqm5Nbvf\n1jHgNF9v20pETUV2dNQTPrUVpdSUlybb2bKDdfopK89+vzRTcqR/TEc9B42Zjbjy0iMLqB5d3cHe\nnhA6JKg62Xegk70Hsu/Z7a7sdnsne/Z3svWF/QfrtHfRlW+1RB4VpSW94VNdnqGqPENVWfZVmbPd\nW97v5yVUlmWoLi/N+byE8sz4HYE5aMxszMmUiInVZUysLuyuChHBgc7u3mDKvncdEla9Ze0Hy1rb\nu9jf0UVbexcvtHXQ1tHF/vYu2jqyr9x75A1ViegNqMohh1YmCa3ksz51e0Ktt25phpIUrrVy0JjZ\nUUsSlclf0FOOcHSVT3d3NsBa2zuT4Omirb27N4jaeoIq2T7kvU9otbV30bKvnefau3KOld0e4mDs\nEJVlJYeE1gcvOolLznjJsPU9HweNmdkwKylRdmRRXrzrjyKC9q5u9vcJsLaOzkNCrW9o7e/oojWn\nbFKBo8KhcNCYmY1BkqgozVBRmmEio/vGrF5GYWZmReWgMTOzonLQmJlZUTlozMysqBw0ZmZWVA4a\nMzMrKgeNmZkVlYPGzMyKyk9xmGpYAAAFuElEQVTYBCTtAJ49wq9PAZ4fxuakabz0Zbz0A9yX0cp9\nyTo2IqYOVslBUyBJ64byKNOxYLz0Zbz0A9yX0cp9OTyeOjMzs6Jy0JiZWVE5aAp3Y9oNGEbjpS/j\npR/gvoxW7sth8DkaMzMrKo9ozMysqBw0ZmZWVA6aIyRpqaT1kp6UdFXa7Tkckm6WtF3Sozll9ZJW\nS9qQvE9Ks41DJWmOpJ9J+oOkxyR9ICkfc/2RVCnpN5J+l/Tln5Py4yT9OunLdySVp93WoZCUkfSg\npB8m+2O1H89IekTSQ5LWJWVj7vcLQNIxkr4n6fHk/5nzR6IvDpojICkDfBV4LfBS4HJJL023VYfl\nG8DSPmVXAWsiYj6wJtkfCzqBv4+IU4BFwHuS/xZjsT8HgAsi4gzgTGCppEXAZ4AvJn1pAd6ZYhsP\nxweAP+Tsj9V+ALw6Is7Mud5kLP5+AXwZ+HFEnAycQfa/T/H7EhF+HeYLOB9YlbP/UeCjabfrMPsw\nD3g0Z389MDPZngmsT7uNR9ivO4CLx3p/gGrgt8B5ZK/aLk3KD/ndG60vYHbyl9YFwA8BjcV+JG19\nBpjSp2zM/X4BE4CnSRaBjWRfPKI5MrOAxpz9pqRsLJseEVsAkvdpKbfnsEmaB5wF/Jox2p9kuukh\nYDuwGngK2BURnUmVsfK79iXgw0B3sj+ZsdkPgAB+IukBSSuSsrH4+3U8sAP4ejKleZOkGkagLw6a\nI6M8ZV4nniJJtcB/Ax+MiN1pt+dIRURXRJxJdkSwEDglX7WRbdXhkfQ6YHtEPJBbnKfqqO5HjldE\nxNlkp8rfI+mP027QESoFzgZuiIizgH2M0JSfg+bINAFzcvZnA5tTastw2SZpJkDyvj3l9gyZpDKy\nIfNfEfH9pHjM9gcgInYB95A973SMpNLko7Hwu/YK4BJJzwC3kp0++xJjrx8ARMTm5H078AOy/wAY\ni79fTUBTRPw62f8e2eApel8cNEdmLTA/WUVTDrwFuDPlNhXqTmB5sr2c7LmOUU+SgP8E/hARX8j5\naMz1R9JUScck21XARWRP1v4MuCypNur7EhEfjYjZETGP7P8bP42ItzLG+gEgqUZSXc82sAR4lDH4\n+xURW4FGSQuSoguB3zMCffGdAY6QpD8h+6+0DHBzRHw65SYNmaRbgMVkbw++DbgauB34LjAX2AS8\nMSKa02rjUEl6JfAL4BEOng/4R7LnacZUfySdDqwk+ztVAnw3Iq6RdDzZkUE98CBwRUQcSK+lQydp\nMXBlRLxuLPYjafMPkt1S4NsR8WlJkxljv18Aks4EbgLKgY3AX5L8rlHEvjhozMysqDx1ZmZmReWg\nMTOzonLQmJlZUTlozMysqBw0ZmZWVA4as5QkdwWeUmgds9HOQWNmZkXloDEbAZJuT27K+FjOjRl7\nPpuXPB9kpaSHk+eFVOdUeZ+k3ybPRDk5+c5CSb9Kbo74q56rvSWdmjzT5qHkWPNHsJtmeTlozEbG\nX0XEOUAD8P7kyvJcC4AbI+J0YDfwtzmfPZ/c1PEG4Mqk7HHgj5ObI/4T8C9J+d8AX05uzNlA9v5W\nZqly0JiNjPdL+h1wP9kbsvYdaTRGxL3J9v8DXpnzWc+NQh8g+xwhgInAbco+JfWLwKlJ+X3AP0r6\nCHBsRLQNay/MjoCDxqzIkvt9XQScH9mnZz4IVPap1vdeULn7PfcD6yJ7vy2Aa4GfRcRpwJ/1HC8i\nvg1cArQBqyRdMEzdMDtiDhqz4psItEREa3KOZVGeOnMlnZ9sXw78cgjHfC7ZfkdPYXITyI0R8RWy\nd+U9vZCGmw0HB41Z8f0YKJX0MNmRyP156vwBWJ7UqSd7PmYg1wP/Kulesnd77vFm4NHkKZ0nA98s\ntPFmhfLdm81SljyC+ofJNJjZuOMRjZmZFZVHNGZmVlQe0ZiZWVE5aMzMrKgcNGZmVlQOGjMzKyoH\njZmZFdX/B4cCNWocTPs2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d4c8a11da0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maximum R^2:  0\n",
      "Best alpha:  0\n",
      "Pearson score = 0.114210961994\n"
     ]
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.stats import pearsonr\n",
    "\n",
    "alphas = [0.1,0.3,0.5,0.75,1,2,3,4,5,8,10,15,20,25,30,35,40,45,50,60]\n",
    "r2 = []\n",
    "max_r2, best_alpha = 0, 0\n",
    "for alpha in alphas:\n",
    "    model = linear_model.Ridge(alpha=alpha, random_state=132)\n",
    "    model.fit(X_train, y_train) # Train the model using the training sets\n",
    "    r = model.score(X_test, np.array(y_test))\n",
    "    print(alpha, r)\n",
    "    if r > max_r2: max_r2, best_alpha = r, alpha\n",
    "    r2.append(r)\n",
    "\n",
    "plt.plot(alphas,r2)\n",
    "plt.xlabel('alphas')\n",
    "plt.ylabel('R^2')\n",
    "plt.title('Ridge')\n",
    "plt.show()\n",
    "\n",
    "print('maximum R^2: ', max_r2)\n",
    "print('Best alpha: ', best_alpha)\n",
    "\n",
    "model = linear_model.Ridge(alpha=best_alpha, random_state=132)\n",
    "model.fit(X_train, y_train) # Train the model using the training sets\n",
    "y_pred = model.predict(X_test) # Make predictions using the testing set\n",
    "pearson = pearsonr(y_test, y_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.016924411922472182"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=1000, max_depth=3, random_state=42)\n",
    "model.fit(X_train, y_train) # Train the model using the training sets\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Autoencoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data:  (728, 300) (728,)\n",
      "test data:  (224, 300) (224,)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import gensim\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from resources.arabic_preprocessing import Arabic_preprocessing\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense\n",
    "\n",
    "seed = 222\n",
    "np.random.seed(seed)\n",
    "    \n",
    "def featurize(words_embeddings, sentence):\n",
    "    \"\"\"average words' vectors\"\"\"\n",
    "    feature_vec = np.zeros((300,), dtype=\"float32\")\n",
    "    words = sentence.split(' ')\n",
    "    retrieved_words = 0\n",
    "    for token in words:\n",
    "        try:\n",
    "            feature_vec = np.add(feature_vec, words_embeddings.wv[token])\n",
    "            retrieved_words += 1\n",
    "        except KeyError:\n",
    "            pass  # if a word is not in the embeddings' vocabulary discard it\n",
    "    np.seterr(divide='ignore', invalid='ignore')\n",
    "    feature_vec = np.divide(feature_vec, retrieved_words)\n",
    "    return feature_vec\n",
    "\n",
    "def read_dataset_preprocess_featurize(csv_path, preprocess_object, words_embeddings, words_embeddings_size):\n",
    "    sample=pd.read_csv(csv_path, delimiter='\\t', skiprows=1, header=None, names=['index', 'tweet','emotion','score'])\n",
    "    sample['tweet'] = sample['tweet'].apply(lambda x : preprocess_object.preprocess_arabic_text(x, stem=False, replace_emojis=True, normalize_arabic=True))\n",
    "    sample = sample.dropna()\n",
    "    tweets = sample['tweet'].tolist()\n",
    "    scores = sample['score'].values\n",
    "    tweets_number = len(tweets)\n",
    "    tweets_embeddings = np.zeros((tweets_number,words_embeddings_size))\n",
    "    for i,tweet in enumerate(tweets):\n",
    "        tweets_embeddings[i,:] = featurize(words_embeddings, tweet)\n",
    "    return tweets_embeddings, scores\n",
    "\n",
    "prep = Arabic_preprocessing()\n",
    "tweeter_embeddings = gensim.models.Word2Vec.load('resources\\\\Twt-SG') # load tweeter word embeddings\n",
    "\n",
    "# prepare train data\n",
    "X_train, y_train = read_dataset_preprocess_featurize('data/2018-EI-reg-Ar-joy-train.txt', prep, tweeter_embeddings, 300)\n",
    "print('training data: ', X_train.shape, y_train.shape)\n",
    "# prepare test data\n",
    "X_test, y_test = read_dataset_preprocess_featurize('data/2018-EI-reg-Ar-joy-dev.txt', prep, tweeter_embeddings, 300)\n",
    "print('test data: ', X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 799 entries, 0 to 799\n",
      "Columns: 301 entries, 0 to score\n",
      "dtypes: float64(301)\n",
      "memory usage: 1.8 MB\n",
      "None\n",
      "<class 'numpy.ndarray'> <class 'numpy.ndarray'>\n",
      "(799, 300) (799,)\n",
      "(600, 300) (199, 300)\n",
      "600 199\n",
      "<class 'numpy.ndarray'> <class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "# drop rows with all zeros\n",
    "indexes_to_drop = []\n",
    "for i in range(X.shape[0]):\n",
    "    s = np.sum(X[i,:])\n",
    "    if s == 0:\n",
    "        indexes_to_drop.append(i)\n",
    "print(indexes_to_drop)\n",
    "\n",
    "df = pd.DataFrame(X)\n",
    "#df['embeddings'] = X\n",
    "df['score'] = y\n",
    "df = df.drop(indexes_to_drop)\n",
    "df = df.dropna()\n",
    "print(df.info())\n",
    "\n",
    "X_new = np.array(df.iloc[:,:300].values)\n",
    "y_new = np.array(df['score'].tolist())\n",
    "print(type(X_new), type(y_new))\n",
    "print(X_new.shape, y_new.shape)\n",
    "\n",
    "# Normalizing data (MaxMin)\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "X_new = scaler.fit_transform(X_new)\n",
    "X_train, X_test = X_new[:600,:], X_new[600:,:]\n",
    "y_train, y_test = y_new[:600], y_new[600:]\n",
    "\n",
    "print(X_train.shape, X_test.shape)\n",
    "print(len(y_train), len(y_test))\n",
    "print(type(y_train), type(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(728, 300) (224, 300)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(np.append(X_train, X_test, axis=0))\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "print(X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### prepare autoencoder model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import initializers\n",
    "\n",
    "input_size = 300\n",
    "hidden_1_size = 200\n",
    "#hidden_2_size = 100\n",
    "code_size = 50\n",
    "init_weights = initializers.Constant(value=0.001)\n",
    "init_bais = initializers.Zeros()\n",
    "\n",
    "input_layer = Input(shape=(input_size,))\n",
    "encoder_hidden_layer_1 = Dense(hidden_1_size, activation='relu', kernel_initializer=init_weights, bias_initializer=init_bais)(input_layer)\n",
    "#encoder_hidden_layer_2 = Dense(hidden_2_size, activation='relu')(encoder_hidden_layer_1)\n",
    "code_layer = Dense(code_size, activation='relu', kernel_initializer=init_weights, bias_initializer=init_bais)(encoder_hidden_layer_1)\n",
    "#decoder_hidden_layer_1 = Dense(hidden_2_size, activation='relu')(code_layer)\n",
    "decoder_hidden_layer_1 = Dense(hidden_1_size, activation='relu', kernel_initializer=init_weights, bias_initializer=init_bais)(code_layer)\n",
    "output_layer = Dense(input_size, activation='sigmoid', kernel_initializer=init_weights, bias_initializer=init_bais)(decoder_hidden_layer_1)\n",
    "\n",
    "autoencoder = Model(input_layer, output_layer)\n",
    "\n",
    "# create a separate encoder model\n",
    "encoder = Model(input_layer, code_layer)\n",
    "\n",
    "# retrieve the last layer of the autoencoder model \n",
    "#decoder_layer1 = autoencoder.layers[-3]\n",
    "decoder_layer1 = autoencoder.layers[-2]\n",
    "decoder_layer2 = autoencoder.layers[-1]\n",
    "\n",
    "# create the decoder model\n",
    "encoded_input_layer = Input(shape=(code_size,))\n",
    "#decoder = Model(encoded_input_layer, decoder_layer3(decoder_layer2(decoder_layer1(encoded_input_layer))))\n",
    "decoder = Model(encoded_input_layer, decoder_layer2(decoder_layer1(encoded_input_layer)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### train the autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "728/728 [==============================] - 1s - loss: 0.0177     \n",
      "Epoch 2/50\n",
      "728/728 [==============================] - 0s - loss: 0.0168     \n",
      "Epoch 3/50\n",
      "728/728 [==============================] - 0s - loss: 0.0167     \n",
      "Epoch 4/50\n",
      "728/728 [==============================] - 0s - loss: 0.0166     \n",
      "Epoch 5/50\n",
      "728/728 [==============================] - 0s - loss: 0.0166     \n",
      "Epoch 6/50\n",
      "728/728 [==============================] - 0s - loss: 0.0165     \n",
      "Epoch 7/50\n",
      "728/728 [==============================] - 0s - loss: 0.0165     \n",
      "Epoch 8/50\n",
      "728/728 [==============================] - 0s - loss: 0.0165     \n",
      "Epoch 9/50\n",
      "728/728 [==============================] - 0s - loss: 0.0164     \n",
      "Epoch 10/50\n",
      "728/728 [==============================] - 0s - loss: 0.0164     \n",
      "Epoch 11/50\n",
      "728/728 [==============================] - 0s - loss: 0.0163     \n",
      "Epoch 12/50\n",
      "728/728 [==============================] - 0s - loss: 0.0163     \n",
      "Epoch 13/50\n",
      "728/728 [==============================] - 0s - loss: 0.0162     \n",
      "Epoch 14/50\n",
      "728/728 [==============================] - 0s - loss: 0.0162     \n",
      "Epoch 15/50\n",
      "728/728 [==============================] - 0s - loss: 0.0161     \n",
      "Epoch 16/50\n",
      "728/728 [==============================] - 0s - loss: 0.0161     \n",
      "Epoch 17/50\n",
      "728/728 [==============================] - 0s - loss: 0.0160     \n",
      "Epoch 18/50\n",
      "728/728 [==============================] - 0s - loss: 0.0159     \n",
      "Epoch 19/50\n",
      "728/728 [==============================] - 0s - loss: 0.0159     \n",
      "Epoch 20/50\n",
      "728/728 [==============================] - 0s - loss: 0.0159     \n",
      "Epoch 21/50\n",
      "728/728 [==============================] - 0s - loss: 0.0158     \n",
      "Epoch 22/50\n",
      "728/728 [==============================] - 0s - loss: 0.0158     \n",
      "Epoch 23/50\n",
      "728/728 [==============================] - 0s - loss: 0.0158     \n",
      "Epoch 24/50\n",
      "728/728 [==============================] - 0s - loss: 0.0157     \n",
      "Epoch 25/50\n",
      "728/728 [==============================] - 0s - loss: 0.0157     \n",
      "Epoch 26/50\n",
      "728/728 [==============================] - 0s - loss: 0.0157     \n",
      "Epoch 27/50\n",
      "728/728 [==============================] - 0s - loss: 0.0157     \n",
      "Epoch 28/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 29/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 30/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 31/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 32/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 33/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 34/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 35/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 36/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 37/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 38/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 39/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 40/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 41/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 42/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 43/50\n",
      "728/728 [==============================] - 0s - loss: 0.0156     \n",
      "Epoch 44/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 45/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 46/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 47/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 48/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 49/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n",
      "Epoch 50/50\n",
      "728/728 [==============================] - 0s - loss: 0.0155     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x27dca8828d0>"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#autoencoder = Model(input_layer, output_layer)\n",
    "autoencoder.compile(optimizer=\"adam\", loss='mse')\n",
    "autoencoder.fit(X_train, X_train, epochs=50, batch_size=16) # , validation_data=[X_test, X_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 300) (1, 300)\n",
      "39.3793481106\n",
      "[ 0.49723702  0.33408686  0.75834309  0.31992505  0.15673994  0.09368528]\n",
      "[ 0.42682332  0.45328617  0.53294772  0.42051524  0.39021105  0.47657228]\n",
      "\n",
      "(1, 300) (1, 300)\n",
      "29.4956617254\n",
      "[ 0.51376499  0.52472909  0.63495132  0.43180492  0.25329582  0.37082469]\n",
      "[ 0.4783676   0.40529844  0.52703923  0.50286949  0.43589267  0.49917406]\n",
      "(224, 300) (224,)\n",
      "608/728 [========================>.....] - ETA: 0s0.0154880270526\n"
     ]
    }
   ],
   "source": [
    "x = X_train[0].reshape((1,300))\n",
    "pred = autoencoder.predict(X_train[0].reshape((1,300)))\n",
    "diff = np.sum(np.absolute(np.subtract(x, pred)))\n",
    "print(x.shape, pred.shape)\n",
    "print(np.sum(diff))\n",
    "print(x[0,:6])\n",
    "print(pred[0,:6])\n",
    "print()\n",
    "x = X_test[0].reshape((1,300))\n",
    "pred = autoencoder.predict(x)\n",
    "diff = np.sum(np.absolute(np.subtract(x, pred)))\n",
    "print(x.shape, pred.shape)\n",
    "print(np.sum(diff))\n",
    "print(x[0,:6])\n",
    "print(pred[0,:6])\n",
    "\n",
    "print(X_test.shape, y_test.shape)\n",
    "print(autoencoder.evaluate(X_train, X_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### extend training data: input training data to autoencoder, and append its output to training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = autoencoder.predict(X_train)\n",
    "pred2 = autoencoder.predict(pred)\n",
    "pred3 = autoencoder.predict(pred2)\n",
    "#print(pred.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_extended = np.append(X_train, pred, axis=0)\n",
    "X_extended1 = np.append(X_extended, pred2, axis=0)\n",
    "X_extended2 = np.append(X_extended1, pred3, axis=0)\n",
    "y_extended = np.append(y_train, y_train, axis=0)\n",
    "y_extended1 = np.append(y_extended, y_train, axis=0)\n",
    "y_extended2 = np.append(y_extended1, y_train, axis=0)\n",
    "# print(X_extended.shape)\n",
    "# print(y_extended.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### train models on original training data and extended training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pearson score =  0.585259933443\n",
      "Pearson score for extented =  0.583132860263\n",
      "Pearson score for extented1 =  0.582243111709\n",
      "Pearson score for extented2 =  0.581706993598\n"
     ]
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.stats import pearsonr\n",
    "\n",
    "def ridge(X_train, y_train, X_test, y_test):\n",
    "    alphas = [0.1,0.3,0.5,0.75,1,2,3,4,5,8,10,15,20,25,30,35,40,45,50,60]\n",
    "    r2 = []\n",
    "    max_r2, best_alpha = 0, 0\n",
    "    for alpha in alphas:\n",
    "        model = linear_model.Ridge(alpha=alpha, random_state=seed)\n",
    "        model.fit(X_train, y_train) # Train the model using the training sets\n",
    "        r = model.score(X_test, y_test)\n",
    "        if r > max_r2: max_r2, best_alpha = r, alpha\n",
    "        r2.append(r)\n",
    "    best_ridge_model = linear_model.Ridge(alpha=best_alpha, random_state=seed)\n",
    "    best_ridge_model.fit(X_train, y_train) # Train the model using the training sets\n",
    "    y_pred = best_ridge_model.predict(X_test) # Make predictions using the testing set\n",
    "    pearson = pearsonr(y_test, y_pred)\n",
    "    return best_ridge_model, pearson[0]\n",
    "\n",
    "# plt.plot(alphas,r2)\n",
    "# plt.xlabel('alphas')\n",
    "# plt.ylabel('R^2')\n",
    "# plt.title('Ridge')\n",
    "# plt.show()\n",
    "\n",
    "# print('maximum R^2: ', max_r2)\n",
    "# print('Best alpha: ', best_alpha)\n",
    "\n",
    "# best_ridge_model = linear_model.Ridge(alpha=best_alpha, random_state=seed)\n",
    "# best_ridge_model.fit(X_extended, y_extended) # Train the model using the training sets\n",
    "# #best_ridge_model.fit(X_train, y_train) # Train the model using the training sets\n",
    "# y_pred = best_ridge_model.predict(X_test) # Make predictions using the testing set\n",
    "# pearson = pearsonr(y_test, y_pred)\n",
    "\n",
    "ridge_model, pearson = ridge(X_train, y_train, X_test, y_test)\n",
    "print('Pearson score = ', pearson)\n",
    "ridge_model_extended, pearson = ridge(X_extended, y_extended, X_test, y_test)\n",
    "print('Pearson score for extented = ', pearson)\n",
    "ridge_model_extended_1, pearson = ridge(X_extended1, y_extended1, X_test, y_test)\n",
    "print('Pearson score for extented1 = ', pearson)\n",
    "ridge_model_extended_2, pearson = ridge(X_extended2, y_extended2, X_test, y_test)\n",
    "print('Pearson score for extented2 = ', pearson)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pearson score =  0.602169061256\n",
      "Pearson score for extented =  0.600833989339\n",
      "Pearson score for extented1 =  0.601245538093\n",
      "Pearson score for extented2 =  0.603199651256\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVR\n",
    "\n",
    "def svr(X_train, y_train, X_test, y_test):\n",
    "    Cs = [0.1,0.3,0.5,0.75,1,2,3,4,5,8,10,15,20,25,30,35,40,45,50,60]\n",
    "    r2 = []\n",
    "    max_r2, best_C = 0, 0\n",
    "    for c in Cs:\n",
    "        model = SVR(C=c)\n",
    "        model.fit(X_train, y_train) # Train the model using the training sets\n",
    "        r = model.score(X_test, y_test)\n",
    "        if r > max_r2: max_r2, best_C = r, c\n",
    "        r2.append(r)\n",
    "    best_svr_model = SVR(C=best_C)\n",
    "    best_svr_model.fit(X_train, y_train) # Train the model using the training sets\n",
    "    y_pred = best_svr_model.predict(X_test) # Make predictions using the testing set\n",
    "    pearson = pearsonr(y_test, y_pred)\n",
    "    return best_svr_model, pearson[0]\n",
    "\n",
    "# plt.plot(Cs,r2)\n",
    "# plt.xlabel('C')\n",
    "# plt.ylabel('R^2')\n",
    "# plt.title('SVR')\n",
    "# plt.show()\n",
    "\n",
    "# print('maximum R^2: ', max_r2)\n",
    "# print('Best C: ', best_C)\n",
    "\n",
    "# best_svr_model = SVR(C=best_C)\n",
    "# best_svr_model.fit(X_extended, y_extended) # Train the model using the training sets\n",
    "# y_pred = best_svr_model.predict(X_test) # Make predictions using the testing set\n",
    "# pearson = pearsonr(y_test, y_pred)\n",
    "# print('Pearson score = ' + str(pearson[0]))\n",
    "\n",
    "svr_model, pearson = svr(X_train, y_train, X_test, y_test)\n",
    "print('Pearson score = ', pearson)\n",
    "svr_model_extended, pearson = svr(X_extended, y_extended, X_test, y_test)\n",
    "print('Pearson score for extented = ', pearson)\n",
    "svr_model_extended_1, pearson = svr(X_extended1, y_extended1, X_test, y_test)\n",
    "print('Pearson score for extented1 = ', pearson)\n",
    "svr_model_extended_2, pearson = svr(X_extended2, y_extended2, X_test, y_test)\n",
    "print('Pearson score for extented2 = ', pearson)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pearson score = 0.56713400609\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "best_random_forest_model = RandomForestRegressor(n_estimators=1000, max_depth=10, random_state=seed)\n",
    "best_random_forest_model.fit(X_extended2, y_extended2) # Train the model using the training sets\n",
    "y_pred = best_random_forest_model.predict(X_test) # Make predictions using the testing set\n",
    "pearson = pearsonr(y_test, y_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R^2:  0.370381648086\n",
      "Pearson score = 0.611294870124\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "\n",
    "ensemble_models = [svr_model_extended_2, ridge_model_extended_2] # , best_random_forest_model\n",
    "avg_pred = np.zeros(X_test.shape[0])\n",
    "for model in ensemble_models:\n",
    "    pred = model.predict(X_test)\n",
    "    avg_pred = np.add(avg_pred, pred)\n",
    "avg_pred = avg_pred / len(ensemble_models)\n",
    "#ridge_pred = best_ridge_model.predict(X_test)\n",
    "#svr_pred = best_svr_model.predict(X_test)\n",
    "#random_forest_pred = best_random_forest_model.predict(X_test)\n",
    "#avg_pred = np.add(np.add(ridge_pred,svr_pred), random_forest_pred) / 3\n",
    "r2 = r2_score(y_test, avg_pred)\n",
    "print('R^2: ', r2)\n",
    "pearson = pearsonr(y_test, avg_pred)\n",
    "print('Pearson score = ' + str(pearson[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Train Autoencoder\n",
    "### 2. Use autoencoder's encoder part to:\n",
    "* train ML models on encoded data\n",
    "* evaluate on encoded test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 728 samples, validate on 224 samples\n",
      "Epoch 1/100\n",
      "728/728 [==============================] - 1s - loss: 0.0186 - val_loss: 0.0173\n",
      "Epoch 2/100\n",
      "728/728 [==============================] - 0s - loss: 0.0167 - val_loss: 0.0170\n",
      "Epoch 3/100\n",
      "728/728 [==============================] - 0s - loss: 0.0163 - val_loss: 0.0167\n",
      "Epoch 4/100\n",
      "728/728 [==============================] - 0s - loss: 0.0156 - val_loss: 0.0157\n",
      "Epoch 5/100\n",
      "728/728 [==============================] - 0s - loss: 0.0146 - val_loss: 0.0149\n",
      "Epoch 6/100\n",
      "728/728 [==============================] - 0s - loss: 0.0139 - val_loss: 0.0144\n",
      "Epoch 7/100\n",
      "728/728 [==============================] - 0s - loss: 0.0133 - val_loss: 0.0140\n",
      "Epoch 8/100\n",
      "728/728 [==============================] - 0s - loss: 0.0129 - val_loss: 0.0136\n",
      "Epoch 9/100\n",
      "728/728 [==============================] - 0s - loss: 0.0126 - val_loss: 0.0134\n",
      "Epoch 10/100\n",
      "728/728 [==============================] - 0s - loss: 0.0123 - val_loss: 0.0132\n",
      "Epoch 11/100\n",
      "728/728 [==============================] - 0s - loss: 0.0120 - val_loss: 0.0130\n",
      "Epoch 12/100\n",
      "728/728 [==============================] - 0s - loss: 0.0118 - val_loss: 0.0128\n",
      "Epoch 13/100\n",
      "728/728 [==============================] - 0s - loss: 0.0116 - val_loss: 0.0126\n",
      "Epoch 14/100\n",
      "728/728 [==============================] - 0s - loss: 0.0114 - val_loss: 0.0125\n",
      "Epoch 15/100\n",
      "728/728 [==============================] - 0s - loss: 0.0113 - val_loss: 0.0124\n",
      "Epoch 16/100\n",
      "728/728 [==============================] - 0s - loss: 0.0111 - val_loss: 0.0122\n",
      "Epoch 17/100\n",
      "728/728 [==============================] - 0s - loss: 0.0109 - val_loss: 0.0122\n",
      "Epoch 18/100\n",
      "728/728 [==============================] - 0s - loss: 0.0108 - val_loss: 0.0121\n",
      "Epoch 19/100\n",
      "728/728 [==============================] - 0s - loss: 0.0107 - val_loss: 0.0119\n",
      "Epoch 20/100\n",
      "728/728 [==============================] - 0s - loss: 0.0106 - val_loss: 0.0119\n",
      "Epoch 21/100\n",
      "728/728 [==============================] - ETA: 0s - loss: 0.010 - 0s - loss: 0.0104 - val_loss: 0.0118\n",
      "Epoch 22/100\n",
      "728/728 [==============================] - 0s - loss: 0.0103 - val_loss: 0.0117\n",
      "Epoch 23/100\n",
      "728/728 [==============================] - 0s - loss: 0.0102 - val_loss: 0.0116\n",
      "Epoch 24/100\n",
      "728/728 [==============================] - 0s - loss: 0.0101 - val_loss: 0.0115\n",
      "Epoch 25/100\n",
      "728/728 [==============================] - 0s - loss: 0.0100 - val_loss: 0.0114\n",
      "Epoch 26/100\n",
      "728/728 [==============================] - 0s - loss: 0.0099 - val_loss: 0.0114\n",
      "Epoch 27/100\n",
      "728/728 [==============================] - 0s - loss: 0.0098 - val_loss: 0.0113\n",
      "Epoch 28/100\n",
      "728/728 [==============================] - 0s - loss: 0.0097 - val_loss: 0.0113\n",
      "Epoch 29/100\n",
      "728/728 [==============================] - 0s - loss: 0.0096 - val_loss: 0.0112\n",
      "Epoch 30/100\n",
      "728/728 [==============================] - 0s - loss: 0.0096 - val_loss: 0.0111\n",
      "Epoch 31/100\n",
      "728/728 [==============================] - 0s - loss: 0.0095 - val_loss: 0.0111\n",
      "Epoch 32/100\n",
      "728/728 [==============================] - 0s - loss: 0.0094 - val_loss: 0.0110\n",
      "Epoch 33/100\n",
      "728/728 [==============================] - 0s - loss: 0.0094 - val_loss: 0.0111\n",
      "Epoch 34/100\n",
      "728/728 [==============================] - 0s - loss: 0.0093 - val_loss: 0.0110\n",
      "Epoch 35/100\n",
      "728/728 [==============================] - 0s - loss: 0.0092 - val_loss: 0.0110\n",
      "Epoch 36/100\n",
      "728/728 [==============================] - 0s - loss: 0.0091 - val_loss: 0.0109\n",
      "Epoch 37/100\n",
      "728/728 [==============================] - 0s - loss: 0.0091 - val_loss: 0.0109\n",
      "Epoch 38/100\n",
      "728/728 [==============================] - 0s - loss: 0.0090 - val_loss: 0.0109\n",
      "Epoch 39/100\n",
      "728/728 [==============================] - 0s - loss: 0.0090 - val_loss: 0.0108\n",
      "Epoch 40/100\n",
      "728/728 [==============================] - 0s - loss: 0.0089 - val_loss: 0.0107\n",
      "Epoch 41/100\n",
      "728/728 [==============================] - 0s - loss: 0.0088 - val_loss: 0.0107\n",
      "Epoch 42/100\n",
      "728/728 [==============================] - 0s - loss: 0.0088 - val_loss: 0.0107\n",
      "Epoch 43/100\n",
      "728/728 [==============================] - 0s - loss: 0.0087 - val_loss: 0.0107\n",
      "Epoch 44/100\n",
      "728/728 [==============================] - 0s - loss: 0.0087 - val_loss: 0.0106\n",
      "Epoch 45/100\n",
      "728/728 [==============================] - 0s - loss: 0.0086 - val_loss: 0.0106\n",
      "Epoch 46/100\n",
      "728/728 [==============================] - 0s - loss: 0.0085 - val_loss: 0.0106\n",
      "Epoch 47/100\n",
      "728/728 [==============================] - 0s - loss: 0.0085 - val_loss: 0.0105\n",
      "Epoch 48/100\n",
      "728/728 [==============================] - 0s - loss: 0.0084 - val_loss: 0.0106\n",
      "Epoch 49/100\n",
      "728/728 [==============================] - 0s - loss: 0.0084 - val_loss: 0.0105\n",
      "Epoch 50/100\n",
      "728/728 [==============================] - 0s - loss: 0.0084 - val_loss: 0.0105\n",
      "Epoch 51/100\n",
      "728/728 [==============================] - 0s - loss: 0.0083 - val_loss: 0.0105\n",
      "Epoch 52/100\n",
      "728/728 [==============================] - 0s - loss: 0.0082 - val_loss: 0.0104\n",
      "Epoch 53/100\n",
      "728/728 [==============================] - 0s - loss: 0.0082 - val_loss: 0.0104\n",
      "Epoch 54/100\n",
      "728/728 [==============================] - 0s - loss: 0.0082 - val_loss: 0.0105\n",
      "Epoch 55/100\n",
      "728/728 [==============================] - 0s - loss: 0.0081 - val_loss: 0.0104\n",
      "Epoch 56/100\n",
      "728/728 [==============================] - ETA: 0s - loss: 0.008 - 0s - loss: 0.0081 - val_loss: 0.0104\n",
      "Epoch 57/100\n",
      "728/728 [==============================] - 0s - loss: 0.0080 - val_loss: 0.0103\n",
      "Epoch 58/100\n",
      "728/728 [==============================] - 0s - loss: 0.0080 - val_loss: 0.0104\n",
      "Epoch 59/100\n",
      "728/728 [==============================] - 0s - loss: 0.0079 - val_loss: 0.0103\n",
      "Epoch 60/100\n",
      "728/728 [==============================] - 0s - loss: 0.0079 - val_loss: 0.0103\n",
      "Epoch 61/100\n",
      "728/728 [==============================] - 0s - loss: 0.0079 - val_loss: 0.0103\n",
      "Epoch 62/100\n",
      "728/728 [==============================] - 0s - loss: 0.0078 - val_loss: 0.0103\n",
      "Epoch 63/100\n",
      "728/728 [==============================] - 0s - loss: 0.0078 - val_loss: 0.0103\n",
      "Epoch 64/100\n",
      "728/728 [==============================] - 0s - loss: 0.0078 - val_loss: 0.0103\n",
      "Epoch 65/100\n",
      "728/728 [==============================] - 0s - loss: 0.0077 - val_loss: 0.0103\n",
      "Epoch 66/100\n",
      "728/728 [==============================] - 0s - loss: 0.0077 - val_loss: 0.0103\n",
      "Epoch 67/100\n",
      "728/728 [==============================] - 0s - loss: 0.0077 - val_loss: 0.0103\n",
      "Epoch 68/100\n",
      "728/728 [==============================] - 0s - loss: 0.0076 - val_loss: 0.0103\n",
      "Epoch 69/100\n",
      "728/728 [==============================] - ETA: 0s - loss: 0.007 - 0s - loss: 0.0076 - val_loss: 0.0102\n",
      "Epoch 70/100\n",
      "728/728 [==============================] - 0s - loss: 0.0076 - val_loss: 0.0102\n",
      "Epoch 71/100\n",
      "728/728 [==============================] - 0s - loss: 0.0076 - val_loss: 0.0103\n",
      "Epoch 72/100\n",
      "728/728 [==============================] - 0s - loss: 0.0075 - val_loss: 0.0103\n",
      "Epoch 73/100\n",
      "728/728 [==============================] - 0s - loss: 0.0075 - val_loss: 0.0102\n",
      "Epoch 74/100\n",
      "728/728 [==============================] - 0s - loss: 0.0075 - val_loss: 0.0102\n",
      "Epoch 75/100\n",
      "728/728 [==============================] - 0s - loss: 0.0075 - val_loss: 0.0102\n",
      "Epoch 76/100\n",
      "728/728 [==============================] - 0s - loss: 0.0074 - val_loss: 0.0102\n",
      "Epoch 77/100\n",
      "728/728 [==============================] - 0s - loss: 0.0074 - val_loss: 0.0102\n",
      "Epoch 78/100\n",
      "728/728 [==============================] - 0s - loss: 0.0074 - val_loss: 0.0102\n",
      "Epoch 79/100\n",
      "728/728 [==============================] - 0s - loss: 0.0073 - val_loss: 0.0102\n",
      "Epoch 80/100\n",
      "728/728 [==============================] - 0s - loss: 0.0073 - val_loss: 0.0102\n",
      "Epoch 81/100\n",
      "728/728 [==============================] - 0s - loss: 0.0073 - val_loss: 0.0102\n",
      "Epoch 82/100\n",
      "728/728 [==============================] - 0s - loss: 0.0073 - val_loss: 0.0102\n",
      "Epoch 83/100\n",
      "728/728 [==============================] - 0s - loss: 0.0072 - val_loss: 0.0102\n",
      "Epoch 84/100\n",
      "728/728 [==============================] - 0s - loss: 0.0072 - val_loss: 0.0102\n",
      "Epoch 85/100\n",
      "728/728 [==============================] - 0s - loss: 0.0072 - val_loss: 0.0102\n",
      "Epoch 86/100\n",
      "728/728 [==============================] - 0s - loss: 0.0072 - val_loss: 0.0102\n",
      "Epoch 87/100\n",
      "728/728 [==============================] - 0s - loss: 0.0072 - val_loss: 0.0102\n",
      "Epoch 88/100\n",
      "728/728 [==============================] - 0s - loss: 0.0071 - val_loss: 0.0102\n",
      "Epoch 89/100\n",
      "728/728 [==============================] - 0s - loss: 0.0071 - val_loss: 0.0102\n",
      "Epoch 90/100\n",
      "728/728 [==============================] - 0s - loss: 0.0071 - val_loss: 0.0102\n",
      "Epoch 91/100\n",
      "728/728 [==============================] - 0s - loss: 0.0071 - val_loss: 0.0102\n",
      "Epoch 92/100\n",
      "728/728 [==============================] - 0s - loss: 0.0071 - val_loss: 0.0102\n",
      "Epoch 93/100\n",
      "728/728 [==============================] - 0s - loss: 0.0070 - val_loss: 0.0102\n",
      "Epoch 94/100\n",
      "728/728 [==============================] - 0s - loss: 0.0070 - val_loss: 0.0101\n",
      "Epoch 95/100\n",
      "728/728 [==============================] - 0s - loss: 0.0070 - val_loss: 0.0102\n",
      "Epoch 96/100\n",
      "728/728 [==============================] - 0s - loss: 0.0070 - val_loss: 0.0102\n",
      "Epoch 97/100\n",
      "728/728 [==============================] - 0s - loss: 0.0070 - val_loss: 0.0102\n",
      "Epoch 98/100\n",
      "728/728 [==============================] - 0s - loss: 0.0070 - val_loss: 0.0102\n",
      "Epoch 99/100\n",
      "728/728 [==============================] - 0s - loss: 0.0069 - val_loss: 0.0102\n",
      "Epoch 100/100\n",
      "728/728 [==============================] - 0s - loss: 0.0069 - val_loss: 0.0102\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x27dbfe92c18>"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_size = 300\n",
    "hidden_1_size = 200\n",
    "#hidden_2_size = 100\n",
    "code_size = 50\n",
    "\n",
    "input_layer = Input(shape=(input_size,))\n",
    "encoder_hidden_layer_1 = Dense(hidden_1_size, activation='relu')(input_layer)\n",
    "#encoder_hidden_layer_2 = Dense(hidden_2_size, activation='relu')(encoder_hidden_layer_1)\n",
    "code_layer = Dense(code_size, activation='relu')(encoder_hidden_layer_1)\n",
    "#decoder_hidden_layer_1 = Dense(hidden_2_size, activation='relu')(code_layer)\n",
    "decoder_hidden_layer_1 = Dense(hidden_1_size, activation='relu')(code_layer)\n",
    "output_layer = Dense(input_size, activation='sigmoid')(decoder_hidden_layer_1)\n",
    "\n",
    "autoencoder = Model(input_layer, output_layer)\n",
    "\n",
    "# create a separate encoder model\n",
    "encoder = Model(input_layer, code_layer)\n",
    "\n",
    "# retrieve the last layer of the autoencoder model \n",
    "#decoder_layer1 = autoencoder.layers[-3]\n",
    "decoder_layer1 = autoencoder.layers[-2]\n",
    "decoder_layer2 = autoencoder.layers[-1]\n",
    "\n",
    "# create the decoder model\n",
    "encoded_input_layer = Input(shape=(code_size,))\n",
    "#decoder = Model(encoded_input_layer, decoder_layer3(decoder_layer2(decoder_layer1(encoded_input_layer))))\n",
    "decoder = Model(encoded_input_layer, decoder_layer2(decoder_layer1(encoded_input_layer)))\n",
    "\n",
    "autoencoder.compile(optimizer=\"adam\", loss='mse')\n",
    "autoencoder.fit(X_train, X_train, epochs=100, batch_size=32, validation_data=[X_test, X_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ridge =  0.550104142095\n",
      "SVR =  0.584658318708\n"
     ]
    }
   ],
   "source": [
    "X_train_encoded = encoder.predict(X_train)\n",
    "X_test_encoded = encoder.predict(X_test)\n",
    "\n",
    "m, p = ridge(X_train_encoded, y_train, X_test_encoded, y_test)\n",
    "print('Ridge = ', p)\n",
    "m, p = svr(X_train_encoded, y_train, X_test_encoded, y_test)\n",
    "print('SVR = ', p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(728, 50)"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_encoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVR =  0.591925181818\n"
     ]
    }
   ],
   "source": [
    "X_train_encoded_extended = encoder.predict(X_extended)\n",
    "#X_train_encoded = np.append(X_train_encoded, encoder.predict(X_extended1))\n",
    "#X_train_encoded = np.append(X_train_encoded, encoder.predict(X_extended2))\n",
    "m, p = svr(X_train_encoded_extended, y_extended, X_test_encoded, y_test)\n",
    "print('SVR = ', p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "10\n",
      "15\n",
      "20\n",
      "30\n",
      "50\n",
      "75\n",
      "100\n",
      "150\n",
      "{'150': 0.59575122251249502, '100': 0.60417638448690214, '10': 0.58039995111757159, '20': 0.60437127242889144, '30': 0.59026453547918634, '15': 0.59347063444202541, '50': 0.59061224207273477, '75': 0.60017652492086149, '5': 0.60230509473363336}\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dropout\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from keras.models import Sequential\n",
    "\n",
    "def baseline_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(80, input_dim=50, activation='relu'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(10, activation='relu'))\n",
    "    #model.add(Dense(20, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    # Compile model\n",
    "    #rmsprop = RMSprop(lr=learning_rate)\n",
    "    model.compile(loss='mse', optimizer='rmsprop', metrics=['mse'])\n",
    "    return model\n",
    "\n",
    "def NN(e):\n",
    "    print(e)\n",
    "    estimator = KerasRegressor(build_fn=baseline_model, epochs=20, batch_size=32)\n",
    "    history = estimator.fit(X_train_encoded, y_train, verbose=0, shuffle=False, validation_data=[X_test_encoded, y_test])\n",
    "    y_pred = estimator.predict(X_test_encoded)\n",
    "    p = pearsonr(y_test, y_pred)[0]\n",
    "    return p\n",
    "\n",
    "epochs = [5,10,15,20,30,50,75,100,150]\n",
    "d = dict()\n",
    "for e in epochs:\n",
    "    d[str(e)] = NN(e)\n",
    "print(d)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}