Experiment_ZW.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "name": "Central_ZW.ipynb",
   "provenance": [],
   "collapsed_sections": [],
   "toc_visible": true,
   "machine_shape": "hm"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "id": "N7b-yuJONlIA",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "from sklearn.pipeline import Pipeline\n",
    "from zeugma.embeddings import EmbeddingTransformer\n",
    "from deslib.util.diversity import double_fault\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "cKSFfy25MDTB",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import *\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "yiaZR6Ws9ftQ",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "import tensorflow.compat.v1 as tf\n",
    "tf.disable_v2_behavior()"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0FTJfofRN_OG",
    "colab_type": "text"
   },
   "source": [
    "# Extractors"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "pd3GfZlBOA4B",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from preprocessing import *\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "import matplotlib.lines as mlines\n",
    "from matplotlib_venn import venn3_unweighted, venn3, venn3_circles, venn2, venn2_unweighted, venn2_circles\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BvellogVODEe",
    "colab_type": "text"
   },
   "source": [
    "# Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "dATufhhOOEFZ",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from deslib.util.aggregation import *"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L010307gOO0t",
    "colab_type": "text"
   },
   "source": [
    "# Embedding"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "3AFZlSL1OPx4",
    "colab_type": "code",
    "outputId": "c62342f8-122b-45ef-9aa4-6a55a3c17e8e",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    }
   },
   "source": [
    "w2v = EmbeddingTransformer('word2vec')\n",
    "glove = EmbeddingTransformer('glove')\n",
    "fasttext = EmbeddingTransformer('fasttext')"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
      "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
     ],
     "name": "stderr"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "zKHIlsLgOc9V",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "uri_train = 'https://raw.githubusercontent.com/wvs2/data-hate/master/wassen/train.csv'\n",
    "uri_val   = 'https://raw.githubusercontent.com/wvs2/data-hate/master/wassen/val.csv'\n",
    "uri_teste = 'https://raw.githubusercontent.com/wvs2/data-hate/master/wassen/test.csv'\n",
    "\n",
    "df_train = pd.read_table(uri_train, sep=',')\n",
    "df_val   = pd.read_table(uri_val, sep=',')\n",
    "df_test = pd.read_table(uri_teste, sep=',')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "4NB0PUyNd-XP",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "train = df_train['text'].fillna(' ').apply(pre_processing)\n",
    "val = df_val['text'].fillna(' ').apply(pre_processing)\n",
    "test, class_test = df_test['text'].fillna(' ').apply(pre_processing), df_test['norm']"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "HzPj6FyvBqFq",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cv = CountVectorizer(analyzer='word', lowercase=True, stop_words='english')\n",
    "cv.fit_transform(train.values.astype('U'))\n",
    "     \n",
    "tfidf =  TfidfVectorizer(analyzer='word', lowercase=True, use_idf=True, stop_words='english')\n",
    "tfidf.fit_transform(train.values.astype('U'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AyJLkXuHeDm2",
    "colab_type": "text"
   },
   "source": [
    "# Funções"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "EIIXIL-ceEk4",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def calc_delta(labels = {}, pred = {}, clfs=[]):\n",
    "    size = len(clfs)\n",
    "    delta = np.zeros(shape=(size, size))\n",
    "    for i, clf1 in enumerate(clfs):\n",
    "        for k, clf2 in enumerate(clfs):\n",
    "            delta[i][k] = double_fault(labels, pred[clf1], pred[clf2])\n",
    "    return delta, clfs"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mZel_M2HeHV5",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def gera_tsne(delta=[], title='', classifiers=[], size=5, escala=0, x_ini=0, y_ini=0,x_fim=0,y_fim=0, legend=False):\n",
    "    simbolos = {}\n",
    "    simbolos[0] = 'X'\n",
    "    simbolos[1] = 'd'\n",
    "    simbolos[2] = '*'\n",
    "    simbolos[3] = \"^\"\n",
    "    simbolos[4] = 'o'\n",
    "    \n",
    "    tsne_model = TSNE(perplexity=50, init='pca', n_iter=2500, random_state=23)\n",
    "    new_values = tsne_model.fit_transform(delta)\n",
    "    \n",
    "    x = []\n",
    "    y = []\n",
    "    for value in new_values:\n",
    "        x.append(value[0])\n",
    "        y.append(value[1])\n",
    "\n",
    "    plt.figure(figsize=(size, size))\n",
    "    plt.title(title)\n",
    "    if escala > 0:\n",
    "        plt.ylim(escala * -1, escala)\n",
    "        plt.xlim(escala * -1, escala)\n",
    "    else:\n",
    "        plt.ylim(y_ini, y_fim)\n",
    "        plt.xlim(x_ini, x_fim)\n",
    "    \n",
    "    dot = 0\n",
    "    for i in range(len(x)):\n",
    "        plt.scatter(x[i], y[i], marker=simbolos[dot], label=classifiers[i], s=100)        \n",
    "        dot = dot+1\n",
    "    if legend:\n",
    "        plt.legend()\n",
    "    plt.show()"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ao4G4qqXSxdx",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def get_classifier(clf, statement, label, ext):\n",
    "    pipe_clf = Pipeline([\n",
    "        ('extractor', ext),\n",
    "        ('clf', clf)\n",
    "    ])\n",
    "    pipe_clf.fit(statement, label)\n",
    "    return pipe_clf"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "6oI9hmsaGYyA",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def get_CNN(ext, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, MAX_SEQUENCE_LENGTH=300, activation='sigmoid', word_embedding=False):\n",
    "    if word_embedding==False:\n",
    "        X_ext = ext.get_feature_names()\n",
    "        model = Word2Vec([X_ext], min_count=1, workers=1, size=300)\n",
    "    else:\n",
    "        model = ext.model\n",
    "\n",
    "    word_index = tokenizer.word_index\n",
    "    \n",
    "    nb_words = min(MAX_NB_WORDS, len(word_index))+1\n",
    "\n",
    "    embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))\n",
    "    for word, i in word_index.items():\n",
    "        if word in model.wv.vocab:\n",
    "            embedding_matrix[i] = model.wv.word_vec(word)\n",
    "\n",
    "    embedding_layer = Embedding(nb_words,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=MAX_SEQUENCE_LENGTH,\n",
    "        trainable=False\n",
    "    )\n",
    "    cnn = Sequential()\n",
    "    cnn.add(embedding_layer)\n",
    "    cnn.add(Dropout(0.2))\n",
    "    cnn.add(Conv1D(64, 3, padding='valid', activation='relu', strides=1))\n",
    "    cnn.add(GlobalMaxPooling1D())\n",
    "    cnn.add(Dense(256))\n",
    "    cnn.add(Dropout(0.2))\n",
    "    cnn.add(Activation('relu'))\n",
    "    cnn.add(Dense(3))\n",
    "    cnn.add(Activation(activation))\n",
    "    cnn.summary()\n",
    "    cnn.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "\n",
    "    return cnn"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Dy6vRjn0qWp4",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def get_oracle(labels, pred={}, clfs=[]):\n",
    "    cont = 0;\n",
    "    list_yes = np.zeros((len(labels)))\n",
    "    for i, k in enumerate(labels):\n",
    "        acertou = False\n",
    "        for clf in clfs:\n",
    "            if pred[clf][i] == k:\n",
    "                acertou = True\n",
    "                break\n",
    "        if acertou:\n",
    "            cont += 1\n",
    "            list_yes[i] = k\n",
    "        else:\n",
    "            if k == 0:\n",
    "                list_yes[i] = 1\n",
    "            else:\n",
    "                list_yes[i] = 0\n",
    "    return cont, list_yes\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "9NSBiaYxt7YV",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def tsne_full(labels,  delta , title='' , escala=0, size=5, x_ini=0, x_fim=0, y_ini=0, y_fim=0, espacamento=50, per=100 , lern=200.0, iterations=2500, ang=0.5, size_dot=10, cores={}, simbolos={}):\n",
    "    tsne_model = TSNE(init='pca', early_exaggeration=espacamento, perplexity=per, learning_rate=lern, random_state=42, n_iter=iterations, angle=ang)\n",
    "    new_values = tsne_model.fit_transform(delta)\n",
    "    \n",
    "\n",
    "    x = []\n",
    "    y = []\n",
    "    for value in new_values:\n",
    "        x.append(value[0])\n",
    "        y.append(value[1])\n",
    "\n",
    "    plt.figure(figsize=(size, size))\n",
    "    plt.title(title)\n",
    "\n",
    "    if escala > 0:\n",
    "        plt.ylim(escala * -1, escala)\n",
    "        plt.xlim(escala * -1, escala)\n",
    "    else:\n",
    "        plt.ylim(y_ini, y_fim)\n",
    "        plt.xlim(x_ini, x_fim)\n",
    "    \n",
    "    dot = 0\n",
    "    for i in range(len(x)):\n",
    "        plt.scatter(x[i], y[i], s=size_dot,  marker=simbolos[dot], c=cores[dot])        \n",
    "        \n",
    "        if labels[i] == 'SVM-CV' and i==40:\n",
    "            posicao=(5, -20)\n",
    "        elif labels[i] == 'MNB-GL':\n",
    "            posicao=(5, 25)\n",
    "        elif i == 56:\n",
    "            posicao=(5, 25)\n",
    "        else:  \n",
    "            posicao=(5, 8)\n",
    "\n",
    "\n",
    "        plt.annotate(labels[i], xy=(x[i], y[i]), xytext=posicao, textcoords='offset points', ha='center', va='bottom')  \n",
    "        dot = dot+1\n",
    "        if dot == 5:\n",
    "            dot = 0\n",
    "\n",
    "    m1 = mlines.Line2D([], [], color=cores[0], marker=simbolos[0], linestyle='None', markersize=10, label='CV')\n",
    "    m2 = mlines.Line2D([], [], color=cores[1], marker=simbolos[1], linestyle='None', markersize=10, label='TFIDF')\n",
    "    m3 = mlines.Line2D([], [], color=cores[2], marker=simbolos[2], linestyle='None', markersize=10, label='Word2Vec')\n",
    "    m4 = mlines.Line2D([], [], color=cores[3], marker=simbolos[3], linestyle='None', markersize=10, label='Glove')\n",
    "    m5 = mlines.Line2D([], [], color=cores[4], marker=simbolos[4], linestyle='None', markersize=10, label='FastText')\n",
    "    \n",
    "\n",
    "    plt.legend(handles=[m1, m2, m3, m4, m5])\n",
    "    plt.show()"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "5xtxw46fxnwz",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def diagrama_venn(p_a, p_b, p_c, label_a, label_b, label_c, txt):\n",
    "  acertos_a = 0 \n",
    "  acertos_a_b = 0\n",
    "  acertos_a_c = 0\n",
    "  acertos_b = 0\n",
    "  acertos_b_c = 0\n",
    "  acertos_c = 0\n",
    "  acertos_juncao = 0\n",
    "  \n",
    "\n",
    "  for index, row in df_test.iterrows():\n",
    "    #   print(p_a[index], row['norm'])\n",
    "      pred_a = p_a[index]\n",
    "      pred_b = p_b[index]\n",
    "      pred_c = p_c[index]\n",
    "\n",
    "      label = row['norm']\n",
    "      if pred_a == label and pred_b == label and pred_c == label :\n",
    "        acertos_juncao += 1\n",
    "      if pred_a == label and pred_b != label and pred_c != label :\n",
    "        acertos_a += 1 \n",
    "      if pred_a != label and pred_b == label and pred_c != label :\n",
    "        acertos_b += 1\n",
    "      if pred_a != label and pred_b != label and pred_c == label :\n",
    "        acertos_c += 1\n",
    "      if pred_a == label and pred_b == label and pred_c != label :\n",
    "        acertos_a_b += 1  \n",
    "      if pred_a != label and pred_b == label and pred_c == label :\n",
    "        acertos_b_c += 1   \n",
    "      if pred_a == label and pred_b != label and pred_c == label :\n",
    "        acertos_a_c += 1 \n",
    "\n",
    "\n",
    "  plt.figure(figsize=(4,4))\n",
    "  v = venn3_unweighted(subsets=(acertos_a, acertos_b, acertos_a_b ,acertos_c, acertos_a_c, acertos_b_c , acertos_juncao), set_labels = (label_a,  label_b,  label_c))\n",
    "  v.get_patch_by_id('001').set_color('white')\n",
    "  v.get_patch_by_id('100').set_color('white')\n",
    "  v.get_patch_by_id('010').set_color('white')\n",
    "  v.get_patch_by_id('111').set_color('white')\n",
    "  v.get_patch_by_id('011').set_color('white')\n",
    "  v.get_patch_by_id('110').set_color('white')\n",
    "  v.get_patch_by_id('101').set_color('white')\n",
    "\n",
    "  c = venn3_circles(subsets=(1, 1, 1, 1, 1, 1, 1), linestyle='solid')\n",
    "  plt.title(txt)\n",
    "  plt.show()"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "y_Uw2tyI44Zx",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def diagrama_venn2(label, pred_a, pred_b, label_a, label_b, title):\n",
    "    acertos_a = 0\n",
    "    acertos_b = 0\n",
    "    acertos_juncao = 0\n",
    "\n",
    "    for i, l in enumerate(label):\n",
    "        if pred_a[i] == l and pred_b[i] != l:\n",
    "            acertos_a += 1\n",
    "        if pred_a[i] != l and pred_b[i] == l:\n",
    "            acertos_b += 1\n",
    "        if pred_a[i] == l and pred_b[i] == l:\n",
    "            acertos_juncao += 1\n",
    "\n",
    "    plt.figure(figsize=(4,4))\n",
    "    v = venn2_unweighted(subsets=(acertos_a, acertos_b, acertos_juncao), set_labels = (label_a,  label_b))\n",
    "    v.get_patch_by_id('01').set_color('white')\n",
    "    v.get_patch_by_id('10').set_color('white')\n",
    "    v.get_patch_by_id('11').set_color('white')\n",
    "\n",
    "    c = venn2_circles(subsets=(1, 1, 1), linestyle='solid')\n",
    "    plt.title(title)\n",
    "    plt.show()"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "yOLW8HgL-ggK",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def autolabel(rects, ax):\n",
    "  for rect in rects:\n",
    "    height = rect.get_height()\n",
    "    ax.annotate('{}'.format(height),\n",
    "    xy=(\n",
    "      rect.get_x() + rect.get_width() / 2, height),\n",
    "      xytext=(0, 3),  # 3 points vertical offset\n",
    "      textcoords=\"offset points\",\n",
    "      ha='center', va='bottom'\n",
    "    )\n",
    "\n",
    "def render_bar(list_certos, title='', size_x=0, size_y=0):    \n",
    "    labels = [l for l in range(len(list_certos)-1, -1,-1)]\n",
    "    \n",
    "    # labels = [5, 4, 3, 2, 1, 0]\n",
    "    x = np.arange(len(labels))\n",
    "    width = 0.35 \n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    rects1 = ax.bar(x - width/2, list_certos.values(), width)\n",
    "\n",
    "    ax.set_ylabel('Instâncias')\n",
    "    ax.set_xlabel('Classificadores')\n",
    "    ax.set_title('Classificadores x Instâncias: '+title)\n",
    "    ax.set_xticks(x)\n",
    "    ax.set_xticklabels(labels)\n",
    "    ax.legend()\n",
    "    autolabel(rects1, ax)\n",
    "    fig.tight_layout()\n",
    "    if size_x != 0:        \n",
    "        plt.rcParams[\"figure.figsize\"] = (size_y, size_x)\n",
    "    plt.show()"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "tu8QgCFuDg_N",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def get_combine(data_test, clfs={}, cnn=False):\n",
    "    list_proba = []\n",
    "\n",
    "    for ext, clf in clfs.items():\n",
    "        list_proba.append(clf.predict_proba(data_test))\n",
    "    \n",
    "    ensemble = np.array(list_proba).transpose((1, 0, 2))\n",
    "    \n",
    "    votes = np.zeros((test.shape[0], len(clfs)))\n",
    "    k = 0\n",
    "    for clf_index, clf in clfs.items():    \n",
    "        if cnn == False:\n",
    "            votes[:, k] = clf.predict(data_test).reshape(test.shape[0])\n",
    "        else:\n",
    "            votes[:, k] = np.argmax(clf.predict(data_test),axis=1).reshape(test.shape[0])\n",
    "        k += 1\n",
    "    \n",
    "    return majority_voting_rule(votes), average_rule(ensemble), maximum_rule(ensemble), minimum_rule(ensemble), median_rule(ensemble), product_rule(ensemble)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "F6aFRH3LewdT",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "def get_combine_all(d_test, d_test_cnn, clfs={}):\n",
    "    list_proba = []\n",
    "\n",
    "    for key, clf in clfs.items():\n",
    "        if key[:3]=='CNN':\n",
    "            list_proba.append(clf.predict_proba(d_test_cnn))\n",
    "        else:\n",
    "            list_proba.append(clf.predict_proba(d_test))\n",
    "    \n",
    "    ensemble = np.array(list_proba).transpose((1, 0, 2))\n",
    "    \n",
    "    votes = np.zeros((d_test.shape[0], len(clfs)))\n",
    "    k = 0\n",
    "    for key, clf in clfs.items():    \n",
    "        if key[:3]=='CNN':\n",
    "            votes[:, k] = np.argmax(clf.predict(d_test_cnn), axis=1).reshape(d_test.shape[0])            \n",
    "        else:\n",
    "            votes[:, k] = clf.predict(d_test).reshape(d_test.shape[0])            \n",
    "        k += 1\n",
    "    \n",
    "    return majority_voting_rule(votes), average_rule(ensemble), maximum_rule(ensemble), minimum_rule(ensemble), median_rule(ensemble), product_rule(ensemble)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0lJqzqv1eAGu",
    "colab_type": "text"
   },
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "osZyNtRvUvB_",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "svm = {\n",
    "    'CV': {\n",
    "        'CLF': SVC(random_state=42, verbose=100, kernel='rbf', gamma=0.1, probability=True),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': SVC(random_state=42, verbose=100, kernel='linear', gamma=0.1, probability=True),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': SVC(random_state=42, verbose=100, kernel='rbf', gamma=1, probability=True),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': SVC(random_state=42, verbose=100, kernel='linear', gamma=0.1, probability=True),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': SVC(random_state=42, verbose=100, kernel='linear', gamma=0.1, probability=True),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "5kttfGP7Wq4C",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "svm_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in svm.items():\n",
    "    svm_clfs[ext] = get_classifier(clf['CLF'], train, df_train['norm'], clf['EXT'])\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ve3DdJBwXrM4",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre = {\n",
    "\n",
    "}\n",
    "for ext, clf in svm_clfs.items():\n",
    "    pre['SVM-'+ext] = clf.predict(val)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "GPSYIaW1eXxP",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, labels = calc_delta(df_val['norm'], pre, pre.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ebagSe-cexAz",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "gera_tsne(delta, 'SVM', list(labels), escala=400)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-8M-bIeFgAGA",
    "colab_type": "text"
   },
   "source": [
    "# LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "B-uvzYWAgBSg",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "lr = {\n",
    "    'CV': {\n",
    "        'CLF': LogisticRegression(random_state=42, verbose=100, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': LogisticRegression(random_state=42, verbose=100, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': LogisticRegression(random_state=42, verbose=100, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': LogisticRegression(random_state=42, verbose=100, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': LogisticRegression(random_state=42, verbose=100, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "638ZQmbrgBvQ",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "lr_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in lr.items():\n",
    "    lr_clfs[ext] = get_classifier(clf['CLF'], train, df_train['norm'], clf['EXT'])\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "oInqnv4LgFt8",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_lr = {\n",
    "\n",
    "}\n",
    "for ext, clf in lr_clfs.items():\n",
    "    pre_lr['LR-'+ext] = clf.predict(val)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "vE2Avd0dgNgd",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, labels = calc_delta(df_val['norm'], pre_lr, pre_lr.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "wL5xsliqgjAC",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "gera_tsne(delta, 'LogisticRegression', list(labels), escala=300)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "60kEPa9EhUlm",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre.update(pre_lr)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-sDXVT4ehmi8",
    "colab_type": "text"
   },
   "source": [
    "# RandomForest"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Kv1PjOulhhwC",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "rf = {\n",
    "    'CV': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, verbose=100, n_estimators=20),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, verbose=100, n_estimators=50),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, verbose=100, n_estimators=50),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, verbose=100, n_estimators=20),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, verbose=100, n_estimators=50),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "LvYypgBgh239",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "rf_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in rf.items():\n",
    "    rf_clfs[ext] = get_classifier(clf['CLF'], train, df_train['norm'], clf['EXT'])\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "FPeIOoZJh7Fd",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_rf = {\n",
    "\n",
    "}\n",
    "for ext, clf in rf_clfs.items():\n",
    "    pre_rf['RF-'+ext] = clf.predict(val)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "msmq4AM9iAoK",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, labels = calc_delta(df_val['norm'], pre_rf, pre_rf.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "oYEC6bpAiEug",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "gera_tsne(delta, 'RandomForest', list(labels), escala=300)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "D7p6Lo2diH0o",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre.update(pre_rf)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rqjoNKjNiPMF",
    "colab_type": "text"
   },
   "source": [
    "# NaiveBayes"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Py7MPTQziQkk",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "nb = {\n",
    "    'CV': {\n",
    "        'CLF': MultinomialNB(alpha=1, fit_prior=True),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': MultinomialNB(alpha=0.1, fit_prior=True),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': BernoulliNB(alpha=0.1, fit_prior=True),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': BernoulliNB(alpha=0.1, fit_prior=True),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': BernoulliNB(alpha=1, fit_prior=True),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "EFM6vWcXinx1",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "nb_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in nb.items():\n",
    "    nb_clfs[ext] = get_classifier(clf['CLF'], train, df_train['norm'], clf['EXT'])\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "d6Egk7_5iseR",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_nb = {\n",
    "\n",
    "}\n",
    "for ext, clf in nb_clfs.items():\n",
    "    pre_nb['NB-'+ext] = clf.predict(val)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "me3Q71VSi1sx",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, labels = calc_delta(df_val['norm'], pre_nb, pre_nb.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "X1ewhlhdi_ER",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "gera_tsne(delta, 'NaiveBayes', list(labels), escala=350)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "jvXLLTDijCOy",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre.update(pre_nb)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bADgN3QvjQcU",
    "colab_type": "text"
   },
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "uYtF4P3pjFmP",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "mlp = {\n",
    "    'CV': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=20, max_iter=20, verbose=100, activation='relu', solver='lbfgs'),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=20, max_iter=20, verbose=100, activation='relu', solver='lbfgs'),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=20, max_iter=20, verbose=100, activation='relu', solver='adam'),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=20, max_iter=20, verbose=100, activation='relu', solver='adam'),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=20, max_iter=20, verbose=100, activation='relu', solver='adam'),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mXWz0oPhjlfh",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "mlp_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in mlp.items():\n",
    "    mlp_clfs[ext] = get_classifier(clf['CLF'], train, df_train['norm'], clf['EXT'])\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "1esFI2Tdjp2v",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_mlp = {\n",
    "\n",
    "}\n",
    "for ext, clf in mlp_clfs.items():\n",
    "    pre_mlp['MLP-'+ext] = clf.predict(val)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "t30ark_1jvE8",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, labels = calc_delta(df_val['norm'], pre_mlp, pre_mlp.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "87MJpG_Tjyb1",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "gera_tsne(delta, 'MLP', list(labels), escala=400)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Jth3maC5j1ZF",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre.update(pre_mlp)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "r7qV51yUB8fc",
    "colab_type": "text"
   },
   "source": [
    "# ExtraTrees"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "uePbrkFJB-tm",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "extra = {\n",
    "    'CV': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, verbose=100, n_estimators=20),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, verbose=100, n_estimators=20),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, verbose=100, n_estimators=50),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, verbose=100, n_estimators=50),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, verbose=100, n_estimators=50),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "hVnG6LtKkBGO",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "extra_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in extra.items():\n",
    "    extra_clfs[ext] = get_classifier(clf['CLF'], train, df_train['norm'], clf['EXT'])\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "qTA_Ao8lCzxP",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_extra = {\n",
    "\n",
    "}\n",
    "for ext, clf in extra_clfs.items():\n",
    "    pre_extra['EXTRA-'+ext] = clf.predict(val)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "OEf0UtSDCwIh",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, labels = calc_delta(df_val['norm'], pre_extra, pre_extra.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "_BIkfHuuDBys",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "gera_tsne(delta, 'EXTRATREES', list(labels), escala=400)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "hvvb-kgZDEtf",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre.update(pre_extra)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rcX82GBrE1XA",
    "colab_type": "text"
   },
   "source": [
    "# KNN"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Dzqc-SqYE2ae",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "knn = {\n",
    "    'CV': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=5, algorithm='auto'),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=3, algorithm='auto'),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=5, algorithm='auto'),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=5, algorithm='auto'),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=5, algorithm='auto'),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "XkFX34wYFYSq",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "knn_clfs = { }\n",
    "for ext, clf in knn.items():\n",
    "    knn_clfs[ext] = get_classifier(clf['CLF'], train, df_train['norm'], clf['EXT'])"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "tSTzqVOMFd2i",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_knn = { }\n",
    "for ext, clf in knn_clfs.items():\n",
    "    pre_knn['KNN-'+ext] = clf.predict(val)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "R7QJK01-Fl-M",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, labels = calc_delta(df_val['norm'], pre_knn, pre_knn.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "twWncWeEFr6K",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "gera_tsne(delta, 'KNN', list(labels), escala=600)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Yv6kEPI6q-mf",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre.update(pre_knn)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BZah9uMIGTEF",
    "colab_type": "text"
   },
   "source": [
    "# CNN"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "1YsOFwIPLXSB",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "MAX_NB_WORDS = 20000\n",
    "MAX_SEQUENCE_LENGTH=300\n",
    "\n",
    "y_train = to_categorical(df_train['norm'])\n",
    "y_val = to_categorical(df_val['norm'])"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mYtIRDDAOy0K",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(train)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Sc2mZae_OSyx",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "seq_train = tokenizer.texts_to_sequences(train)\n",
    "seq_val = tokenizer.texts_to_sequences(val)\n",
    "seq_test = tokenizer.texts_to_sequences(test)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "AgNORNP1OUw_",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "data_train = pad_sequences(seq_train, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "data_val = pad_sequences(seq_val, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "data_test = pad_sequences(seq_test, maxlen=MAX_SEQUENCE_LENGTH)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "RHDmm1rRNPb-",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_cv = get_CNN(cv, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, activation='sigmoid')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "sUdMQOOiOFBZ",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_cv.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=20)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "UblW-NhUPKw8",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_tfidf = get_CNN(tfidf, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, activation='softmax')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "8az0g6xFP8j2",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_tfidf.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=200)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "cfIHzRg3QB5-",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_w2v = get_CNN(w2v, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, activation='sigmoid', word_embedding=True)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "FH4PoJ7dQdZ7",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_w2v.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=200)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "26ZolVDnRP_m",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_glove = get_CNN(glove, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=25, activation='sigmoid', word_embedding=True)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "YGDEprwaRXQl",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_glove.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=200)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "9CWqFsbpRbka",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_fast = get_CNN(fasttext, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, activation='sigmoid', word_embedding=True)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mBWlvlH5RoKd",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_fast.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=200)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "M-gnBDLyRrSE",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_cnn = {\n",
    "    'CNN-CV': np.argmax(cnn_cv.predict(data_val), axis=1),\n",
    "    'CNN-TFIDF': np.argmax(cnn_tfidf.predict(data_val), axis=1),\n",
    "    'CNN-W2V': np.argmax(cnn_w2v.predict(data_val), axis=1),\n",
    "    'CNN-GLOVE': np.argmax(cnn_glove.predict(data_val), axis=1),\n",
    "    'CNN-FAST': np.argmax(cnn_fast.predict(data_val), axis=1),\n",
    "}"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "DlBlGac5R6n9",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, labels = calc_delta(df_val['norm'], pre_cnn, pre_cnn.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "wlDBeP3pSWuP",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "gera_tsne(delta, 'CNN', list(labels), escala=300)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "sWz1wHJkSeuG",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre.update(pre_cnn)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3thDFXCGo1AE",
    "colab_type": "text"
   },
   "source": [
    "# PREDICT TEST"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "MN75qpvwo2i-",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_test = {}\n",
    "\n",
    "for ext, clf in svm_clfs.items():\n",
    "    pre_test['SVM-'+ext] = clf.predict(test)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "WoSYVCqSpj0m",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "for ext, clf in lr_clfs.items():\n",
    "    pre_test['LR-'+ext] = clf.predict(test)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "oyShoq_vprPV",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "for ext, clf in rf_clfs.items():\n",
    "    pre_test['RF-'+ext] = clf.predict(test)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "QJXk393yptuW",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "for ext, clf in nb_clfs.items():\n",
    "    pre_test['NB-'+ext] = clf.predict(test)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "0m4AawQrpyeV",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "for ext, clf in mlp_clfs.items():\n",
    "    pre_test['MLP-'+ext] = clf.predict(test)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "J8xIyAyXp1Rq",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "for ext, clf in extra_clfs.items():\n",
    "    pre_test['EXTRA-'+ext] = clf.predict(test)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "m_tl1wh0p3Uy",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "for ext, clf in knn_clfs.items():\n",
    "    pre_test['KNN-'+ext] = clf.predict(test)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "y7lZKQXvp5eU",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "pre_test['CNN-CV'] = np.argmax(cnn_cv.predict(data_test), axis=1)\n",
    "pre_test['CNN-TFIDF'] = np.argmax(cnn_tfidf.predict(data_test), axis=1)\n",
    "pre_test['CNN-W2V'] = np.argmax(cnn_w2v.predict(data_test), axis=1)\n",
    "pre_test['CNN-GLOVE'] = np.argmax(cnn_glove.predict(data_test), axis=1)\n",
    "pre_test['CNN-FAST'] = np.argmax(cnn_fast.predict(data_test), axis=1)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "euKKlCZXB2xw",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "k = 0\n",
    "for clf, pre_clf in pre_test.items():\n",
    "    print(clf)\n",
    "    print(f1_score(class_test, pre_clf, average='macro'))\n",
    "    print(accuracy_score(class_test, pre_clf))\n",
    "    k += 1\n",
    "    if k % 5 == 0:\n",
    "        print(\"\")\n",
    "    "
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Az-Rw-clqPtU",
    "colab_type": "text"
   },
   "source": [
    "# ORACLE"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "MT3R88LxqRJ3",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['SVM-CV', 'SVM-TFIDF', 'SVM-W2V', 'SVM-GLOVE', 'SVM-FAST']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "oLkabHCZqMyJ",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['LR-CV', 'LR-TFIDF', 'LR-W2V', 'LR-GLOVE', 'LR-FAST']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "P1UHO-kQsIqf",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['RF-CV', 'RF-TFIDF', 'RF-W2V', 'RF-GLOVE', 'RF-FAST']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "_clGW9dis4wk",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['NB-CV', 'NB-TFIDF', 'NB-W2V', 'NB-GLOVE', 'NB-FAST']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "eeTjVRDxs8ll",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['MLP-CV', 'MLP-TFIDF', 'MLP-W2V', 'MLP-GLOVE', 'MLP-FAST']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "RpWfFTUMtAJ4",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['EXTRA-CV', 'EXTRA-TFIDF', 'EXTRA-W2V', 'EXTRA-GLOVE', 'EXTRA-FAST']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "TyjokiKutFGq",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['KNN-CV', 'KNN-TFIDF', 'KNN-W2V', 'KNN-GLOVE', 'KNN-FAST']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "nS2agY4qtI6p",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['CNN-CV', 'CNN-TFIDF', 'CNN-W2V', 'CNN-GLOVE', 'CNN-FAST']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TBmJnHDltRLU",
    "colab_type": "text"
   },
   "source": [
    "# TSNE-FULL"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "dXSbDtpZtNdO",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "delta, cols = calc_delta(df_val['norm'], pre, pre.keys())"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "q1oaGeYhueD9",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cores = {}\n",
    "cores[0] = '#DAA520'\n",
    "cores[1] = '#FF0000'\n",
    "cores[2] = '#0000FF'\n",
    "cores[3] = '#228B22'\n",
    "cores[4] = '#000000'\n",
    "\n",
    "simbolos = {}\n",
    "simbolos[0] = 'X'\n",
    "simbolos[1] = 'd'\n",
    "simbolos[2] = '*'\n",
    "simbolos[3] = \"^\"\n",
    "simbolos[4] = 'o'\n",
    "\n",
    "tsne_full(list(cols), delta, 'Todos', cores=cores, simbolos=simbolos, x_ini=-70, x_fim=80, y_ini=-70, y_fim=120, size=10, size_dot=70, per=25)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FH2cEv8PwlU8",
    "colab_type": "text"
   },
   "source": [
    "# Oracle Combinados"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "dHvV5DK8LFVj",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "# TODOS\n",
    "clfs = list(pre.keys())\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "DefuGr4Dwm3m",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "# EXTERNOS\n",
    "clfs = ['NB-GLOVE', 'SVM-FAST', 'CNN-GLOVE', 'NB-FAST', 'EXTRA-TFIDF']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "9ImDessUw3YD",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs = ['LR-W2V','SVM-W2V', 'KNN-FAST', 'LR-FAST', 'CNN-TFIDF']\n",
    "\n",
    "oracle, df_yes = get_oracle(df_test['norm'], pre_test, clfs)\n",
    "\n",
    "print(\"{} de {}\".format(oracle, df_test.shape[0]))\n",
    "print('Acc: ', accuracy_score(df_test['norm'], df_yes))\n",
    "print('F1: ', f1_score(df_test['norm'], df_yes, average='macro'))"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cbcEiqVexr33",
    "colab_type": "text"
   },
   "source": [
    "# VENN"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "9NDrEAToxsw8",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "diagrama_venn(pre_test['SVM-CV'], pre_test['EXTRA-GLOVE'], pre_test['MLP-W2V'], 'LR-CV', 'NB-W2V', 'RF-GLOVE', 'Conjunto 1')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "glnRdDtRz9ej",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "diagrama_venn(pre_test['EXTRA-CV'], pre_test['CNN-W2V'], pre_test['EXTRA-FAST'], 'EXTRA-TFIDF', 'RF-W2V', 'LR-CV', 'Conjunto 2')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "rhhLbLQv5t4k",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "diagrama_venn2(df_test['norm'], pre_test['KNN-TFIDF'], pre_test['SVM-GLOVE'], 'KNN-TFIDF', 'SVM-GLOVE', 'Conjunto 3')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "hOgRoMrE6OOT",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "diagrama_venn2(df_test['norm'], pre_test['KNN-TFIDF'], pre_test['MLP-CV'], 'KNN-TFIDF', 'MLP-CV', 'Conjunto 4')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7Ksstwwr8waY",
    "colab_type": "text"
   },
   "source": [
    "# BAR\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "BCeVM4sw8-BG",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "list_certos = {\n",
    "    5: 0, 4: 0, 3: 0, 2:0, 1:0, 0:0\n",
    "}\n",
    "EXTS = ['CV','TFIDF', 'W2V','GLOVE', 'FAST']"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "q9PfYSWA9Uhx",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "svm_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for ext in EXTS:\n",
    "        if pre_test['SVM-'+ext][i]==l:\n",
    "            k += 1\n",
    "    svm_certos[k] = svm_certos[k]+1\n",
    "\n",
    "render_bar(svm_certos, title='SVM')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Z2_Z5DGT9_sH",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "lr_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for ext in EXTS:\n",
    "        if pre_test['LR-'+ext][i]==l:\n",
    "            k += 1\n",
    "    lr_certos[k] = lr_certos[k]+1\n",
    "\n",
    "render_bar(lr_certos, title='LogisticRegression')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "jY1Qws_P9WbD",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "rf_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for ext in EXTS:\n",
    "        if pre_test['RF-'+ext][i]==l:\n",
    "            k += 1\n",
    "    rf_certos[k] = rf_certos[k]+1\n",
    "\n",
    "render_bar(rf_certos, title='RandomForest')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "pggmZleZ_4vP",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "nb_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for ext in EXTS:\n",
    "        if pre_test['NB-'+ext][i]==l:\n",
    "            k += 1\n",
    "    nb_certos[k] = nb_certos[k]+1\n",
    "\n",
    "render_bar(nb_certos, title='NaiveBayes')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "LNOV-YK-AA6X",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "mlp_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for ext in EXTS:\n",
    "        if pre_test['MLP-'+ext][i]==l:\n",
    "            k += 1\n",
    "    mlp_certos[k] = mlp_certos[k]+1\n",
    "\n",
    "render_bar(mlp_certos, title='MLP')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "TWD0UWrJAIY0",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "extra_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for ext in EXTS:\n",
    "        if pre_test['EXTRA-'+ext][i]==l:\n",
    "            k += 1\n",
    "    extra_certos[k] = extra_certos[k]+1\n",
    "\n",
    "render_bar(extra_certos, title='ExtraTrees')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "60uO4nmpAXAT",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "knn_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for ext in EXTS:\n",
    "        if pre_test['KNN-'+ext][i]==l:\n",
    "            k += 1\n",
    "    knn_certos[k] = knn_certos[k]+1\n",
    "\n",
    "render_bar(knn_certos, title='KNN')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "jHLPbvPKAkrP",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for ext in EXTS:\n",
    "        if pre_test['CNN-'+ext][i]==l:\n",
    "            k += 1\n",
    "    cnn_certos[k] = cnn_certos[k]+1\n",
    "\n",
    "render_bar(cnn_certos, title='CNN')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "sMiYx9FnA1vy",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs_combinados = ['NB-GLOVE', 'SVM-FAST', 'CNN-GLOVE', 'NB-FAST', 'EXTRA-TFIDF']\n",
    "clfs_combinados_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for clf in clfs_combinados:\n",
    "        if pre_test[clf][i]==l:\n",
    "            k += 1\n",
    "    clfs_combinados_certos[k] = clfs_combinados_certos[k]+1\n",
    "\n",
    "render_bar(clfs_combinados_certos, title='Combinados (B)')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "l2tgYCTzl9Vv",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "clfs_combinados = ['LR-W2V','SVM-W2V', 'KNN-FAST', 'LR-FAST', 'CNN-TFIDF']\n",
    "clfs_combinados_certos = list_certos.copy()\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for clf in clfs_combinados:\n",
    "        if pre_test[clf][i]==l:\n",
    "            k += 1\n",
    "    clfs_combinados_certos[k] = clfs_combinados_certos[k]+1\n",
    "\n",
    "render_bar(clfs_combinados_certos, title='Combinados (C)')"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ILJ_cc9RbwPj",
    "colab_type": "code",
    "outputId": "36fb72fa-ad0d-4dad-dcd2-93d48376e8ac",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 386
    }
   },
   "source": [
    "clfs_todos = list(pre.keys())\n",
    "clfs_todos_certos = {k:0 for k in range(len(clfs_todos), -1, -1)}\n",
    "for i, l in enumerate(df_test['norm']):\n",
    "    k = 0\n",
    "    for clf in clfs_todos:\n",
    "        if pre_test[clf][i]==l:\n",
    "            k += 1\n",
    "    clfs_todos_certos[k] = clfs_todos_certos[k]+1\n",
    "render_bar(clfs_todos_certos, title='Todos', size_x=5, size_y=10)"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ],
     "name": "stderr"
    },
    {
     "output_type": "display_data",
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAFgCAYAAACmDI9oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde3wU9b3/8deHu5WbF4JALKBgA0GI\nQkFOkRY4WAUEuRSwaQuIP6ut9VpaTilVUAREW7HQes4R5KIHlIvEgiLKRdGqFDUgIgjVWIKIXOSq\nQIDv74+ZpEOyu9nNZrMJeT8fjzzY/c58dj4z+2Xns7PfmTHnHCIiIiIi4qmS7ARERERERMoTFcgi\nIiIiIgEqkEVEREREAlQgi4iIiIgEqEAWEREREQlQgSwiIiIiEqACWUQSxszuN7OnE/j6H5rZD/zH\nZmZPmdlXZrbOzK42s60JWOYsM3uwtF+3ojKzX5jZPjNrUEqv9zsze7I0XqssmNmkipSviERHBbKI\nxMXMfmxm683siJntMrOXzKxLWSzbOZfunFvjP+0C9ARSnXMdnXNrnXPfKYs8yjMzG25mb5TC6+SY\n2X8WajsPGA78HHgg3mUAOOcecs7dXBqvFYpfgB/x/46Z2anA8w8TtVwRqVhUIItIiZnZPcBjwENA\nQ+DbwF+AfklIpymQ45w7moRlF8vMqiU7hwRoAdzunFsI/MPMaiQ7oeL4BXht51xt4Fbgrfznzrn0\nZOcnIuWDCmQRKREzqweMB37pnFvsnDvqnMtzzv3NOTcqTMwCM/vCzA6a2etmlh6Y1svMNpvZYTPb\naWa/9tsvNLOlZnbAzPab2Vozq+JPyzGz/zSzkcCTQGf/SOA4M/uBmeUGXv9iM1tsZnv8IQHT/PZL\nzWyV37bXzJ4xs/qBuCvM7D0/r2eBWoXW6f+Z2XY/txfMrHFgmjOzX5rZNmCb35ZmZq/48281s8HF\nbYMQ2/GvZrYo8Hyyma00M4vifcsxs1+b2Ub/fXjWzGpF2tZmNhfvy8/f/O37G//lfgO8YGYHgWFA\ny8ByZpnZdDNb5q/PO2Z2aWB6emA77Daz3/ntZwzLKUmfiZeZfd9/zw+a2dtm9t3AtBZm9qa/zJeA\n8wrFDvRzOmBmr5pZcJuMNe9XlkNm9pGZXV0a+YpI6VOBLCIl1RmvWHw+hpiX8IqoFOA94JnAtBnA\nz51zdYA2wCq//V4gF2iAd5T6d4ALvqhzbgZnHg28LzjdzKoCS4HPgGZAE2B+/mRgItAYaAVcDNzv\nx9UAlgBzgfOBBcDAwOt292MHA438189/3Xw3AJ2A1mZ2LvAK8H/+NhgK/MXMWhezDQq7F7jcvOET\nVwMjgWHOORdm/sIGA9cCzYG2eMMk8l+3yLZ2zv0U+Bdwvb99H/bnj/R+4q/fOLwicjswAcDM6gCv\nAsvxtnsLYGWYXEvSZ/AL1JiH+phZCvA3YBJwAfAE8KKZ1fO/gCwAXvenPQL8NBB7OTAL+IWf72tA\nlplVM7N2wAggA6gH9Mbb1iJSDqlAFpGSugDY65w7GW2Ac26mc+6wc+44XhHazrwj0QB5eEVkXefc\nV8659wLtjYCm/hHqtTEUgvk64hVio/wj3cecc2/4OW13zr3inDvunNsD/BH4vh93FVAdeMxf9kLg\nH4HXzQRmOufe89fpv/COYjcLzDPRObffOfcN0AdvGMhTzrmTzrn3gUXAj4rZBoW349d4hdkfgaeB\nXznnYim2HnfOfe6c249XDGYElh/1ti7m/QR43jm3zu8jzwSW0wf4wjn3qP9eHHbOvVOCZYTdXs65\n+vnvcYz6AdnOuef892gWXiF7HV6h3goY55w74ZxbiVfk5xvqr/Ma59wJvKFHDYAOwEngHKA1UNU5\n94lz7tMS5CciZUAFsoiU1D7gQotybK2ZVTXvjP9/mtkhIMefdKH/70CgF/CZmb1mZp399il4Rx9X\nmNknZja6BLleDHwWqpg3s4ZmNt//if4QXsGZn1NjYGehIvGzwOPGwefOuSN426VJYJ4dgcdNgU7+\n0c0DZnYAr8i+yJ8ebhsU4ReUn+AdAX8uwrqH8kXg8ddAbf9x1Ns6ivcz0nIuBv5ZXJJx9Jl4nPGe\n+j7De08bA3ucc8cKTQsZ65w7BewEmjjnPgRG4x1F/9K8oTwNSyFfEUkAFcgiUlJvAcfxhhBE48d4\nR+f+E+8n5mZ+uwE45/7hnOuH99P0Evyizz96eK9z7hKgL3CPmfWIMdcdwLfDFPMP4Q3ZuNw5Vxf4\nSX5OwC6gSaGxvd8OPP4cr+j1VsQbQnEBXlGUL1hc7wBe849u5v/Vds7dFmkbhGJmvwRq+jn8Jtx8\nsShmWxc+khzx/SzGDuCSKOYrUZ+J0xnvqe/beO/pLrwvhbUKTQsZ6w/taeLH4pyb7Zz7D7x1rwXo\ncoEi5ZQKZBEpEefcQeAPwHQzu8HMvmVm1c3sOjN7OERIHbyCeh/wLbzCFPDG+ppZppnVc87lAYeA\n0/60Pv6JUQYcBE7lT4vBOrziZpKZnWtmtczse4G8jgAHzawJEDzB8C28n8bv8NdtAN5wjXzzgBFm\nlmFmNf11esc5lxMmj6XAZWb2U//1qpvZd82sVaRtUJiZXYZXXP0Eb6jFb8wsI9S8sShmW+/mzKI2\n7PsZhaVAIzO7y8xqmlkdM+sUYr4S9Zk4vQBcYWaD/LHDP8Mrgl8CPga2AmP95XfDG8ud71mgv5l1\nNbPqeEeM9wHrzay1eSf/1QS+8f9KI18RSQAVyCJSYs65R4F7gN8De/CODN6OdzSvsDl4Pz/vBDYD\nbxea/lMgx/8p/Va8oQfgjft8Fa+IfQv4i3NudYx5ngKuxzsZ7F94Y0qH+JPHAVfiFYTLgMWBuBPA\nALyT2Pb7McHprwJj8cYR7wIuxRuHGi6Pw8A1/jyf4w1BmIx3JDjSNijgHwV/GpjsnNvgnNuGdzLd\nXL/4ikekbT0R+L0/NOTXFP9+huVvh55478kXeFf46BZi1pL2Gcy72kbMV4lwzu3GO3o+Bq+4vR3o\n45w76A+1Geznuh/vyP3TgdiNeCdM/jfe/4ceQD9/aM85wKPAXry+Uhuv74hIOWSxn+siIiIiInL2\n0hFkEREREZEAFcgiIiIiIgEqkEVEREREAlQgi4iIiIgERHWB//LqwgsvdM2aNUt2GiIiIiJSAb37\n7rt7nXMNCrdX6AK5WbNmrF+/PtlpiIiIiEgFZGaF75wJaIhFVE6dOsUVV1xBnz59ALj66qvJyMgg\nIyODxo0bc8MN3o3E1qxZQ7169QqmjR8/Pplpi4iIiEgJVOgjyGVl6tSptGrVikOHDgGwdu3agmkD\nBw6kX79+Bc+vvvpqli5dWuY5ioiIiEjp0BHkYuTm5rJs2TJuvvnmItMOHTrEqlWrCo4gi4iIiEjF\npyPIxbjrrrt4+OGHOXz4cJFpS5YsoUePHtStW7eg7a233qJdu3Y0btyYRx55hPT09LJMV0RERKRS\nysvLIzc3l2PHjhWZVqtWLVJTU6levXpUr6UCOYKlS5eSkpJC+/btWbNmTZHp8+bNO+PI8pVXXsln\nn31G7dq1efHFF7nhhhvYtm1bGWYsIiIiUjnl5uZSp04dmjVrhpkVtDvn2LdvH7m5uTRv3jyq19IQ\niwjefPNNXnjhBZo1a8bQoUNZtWoVP/nJTwDYu3cv69ato3fv3gXz161bl9q1awPQq1cv8vLy2Lt3\nb1JyFxEREalMjh07xgUXXHBGcQxgZlxwwQUhjyyHowI5gokTJ5Kbm0tOTg7z58+ne/fuPP300wAs\nXLiQPn36UKtWrYL5v/jiC5xzAKxbt47Tp09zwQUXJCV3ERERkcqmcHFcXHs4GmJRQvPnz2f06NFn\ntC1cuJC//vWvVKtWjXPOOYf58+fH/IaIiIiISHJZ/hHPiqhDhw5ONwoRERERkY8++ohWrVrFNN3M\n3nXOdSg8r4ZYiIiIiMhZIdyB31gPCKtAFhEREZEKr1atWuzbt69IMZx/FYvgeWPF0RhkEREREanw\nUlNTyc3NZc+ePUWm5V8HOVoqkEug2ehlRdpyJvUOMaeIiIiIlIXq1atHfZ3j4miIhYiIiIhIgApk\nEREREZEAFcgiIiIiIgEqkEVEREREAlQgi4iIiIgEqEAWEREREQlQgSwiIiIiEqACWUREREQkQAWy\niIiIiEiACmQRERERkQAVyCIiIiIiASqQRUREREQCVCCLiIiIiASoQBYRERERCVCBLCIiIiISoAJZ\nRERERCRABbKIiIiISIAKZBERERGRABXIIiIiIiIBKpBFRERERAJUIIuIiIiIBKhAFhEREREJUIEs\nIiIiIhKgAllEREREJEAFsoiIiIhIgApkEREREZEAFcgiIiIiIgEqkEVEREREAlQgi4iIiIgEqEAW\nEREREQlQgSwiIiIiEqACWUREREQkQAWyiIiIiEiACmQRERERkQAVyCIiIiIiASqQRUREREQCVCCL\niIiIiASoQBYRERERCVCBLCIiIiISoAJZRERERCQg4QWymVU1s/fNbKn/vLmZvWNm283sWTOr4bfX\n9J9v96c3S3RuIiIiIiKFlcUR5DuBjwLPJwN/cs61AL4CRvrtI4Gv/PY/+fOJiIiIiJSphBbIZpYK\n9Aae9J8b0B1Y6M8yG7jBf9zPf44/vYc/v4iIiIhImUn0EeTHgN8Ap/3nFwAHnHMn/ee5QBP/cRNg\nB4A//aA//xnM7BYzW29m6/fs2ZPI3EVERESkEkpYgWxmfYAvnXPvlubrOuf+xznXwTnXoUGDBqX5\n0iIiIiIiVEvga38P6GtmvYBaQF1gKlDfzKr5R4lTgZ3+/DuBi4FcM6sG1AP2JTA/EREREZEiEnYE\n2Tn3X865VOdcM2AosMo5lwmsBgb5sw0DsvzHL/jP8aevcs65ROUnIiIiIhJKMq6D/FvgHjPbjjfG\neIbfPgO4wG+/BxidhNxEREREpJJL5BCLAs65NcAa//EnQMcQ8xwDflQW+YiIiIiIhKM76YmIiIiI\nBKhAFhEREREJUIEsIiIiIhKgAllEREREJEAFsoiIiIhIgApkEREREZEAFcgiIiIiIgEqkEVERERE\nAlQgi4iIiIgEqEBOkGPHjtGxY0fatWtHeno69913HwDDhw+nefPmZGRkkJGRQXZ2NgBZWVm0bduW\njIwMOnTowBtvvJHM9EVEREQqrTK51XRlVLNmTVatWkXt2rXJy8ujS5cuXHfddQBMmTKFQYMGnTF/\njx496Nu3L2bGxo0bGTx4MFu2bElG6iIiIiKVmgrkBDEzateuDUBeXh55eXmYWdj58+cFOHr0aMR5\nRURERCRxNMQigU6dOkVGRgYpKSn07NmTTp06ATBmzBjatm3L3XffzfHjxwvmf/7550lLS6N3797M\nnDkzWWmLiIiIVGoqkBOoatWqZGdnk5uby7p169i0aRMTJ05ky5Yt/OMf/2D//v1Mnjy5YP7+/fuz\nZcsWlixZwtixY5OYuYiIiEjlpQK5DNSvX59u3bqxfPlyGjVqhJlRs2ZNRowYwbp164rM37VrVz75\n5BP27t2bhGxFREREKjcVyAmyZ88eDhw4AMA333zDK6+8QlpaGrt27QLAOceSJUto06YNANu3b8c5\nB8B7773H8ePHueCCC5KTvIiIiEglppP0EmTXrl0MGzaMU6dOcfr0aQYPHkyfPn3o3r07e/bswTlH\nRkYGTzzxBACLFi1izpw5VK9enXPOOYdnn31WJ+qJiIiIJIHlH7WsiDp06ODWr19f5sttNnpZkbac\nSb3LPA8RERERKTkze9c516Fwu4ZYiIiIiIgEqEAWEREREQlQgSwiIiIiEqACWUREREQkQFexKGM6\nwU9ERESkfNMRZBERERGRABXIIiIiIiIBKpBFRERERAJUIIuIiIiIBKhAFhEREREJUIEsIiIiIhKg\nAllEREREJEAFsoiIiIhIgApkEREREZEAFcgiIiIiIgEqkEVEREREAlQgi4iIiIgEqEAWEREREQlQ\ngSwiIiIiEqACWUREREQkQAWyiIiIiEiACmQRERERkQAVyCIiIiIiASqQRUREREQCVCCLiIiIiASo\nQBYRERERCVCBXA7t2LGDbt260bp1a9LT05k6dSoAGzZsoHPnzlx++eVcf/31HDp06Iy4f/3rX9Su\nXZtHHnkkGWmLiIiInBVUIJdD1apV49FHH2Xz5s28/fbbTJ8+nc2bN3PzzTczadIkPvjgA/r378+U\nKVPOiLvnnnu47rrrkpS1iIiIyNlBBXI51KhRI6688koA6tSpQ6tWrdi5cycff/wxXbt2BaBnz54s\nWrSoIGbJkiU0b96c9PT0pOQsIiIicrZQgVzO5eTk8P7779OpUyfS09PJysoCYMGCBezYsQOAI0eO\nMHnyZO67775kpioiIiJyVlCBXI4dOXKEgQMH8thjj1G3bl1mzpzJX/7yF9q3b8/hw4epUaMGAPff\nfz933303tWvXTnLGIiIiIhVftUS9sJnVAl4HavrLWeicu8/MmgPzgQuAd4GfOudOmFlNYA7QHtgH\nDHHO5SQqv/IuLy+PgQMHkpmZyYABAwBIS0tjxYoVAHz88ccsW7YMgHfeeYeFCxfym9/8hgMHDlCl\nShVq1arF7bffnrT8RURERCqqhBXIwHGgu3PuiJlVB94ws5eAe4A/Oefmm9kTwEjgr/6/XznnWpjZ\nUGAyMCSB+ZVbzjlGjhxJq1atuOeeewrav/zyS1JSUjh9+jQPPvggt956KwBr164tmOf++++ndu3a\nKo5FRERESihhQyyc54j/tLr/54DuwEK/fTZwg/+4n/8cf3oPM7NE5Veevfnmm8ydO5dVq1aRkZFB\nRkYGL774IvPmzeOyyy4jLS2Nxo0bM2LEiGSnKiIiInLWSeQRZMysKt4wihbAdOCfwAHn3El/llyg\nif+4CbADwDl30swO4g3D2JvIHMujLl264JwLOe3OO++MGHv//fcnICMRERGRyiOhJ+k550455zKA\nVKAjkBbva5rZLWa23szW79mzJ+4cRURERESCyuQqFs65A8BqoDNQ38zyj1ynAjv9xzuBiwH86fXw\nTtYr/Fr/45zr4Jzr0KBBg4TnLiIiIiKVS8IKZDNrYGb1/cfnAD2Bj/AK5UH+bMOALP/xC/5z/Omr\nXLhxBiIiIiIiCZLIMciNgNn+OOQqwHPOuaVmthmYb2YPAu8DM/z5ZwBzzWw7sB8YmsDcKqRmo5cV\nacuZ1DsJmYiIiIicvRJWIDvnNgJXhGj/BG88cuH2Y8CPEpWPiIiIiEg0dCc9EREREZGAmAtkM6ti\nZnUTkYyIiIiISLJFVSCb2f+ZWV0zOxfYBGw2s1GJTU1EREREpOxFewS5tXPuEN5d714CmgM/TVhW\nIiIiIiJJEm2BXN3MquMVyC845/LwbhstIiIiInJWibZA/m8gBzgXeN3MmgKHEpWUiIiIiEiyRHWZ\nN+fc48DjgabPzKxbYlISEREREUmeqK+DbGa9gXSgVqB5fKlnJCIiIiKSRNFexeIJYAjwK8DwbujR\nNIF5iYiIiIgkRbRjkP/DOfcz4Cvn3DigM3BZ4tISEREREUmOaAvkb/x/vzazxkAe0CgxKYmIiIiI\nJE+0Y5CXmll9YArwHt4l3p5MWFYiIiIiIkkS7VUsHvAfLjKzpUAt59zBxKUlIiIiIpIcEQtkM+vu\nnFtlZgNCTMM5tzhxqYmIiIiIlL3ijiB/H1gFXB9imgNUIIuIiIjIWSVigeycu8//d0TZpCMiIiIi\nklzRXgf5If8kvfzn55nZg4lLS0REREQkOaK9zNt1zrkD+U+cc18BvRKTkoiIiIhI8kRbIFc1s5r5\nT8zsHKBmhPlFRERERCqkaAvkZ4CVZjbSzEYCrwCzE5eWlMSOHTvo1q0brVu3Jj09nalTpwIwduxY\n2rZtS0ZGBtdccw2ff/45AFOmTCEjI4OMjAzatGlD1apV2b9/fzJXQURERCTpoiqQnXOTgQlAK//v\nAefcw4lMTGJXrVo1Hn30UTZv3szbb7/N9OnT2bx5M6NGjWLjxo1kZ2fTp08fxo8fD8CoUaPIzs4m\nOzubiRMn8v3vf5/zzz8/yWshIiIiklzR3kkP59xLwEsJzEXi1KhRIxo18u4AXqdOHVq1asXOnTtp\n3bp1wTxHjx7FzIrEzps3jxtvvLHMchUREREpr6IqkP0bhUwGUgDz/5xzrm4Cc5M45OTk8P7779Op\nUycAxowZw5w5c6hXrx6rV68+Y96vv/6a5cuXM23atGSkKiIiIlKuRDsG+WGgr3OunnOurnOujorj\n8uvIkSMMHDiQxx57jLp1vbdpwoQJ7Nixg8zMzCKF8N/+9je+973vaXiFiIiICNEXyLudcx8lNBMp\nFXl5eQwcOJDMzEwGDChyh3AyMzNZtGjRGW3z58/X8AoRERERX7RjkNeb2bPAEuB4fqNzTreaLkec\nc4wcOZJWrVpxzz33FLRv27aNli1bApCVlUVaWlrBtIMHD/Laa6/x9NNPl3m+IiIiIuVRtAVyXeBr\n4JpAmwNUIJcjb775JnPnzuXyyy8nIyMDgIceeogZM2awdetWqlSpQtOmTXniiScKYp5//nmuueYa\nzj333GSlLSIiIlKuRFUgO+dGJDoRiV+XLl1wzhVp79Ur/E0Phw8fzvDhwxOYlYiIiEjFEu1VLGoB\nI4F0oFZ+u3PupgTlJSIiIiKSFBFP0jOzLmbWH5gLXAT8EHgNSAUOJz49EREREZGyFbZANrP/BwwH\n/ga0cM6NBY4652YDvYFOZZKhiIiIiEgZijTE4gXgSuBa4ITfdsDM2gBf4N00RCqIZqOXFWnLmdQ7\nCZmIiIiIlG9hC2Tn3G7gNjM7B7jIzM4Dfo9XONcGxpZNiiIiIiIiZafYk/Scc9+Y2Urn3FfA68Al\nAGbWPNHJiYiIiIiUtWjvpLcoRNvC0kxERERERKQ8iHgE2czS8C7tVs/Mgvctrkvgcm8iIiIiImeL\n4oZYfAfoA9QHrg+0Hwb+X6KSEhERERFJlogFsnMuC8gys87OubfKKCcRERERkaSJdgxyfzOra2bV\nzWylme0xs58kNDMRERERkSSItkC+xjl3CG+4RQ7QAhiVqKRERERERJIl2gK5uv9vb2CBc+5ggvIR\nEREREUmqYq+D7PubmW0BvsG7eUgD4Fji0hIRERERSY6ojiA750YD/wF0cM7lAUeBfolMTEREREQk\nGaI9ggyQBjQzs2DMnFLOR0REREQkqaIqkM1sLnApkA2c8psdKpBFRERE5CwT7RHkDkBr55xLZDIi\nIiIiIskW7VUsNgEXJTIREREREZHyINojyBcCm81sHXA8v9E51zchWYmIiIiIJEm0BfL9iUxCRERE\nRKS8iKpAds69luhERERERETKg4hjkM3ssJkdCvF32MwOFRN7sZmtNrPNZvahmd3pt59vZq+Y2Tb/\n3/P8djOzx81su5ltNLMrS281RURERESiE7FAds7Vcc7VDfFXxzlXt5jXPgnc65xrDVwF/NLMWgOj\ngZXOuZbASv85wHVAS//vFuCvcayXiIiIiEiJRHsVi5g553Y5597zHx8GPgKa4N2Bb7Y/22zgBv9x\nP2CO87wN1DezRonKT0REREQklIQVyEFm1gy4AngHaOic2+VP+gJo6D9uAuwIhOX6bYVf6xYzW29m\n6/fs2ZOwnEVERESkckp4gWxmtYFFwF3OuTPGLfs3Honp5iPOuf9xznVwznVo0KBBKWYqIiIiIpLg\nAtnMquMVx8845xb7zbvzh074/37pt+8ELg6Ep/ptIiIiIiJlJmEFspkZMAP4yDn3x8CkF4Bh/uNh\nQFag/Wf+1SyuAg4GhmKIiIiIiJSJaG8UUhLfA34KfGBm2X7b74BJwHNmNhL4DBjsT3sR6AVsB74G\nRiQwNxERERGRkBJWIDvn3gAszOQeIeZ3wC8TlY+IiIiISDTK5CoWIiIiIiIVhQpkEREREZEAFcgi\nIiIiIgEqkEVEREREAlQgi4iIiIgEqEAWEREREQlQgSwiIiIiEqACWUREREQkQAWyiIiIiEiACmQR\nERERkQAVyCIiIiIiASqQRUREREQCVCCLiIiIiASoQBYRERERCVCBLCIiIiISoAJZRERERCRABbKI\niIiISIAKZClw0003kZKSQps2bQraxo4dS9u2bcnIyOCaa67h888/B+Crr76if//+tG3blo4dO7Jp\n06ZkpS0iIiJSqlQgS4Hhw4ezfPnyM9pGjRrFxo0byc7Opk+fPowfPx6Ahx56iIyMDDZu3MicOXO4\n8847k5GyiIiISKlTgSwFunbtyvnnn39GW926dQseHz16FDMDYPPmzXTv3h2AtLQ0cnJy2L17d9kl\nKyIiIpIgKpClWGPGjOHiiy/mmWeeKTiC3K5dOxYvXgzAunXr+Oyzz8jNzU1mmiIiIiKlQgWyFGvC\nhAns2LGDzMxMpk2bBsDo0aM5cOAAGRkZ/PnPf+aKK66gatWqSc5UREREJH7Vkp2AVByZmZn06tWL\ncePGUbduXZ566ikAnHM0b96cSy65JMkZioiIiMRPR5Alom3bthU8zsrKIi0tDYADBw5w4sQJAJ58\n8km6du16xnhlERERkYpKR5ClwI033siaNWvYu3cvqampjBs3jhdffJGtW7dSpUoVmjZtyhNPPAHA\nRx99xLBhwzAz0tPTmTFjRpKzFxERESkdKpClwLx584q0jRw5MuS8nTt35uOPP050SiIiIiJlTkMs\nREREREQCVCCLiIiIiASoQBYRERERCVCBLCIiIiISoJP0JKJmo5eFbM+Z1LuMMxEREREpGzqCLCIi\nIiISoAJZRERERCRABbKIiIiISIAKZBERERGRABXIErebbrqJlJQU2rRpU9C2f/9+evbsScuWLenZ\nsydfffUVAAcPHuT666+nXbt2pKen89RTTyUrbREREZGQVCBL3IYPH87y5cvPaJs0aRI9evRg27Zt\n9OjRg0mTJgEwffp0WrduzYYNG1izZg333nsvJ06cSEbaIiIiIiGpQJa4de3alfPPP/+MtqysLIYN\nGwbAsGHDWLJkCQBmxuHDh3HOceTIEc4//3yqVdPVBkVERKT8UGUiCbF7924aNWoEwEUXXcTu3bsB\nuP322+nbty+NGzfm8OHDPKwUMvYAACAASURBVPvss1Spou9pIiIiUn6oMpGEMzPMDICXX36ZjIwM\nPv/8c7Kzs7n99ts5dOhQkjMUERER+TcVyJIQDRs2ZNeuXQDs2rWLlJQUAJ566ikGDBiAmdGiRQua\nN2/Oli1bkpmqiIiIyBlUIEtC9O3bl9mzZwMwe/Zs+vXrB8C3v/1tVq5cCXjDMLZu3coll1yStDxF\nRERECtMYZInbjTfeyJo1a9i7dy+pqamMGzeO0aNHM3jwYGbMmEHTpk157rnnABg7dizDhw/n8ssv\nxznH5MmTufDCC5O8BiIiIiL/pgJZ4jZv3ryQ7flHioMaN27MihUrEp2SiIiISIlpiIWIiIiISIAK\nZBERERGRABXIIiIiIiIBKpBFRERERAJ0kp4kTLPRy4q05UzqnYRMRERERKKXsCPIZjbTzL40s02B\ntvPN7BUz2+b/e57fbmb2uJltN7ONZnZlovISEREREYkkkUMsZgHXFmobDax0zrUEVvrPAa4DWvp/\ntwB/TWBeIiIiIiJhJaxAds69Duwv1NwPmO0/ng3cEGif4zxvA/XNrFGichMRERERCaesT9Jr6Jzb\n5T/+AmjoP24C7AjMl+u3FWFmt5jZejNbv2fPnsRlKiIiIiKVUtKuYuGcc4ArQdz/OOc6OOc6NGjQ\nIAGZiYiIiEhlVtYF8u78oRP+v1/67TuBiwPzpfptIiIiIiJlqqwL5BeAYf7jYUBWoP1n/tUsrgIO\nBoZiiIiIiIiUmYRdB9nM5gE/AC40s1zgPmAS8JyZjQQ+Awb7s78I9AK2A18DIxKVl5RfW7duZciQ\nIQXPP/nkE8aPH8++ffvIysqiSpUqpKSkMGvWLBo3bpzETEVERORslrAC2Tl3Y5hJPULM64BfJioX\nqRi+853vkJ2dDcCpU6do0qQJ/fv357zzzuOBBx4A4PHHH2f8+PE88cQTyUxVREREzmK6k56USytX\nruTSSy+ladOmZ7QfPXoUM0tSViIiIlIZqECWcmn+/PnceOO/f4QYM2YMc+bMoV69eqxevTqJmYmI\niMjZLmmXeRMJ58SJE7zwwgv86Ec/KmibMGECO3bsIDMzk2nTpiUxOxERETnbqUCWcuell17iyiuv\npGHDhkWmZWZmsmjRoiRkJSIiIpWFCmQpd+bNm3fG8Ipt27YVPM7KyiItLS0ZaYmIiEgloQJZypWj\nR4/yyiuvMGDAgIK20aNH06ZNG9q2bcuKFSuYOnVqyNgDBw4waNAg0tLSaNWqFW+99RYLFiwgPT2d\nKlWqsH79+rJaDREREanAdJKelCvnnnsu+/btO6Mt2iEVd955J9deey0LFy7kxIkTfP3119SvX5/F\nixfz85//PBHpioiIyFlIBbKcFQ4ePMjrr7/OrFmzAKhRowY1atSgfv36yU0sgmbNmlGnTh2qVq1K\ntWrVWL9+PUOGDGHr1q2Ad0S8fv36BdeGFhERkbKhAlnOCp9++ikNGjRgxIgRbNiwgfbt2zN16lTO\nPffcZKcW0erVq7nwwgsLnj/77LMFj++9917q1auXjLREREQqNY1BlrPCyZMnee+997jtttt4//33\nOffcc5k0aVKy0yox5xzPPffcGScrioiISNnQEWQpl5qNXlakLWdS77Dzp6amkpqaSqdOnQAYNGhQ\nTAVyqOEOY8eOJSsriypVqpCSksKsWbNo3LhxqcWaGddccw1mxs9//nNuueWWgmlr166lYcOGtGzZ\nMup1EBERkdKhI8hyVrjooou4+OKLC8bvrly5ktatW8f0GqtXryY7O7vgahejRo1i48aNZGdn06dP\nH8aPH1+qsW+88QbvvfceL730EtOnT+f1118vmFb4UnciIiJSdlQgy1njz3/+M5mZmbRt25bs7Gx+\n97vf8fzzz5Oamspbb71F7969+eEPfxj169WtW7fg8dGjRzGzUo1t0qQJACkpKfTv359169YB3nCR\nxYsXM2TIkKiXJyIiIqVHQyzkrJGRkVHkWsf9+/enf//+xcaGG+4wZswY5syZQ7169Vi9enWpxR49\nepTTp09Tp04djh49yooVK/jDH/4AwKuvvkpaWhqpqakxbwMRERGJn44gixB+uMOECRPYsWMHmZmZ\nTJs2rdRid+/eTZcuXWjXrh0dO3akd+/eXHvttQDMnz9fwytERESSSAWyCOGHO+TLzMwMe8OSksRe\ncsklbNiwgQ0bNvDhhx8yZsyYgmmzZs3i1ltvjXudREREpGRUIEuld/ToUQ4fPlzweMWKFbRp04Zt\n27YVzJOVlUVaWlqpxoqIiEj5pDHIctaJ9RJxu3fvLhinfPLkSX784x9z7bXXMnDgQLZu3UqVKlVo\n2rQpTzzxRKnGxuvUqVN06NCBJk2asHTpUj799FOGDh3Kvn37aN++PXPnzqVGjRqlvlwREZGznQpk\nqfTyhzsUFm5IRWnFxmvq1Km0atWKQ4cOAfDb3/6Wu+++m6FDh3LrrbcyY8YMbrvttlJb3rFjx+ja\ntSvHjx/n5MmTDBo0iHHjxrFy5UpGjRrF6dOnqV27NrNmzaJFixZRxa5atYpf//rXnDhxgvbt2zNj\nxgyqVdPHkoiIJJeGWIgkWbPRy0L+RZKbm8uyZcu4+eabAe/Oe6tWrWLQoEEADBs2jCVLlhSJO3bs\nGB07dqRdu3akp6dz3333AXD11VeTkZFBRkYGjRs35oYbbigSW7NmTVatWsWGDRvIzs5m+fLlvP32\n29x2220888wzZGdn8+Mf/5gHH3wwqti///3vDBs2jPnz57Np0yaaNm3K7NmzSy3neNZVRERKT0X8\nPNahGpEK6K677uLhhx8uGP+8b98+6tevX3D0NTU1lZ07dxaJyy9Ua9euTV5eHl26dOG6665j7dq1\nBfMMHDiQfv36FYk1M2rXrg1AXl4eeXl5mBlmVnAU++DBgyHvNhgqtmrVqtSoUYPLLrsMgJ49ezJx\n4kRGjhxZKjnHs66VTbgj/MOHD+e1116jXr16gHcCaUZGRpKzFZGKpiJ+HusIskgFs3TpUlJSUmjf\nvn3MseGK3HyHDh1i1apVYb/Fnzp1ioyMDFJSUujZsyedOnXiySefpFevXqSmpjJ37lxGjx4dVWzH\njh05efJkwbWrFy5cyI4dO0ot53jXtTIJ9+sAwJQpU8jOziY7O1vFcQUT7qhdvjvuuKPg/4hIIlXE\nz2MVyCIBsQ51KK3YWLz55pu88MILNGvWjKFDh7Jq1SruvPNODhw4wMmTJwFvCEb+5ecKC1Xk5luy\nZAk9evQ4406AQVWrViU7O5vc3FzWrVvHpk2b+NOf/sSLL75Ibm4uI0aM4J577okq9sMPP2T+/Pnc\nfffddOzYkTp16lC1atVSzbmkceEKi8zMTL7zne/Qpk0bbrrpJvLy8kLmW9EUt/NKhIq4jcPlPG3a\nNFq0aIGZsXfv3iRn+W+RvvisX7+er776KskZlg8V7X2NV7LWN559TzKoQBapYCZOnEhubi45OTnM\nnz+f7t2788wzz9CtWzcWLlwIwOzZs8P+VBWqyM03b968qG5SUr9+fbp168ZLL73Ehg0bCj7ohgwZ\nwt///veoYpcvX07nzp1Zu3Yt69ato2vXrgXDLUor55LGhSssMjMz2bJlCx988AHffPMNTz75ZMR1\njVW4Hdenn35Kp06daNGiBUOGDOHEiROlGgvhd15jxoyhbdu23H333Rw/frzU1jVZ2zge4XL+3ve+\nx6uvvkrTpk2TneIZwn3xOXXqFKNGjeLhhx9OyHLj7YtlraK9r/FK1vqWxr6nLKlAFjlLTJ48mT/+\n8Y+0aNGCffv2FRnLW1iwUAXYu3cv69ato3fv0JfE27NnDwcOHADgm2++4ZVXXqFVq1YcPHiQjz/+\nGKCgLZrYtLQ0vvzySwCOHz/O5MmTi71BSqw5lzQuXGHRq1evgnHXHTt2JDc3N+JyYxVux5V/hZLt\n27dz3nnnMWPGjFKNhdA7r4kTJ7Jlyxb+8Y9/sH//fiZPnlxq65qsbRyPcDlfccUVNGvWLLnJhRHq\ni8+0adPo27cvjRo1Ssgy4+mLO3bsoFu3brRu3Zr09HSmTp0KwIYNG+jcuTOXX345119/fcF5D6Wh\nIr6v8WyneNa3NN6fkn6OlzUVyCIV2A9+8AOWLl0KeJecW7duHdu3b2fBggXUrFmzyPzhClXwxgD3\n6dOHWrVqhVzWrl276NatG23btuW73/0uPXv2pE+fPvzv//4vAwcOpF27dsydO5cpU6ZEHTtlyhRa\ntWpF27Ztuf766+nevXup5RzPukLknwPz8vKYO3duwe3BS0u4HVc0VyiJJzYouPNq1KgRZkbNmjUZ\nMWJEkbtEhttZDhkypODM9GbNmoUdu1zSbRzvcuMRKefyqPAXn9dff50FCxbwq1/9KmHLjKcvVqtW\njUcffZTNmzfz9ttvM336dDZv3szNN9/MpEmT+OCDD+jfv3/Iz5l4lPR9TVZfjHc7lXR9S7rceD+P\nk0FXsRCpRHbt2sWwYcM4deoUp0+fZvDgwfTp0weA+fPnhz3BDqBt27a8//77Rdr79+9fcLOUWGOn\nTJlS7I6upDnHs67w78LiwIED9O/fn02bNtGmTRsAfvGLX9C1a1euvvrqiK9REqdOnaJ9+/Zs376d\nX/7yl1x66aVRXaEkntg9e/ZQvXp16tevX7Dz+u1vf8uuXbto1KgRzjmWLFlSsP758neWV155JYcP\nH6Z9+/b07NmTZ599tmCee++9t+AqGIWVdBvHu9x4RMq5PMv/4rN69Wq2b99ecK3yr7/+mhYtWrB9\n+/ZSXV5J+2KjRo0KjmzXqVOHVq1asXPnTj7++GO6du0KeFe8+eEPf8gDDzxQavmW9H1NVl+MdzuV\ndH1Lutx4P4+TQQWySCUSrlAFWLNmTdkmE6WS5lxa6xo8otqmTRvGjRvHnj17+O///u+oXyMWhXdc\nW7ZsSXhsuJ1X9+7d2bNnD845MjIyitwRMtzOsnXr1oB3fe7nnnuOVatWRVx+rNu4tJYbj8I5l0fh\nvvh88cUXBfPUrl271ItjiK8f58vJyeH999+nU6dOpKenk5WVxQ033MCCBQtCXvGmNMT6vpaHvhjP\ndoqnH8ey3Iq479EQCxGRQsL9HPjkk0/y8ssvM2/ePKpUSezHZ/6O66233or6CiUljc3feW3cuJFN\nmzbxhz/8AYBVq1bxwQcfsGnTJp5++umIlwQL7izzrV27loYNG9KyZcsi85fWNo51ufGI9DNxccL9\nFL9gwQLS09OpUqVKwSUPSyMOwg9tiiffUaNGkZaWRtu2benfv3/B9ginpP34yJEjDBw4kMcee4y6\ndesyc+ZM/vKXv9C+fXsOHz5MjRo1olqPaMTzvgbF2hdLYxuXZDuVxvqW5fuTLCqQRSqwsrq0XGUT\nrrC49dZb2b17N507dyYjI4Px48cXiQ2309u/fz89e/akZcuW9OzZM+QltsKdCBnNFUriiY1X4Z1l\nvkhnpsezjeNZ7k033URKSsoZR8uiPcEoXM6PP/44qamp5Obm0rZt24I7XAaFG7vZpk0bFi9eXPDz\ndGnFQfgvPoW3YSzL7dmzJ5s2bWLjxo1cdtllTJw4sUhsvH0xLy+PgQMHkpmZyYABAwBIS0tjxYoV\nvPvuu9x4441ceumlYdc7VvG8r/lK0hfj2cZQ8u0U7/qW9fuTLBpiISJlKlQRnzOpfJ29HO7nwPyj\nX5GEG5M4a9YsevTowejRo5k0aRKTJk0qclWIcEMdWrduzdChQ/n973/PFVdcEfIKJfHExiPUzhK8\nbbV48WLefffdkHHxbON4ljt8+HBuv/12fvaznxW03XzzzTzyyCN8//vfZ+bMmUyZMiXkuM1wOd9x\nxx3ccccdEfMN91N8z549ExIXr3DLveaaawrmueqqqwoK3qB4+qJzjpEjR9KqVaszrqn+5ZdfkpKS\nwunTp3nwwQeLveJNLOJ5X6HkfTGebRzPdopnfZPx/iSLCmSRSiqeQjVZRW5JlxvuyHoicg6308vK\nyioYazds2DB+8IMfFCmQw+248q9QEkk8sSUVbmcJ8Oqrr5KWlkZqamq5Wm7Xrl3Jyck5oy3RJ4AV\nFuqn+ETGxSvccmfOnMmQIUOKzB9PX3zzzTeZO3cul19+ecGVHx566CG2bdvG9OnTARgwYAAjRowo\n6eqUqtL6PxDrNk7WdiqN5Zbl53E8VCCLiCRIcKe3e/fugsL5oosuYvfu3UnOLrRYvoSE21n26tWL\n+fPnJ+zC/6W93LI6AQzC/xSfqLh4hVvuhAkTqFatGpmZmaW6vC5duuCcCzntzjvvLNVllYbS6Isl\n2cbJ2k4V7f2JhwpkEZEEiFTQ5N8Io6KLtLOcNWtWhVnuzJkzueOOO3jggQfo27dvwk4wCvdTfKLi\ngkry60u45c6aNYulS5eycuXKctWPb7rpJpYuXUpKSkrBXdrGjh1LVlYWVapUISUlhVmzZtG4ceNS\nW2a8fbGibePKRAWyiEgEpVVYNGzYsOC6wrt27SIlJSUh+VZEyR6Xnn+CEXjDLZYtK/2TXSP9FJ+I\nuHiFW+7y5ct5+OGHee211/jWt75VZvlEI9T48lGjRhUMl3n88ccZP358kcsVJktF3MaViQpkEZFS\nFG6n17dvX2bPns3o0aOjuppERRwjXlGV5ASjWLdxuJ/ijx8/zq9+9Sv27NlD7969ycjI4OWXX447\nLl7hlnvHHXdw/PjxgpMEr7rqqogFZ1n2xVDjy4O/3hw9erTYo7FlOT62tLZxPPRZEZ4KZBGRUhRu\npzd69GgGDx7MjBkzaNq0Kc8991ySM62cbrzxRtasWcPevXtJTU1l3LhxHDlyJOEngEX6KT7SnShL\nGhevcMvt1atXwpZZWGkVb2PGjGHOnDnUq1eP1atXl0ZqpaK0tnFFO2m6olCBLCJSiiIVNCtXrizj\nbMpOss5Mj3UnPW/evJDtZ9sJRvJvEyZMYMKECUycOJFp06Yxbty4hCznbC8YKxvdKERERETOepmZ\nmSxatCjZaUgFoQJZREREzkrbtm0reJyVlVWiW0hL5aQhFiIiIhJWRRk6EGp8+YsvvsjWrVupUqUK\nTZs2LTdXsCisomzjykQFsohIgminJ6FUhDtCVkShxpeX9q3VpfJQgSwiIpWSvsCISDgagywiIiIi\nEqACWUREREQkQEMsRERE5Kyi4TMSLx1BFhEREREJUIEsIiIiIhKgAllEREREJEAFsoiIiIhIQLkq\nkM3sWjPbambbzWx0svMRERERkcqn3BTIZlYVmA5cB7QGbjSz1snNSkREREQqm3JTIAMdge3OuU+c\ncyeA+UC/JOckIiIiIpWMOeeSnQMAZjYIuNY5d7P//KdAJ+fc7YXmuwW4xX/6HWBrmSYa3oXAXsWW\ny2VWttiKlm9FjK1o+VbE2IqWb0WMrWj5VsTYipZvRY2NR1PnXIMirc65cvEHDAKeDDz/KTAt2XnF\nkP96xZbPZVa22IqWb0WMrWj5VsTYipZvRYytaPlWxNiKlm9FjU3EX3kaYrETuDjwPNVvExEREREp\nM+WpQP4H0NLMmptZDWAo8EKScxIRERGRSqZashPI55w7aWa3Ay8DVYGZzrkPk5xWLP5HseV2mZUt\ntqLlWxFjK1q+FTG2ouVbEWMrWr4VMbai5VtRY0tduTlJT0RERESkPChPQyxERERERJJOBbKIiIiI\nSIAK5BIys6pm9r6ZLfWfNzezd/zbZD/rn2hYOKaWma0zsw1m9qGZjfPbu5vZe2a2ycxmm1mRseER\nYnv4sdlm9oaZtYghdq0fl21mn5vZkhhizcwmmNnHZvaRmd0RQ+wsM/s0sOyMaGMD0x83syNh3ptw\ny53ht200s4VmVjvKuGf8W6BvMrOZZlY9hmXe7vcJZ2YXxphvNH3qYjNbbWab/dg7/fZ2ZvaWmX1g\nZn8zs7oxxGaY2dv+e7PezDpGGfds4D3NMbPsaJfpT/uVmW3x2x+OId/7zWxnYNm9YlmuP/3ecO9R\nhOU+4PelbDNbYWaNY4id4q/rRjN73szqRxn3I//5aTPrUHh5xcSeb2avmNk2/9/zwsTPNLMvzWxT\noC2a/hQqLmJfKia22P4ULtZvj9ifIiy32P4Uabn+tLD9KcJyo+lPoeIi9qViYovtTxFio+pPIV7r\nTvM+Sz80s7uiifHj7vZjNpnZPDOrFWXcdwLvZbaZHYpxufXN219sMW9f1zmG2Bz//0y2ma2PNi4Q\nf0adEWVMxH1nMbERPyeLiQ37/yGK2GvN28duN7PRscYnTLKvM1dR/4B7gP8DlvrPnwOG+o+fAG4L\nEWNAbf9xdeAd4D+AHcBlfvt4YGSUsVcBHwOt/PZfALOijS00zyLgZzEsdwQwB6jiT0uJIXYWMKiY\n7Rs2Z6ADMBc4EkssUDcwzx+B0VHG9fKnGTAvhvf2KuAKoBmQA1wYY77R9KlGwJX+4zp+f2iNd1WY\n7/vtNwEPxBC7ArjOb+8FrIkmrtA8jwJ/iGGZ3YBXgZoR+lO42PuBXxfTn8LmjHd5yZeBz0K9RxGW\nG+xPdwBPxBB7DVDNb58MTI4yrhXeDZLWAB1iWVfgYfw+D4wuvMxAfFfgSmBToC2a/hQqLmJfihQb\nTX+KsNxi+1OE2GL7U6Sci+tPEZYbTX8KFRexLxUTW2x/ihAbVX8q9DptgE3At/AuEvAq0CKKuCbA\np8A5/vPngOHFxYV4narAF3g3hog2ZjZws/+4BlA/hticcH0gyvgz6owoY4rd30eILfazPZY+EsN7\n8k/gEn/7boh2mYn+0xHkEjCzVKA38KT/3IDuwEJ/ltnADYXjnCf/qGd1/+8UcMI597Hf/gowMMpY\n5//lH82pB3weQ2z++tT18y9yBDlC7G3AeOfcaX++L2NdbiThYs2sKjAF+E2ssc65Q/76GnBO4Vwi\nxL3oT3PAOrxrdEe7zPedczklWVei61O7nHPv+Y8PAx/h7UwuA173ZwvXp8LFRuxTEeKAgu07GO/L\nRLTLvA2Y5Jw77k8L1Z8iLjeSYmL/hNefQvbNcLH5/cl3bqj4CLErnHMn/dneplCfihD3kXMu4t1D\nI6xrP7x+BGH6kx/zOrC/UHM0/SlUXLGfTxFigcj9KUJssf2puOUWJ0JsxP4ULjbK/hQqLmJfKia2\n2P4ULpYo+1MhrYB3nHNf+zm/BgyIIg68gvoc835h/RZh+lIxegD/dM59Fs3MZlYPr/CbAeCcO+Gc\nO1CC5cascJ0RrTj3u/F8xpb0/1JHYLtz7hPn3AlgPl7fSjoVyCXzGN4H4Gn/+QXAgcCHVC5hOpX/\nk0k28CXejmYdUC3w89YgzrxhSthY59w7wM3Ai2aWi3f3wUkxxOa7AVhZ6AO6uNhLgSHm/Wz6kpm1\njHG5E/yfBP9kZjVjiL0deME5tytUTHHLNbOn8I4gpAF/jmU7mTe04qfA8hjXtVgh+sU/ibJPBV6j\nGd4R63eAD/n3h8yPCNOnwsTeBUwxsx3AI8B/RRmX72pgt3NuWwzLvAy42rwhJa+Z2XdjiAW43e9P\nM4v7qTcYa2b9gJ3OuQ2RYsIt17xhRjuATOAPMeac7ybgpRLExZpvw8D/my+AhjG8VEz9KSDqvhRB\nVP2pkJj6UwhR96egWPtTiPio+1MYEftSKStJf9qE975cYGbfwvtVodi+5Jzbidd//gXsAg4651aU\nIOehhPmiFUZzYA/wlD/U4UkzOzeGeAesMLN3zeyWWBKlaJ0RtXj2RYHXaEYJP3di1ATvV/R8xe7r\nyooK5BiZWR/gS+fcuyWJd86dcs5l4H3L7wik4/2n/ZOZrQMO4x1VLjbWzNoAdwO9nHOpwFN4Qwei\njc13IxE+NMLE1gSOOec6AP8LzIwh9r/wCtTvAucDv40ytivezrlIYRvt+jrnRgCN8b4ZD4k2zvcX\n4HXn3NpYlhmNEP0iLdpYAPPGUy8C7vK/6NwE/MLM3sX7qexEDLG3AXc75y7G618zoozLF7E/hYmt\nhtcXrgJGAc/5Rw6jif0r3he2DLyd56PRLBc4CfyOKAuRUOvrnBvjb6dn8L68RR3rt4/x83gmlriS\n5pvPOZf/61O0ou5PhUTVl4pRbH8KIer+FELU/SnIL/ii7k+hRNufwiw/Yl9KpGj7k3PuI7xhICvw\nDjRkE2Z/F+R/SemHV7A2Bs41s5/EkqN553D0BRbEEFYNb9jAX51zVwBH8YaTRKuLc+5K4Drgl/4+\nLJpcS7XOiGVf5C+/xJ87ZxMVyLH7HtDXzHLwfgroDkwF6tu/T64r9jbZ/s80q4FrnXNvOeeuds51\nxPsZ8+MoY68D2gW+HT6LN6Y5quUCmHcSSUdgWaS4ELG5wGJ/0vNA22hj/Z9xnP/z51P+8qOJ7Qa0\nALb72/9bZrY9hpzz207hvXdFfiYOF2dm9wEN8MaERRRqmdEKxHYmyj7lH9leBDzjnFvsv84W59w1\nzrn2eMXFP6ONBYbx7/d2ASHenzBx+PkOwOuLIYWJzQUW+/1iHd5Rk1AnzIVa193+DuE03pe1cCeC\nFY69FG+Hu8HvT6nAe2Z2UbTrG/AMYfpThG01HOjz/9u7/1iv6jqO48+XEJDUMEryRtkdU7MSZNJa\na4jIDzNzNjbWL4q0mtnsl3+45nIsrdYarbXlzAKFhmZ/wFRaLbEYgzXnSPD+MMG0rLC8ZWplDAfz\n3R+fzxdOcL7nfL5wL+D2emzf7d7zPe/zOef7fd/ved/P93POB1iWC4xe2+yqS+yIpL78fB+pd6lI\naT7VaM2lJiX51EVRPtUpzacaxflUoGs+1WnLpTFyVPkUEbdHxJyImAc8T8v5LlsE/DEi/hER+0k5\n1Xiuq/F+YEdEjPQQswfYUznHricVzEVyz3dniM89lOfSEXWGpDtL26203/O56Fg+d47S0/z/twit\n9dPx4gK5RxFxQ0S8OSL6ST2/myNiGSkJl+bVPgncd3ispNOVrzCW9GpgMbBL0rS8bCKpN/W2wtjH\ngCmSzsmrdZYVtZuf0HJN9gAABodJREFUXkq6AGBf3fE2xN5LKlgBLqLmQ67heDsfqiIN76i7Crwu\n9uGIOCMi+vPrvzci6u7aURe7W/kOH7ndKyqvQdv+fgZ4H/DRfNLs5XVq1fDeluSUSL1yj0XEdyvL\nOzl1CnAj9TlVG0sa23dR/nkB8PvCOEgnsl0RsafLsXaLPZhPOZ8nAM8WHmtfZbUl1OfTEbERMRQR\n0yr5tId0gcozhe1WhxV9kJr3uyH2UtLXp1dExN7SuBINsRtJeQRd8qlhm6351EVjLhVozKcGrfnU\nTUk+1SnNp4Z2W/OpS1xjLo2ho8qnSi6dSfrn5ycFYX8G3iPp1JzfC6k517Xo+ZuI/N79RdLb8qKF\nwO9KYiVNlvTazs+kiylLc6muzijqMT/Gc9FRf+4cg+3A2Up3bZpAOt6Nx6ntZnESXCn4Sn0A8zl0\nF4sZpPHET5B6SibWrD8L2AkMkv5QVuTlK0l/7LtJX2nUtdUtdgkwRLrycwswozQ2P7eF1Kvb7Ri7\ntXsaqdd5CHiQ1JNdGrs5xw0Dd5KvuC3d58o63e5icUQs6Z/B31TavYvKVeMt+3uA1GP2SH7U7Uu3\n2C+STpQHSMXC6h5iS3JqLumrzcHK/l0GfIn0T8vjpHHp6iF2LvBwzqmHgDklcfm5tcA1DfnUrc0J\nOReGgR3Agh5i1+X3dZD0wdpXGnvYOk9RfxeLbu1uyPs7CPyMdBFdaewTpHF3nWW3FcYtyfn0EjAC\n3N9Dm68Hfk0qUn8FTO3yHt1NGlqwP7f16cJ8qotrzKWm2JJ8ami3NZ8aYlvzqWmf2/Kpod2SfKqL\na8ylltjWfGqILcqnmm1tIxWZA8DCkpgcdxOp0BvO79ERn4cNsZOBfwJTSmMqsbOB3+b35V7gdYVx\nM/IxDpDG8H+117bzdubT210sWs+dDbGtn5O9/B320O5lpM+WJ4/2dRqLh6eaNjMzMzOr8BALMzMz\nM7MKF8hmZmZmZhUukM3MzMzMKlwgm5mZmZlVuEA2MzMzM6twgWxmNkoknSHpp5KeVJpe9heSzpFU\ndP/TwjZulrQo/3yhpEclPSJpuqT1o7D9/tHcXzOzV6Lx7auYmVmbfJP9e4AfR8RH8rLzgTeOZjsR\nUZ3KeBnwrYjozLK1tCZkTEkaHxEHjne7ZmZjyT3IZmaj42Jgf0QcnGUuIgZIkzgAB3tnt0nakR/v\nzcv7JG3NPcHDuWd4nKS1+fchSdfldddKWppnePwQ8HVJd1V7fnPsd3LsoKQv5OUrJG3Py3+Ui3ok\nzZE0IGkAuLayv5Mkrcnt75TUmZ3uSkkbJW0mTRaBpOvztgcl3ZSXTZb087ztYUkfHruX38xs9LgH\n2cxsdJxHmjWuyd+BxRGxL08tfDfwLuBjpFnMvilpHHAqaQav6RFxHkBn+tiOiFgtaS5plq31kvor\nT18N9AOzI+KApKl5+S0RcXPe3jrgctKsbWuAz0fEVkkrK9u5NjUVMyWdC2zSoantLwBmRcRzki4B\nzgbeDQjYKGkecDrw14j4QG5zSsvrY2Z2UnAPspnZ8fMqYJWkIdL04e/Iy7cDV0n6GjAzIv4D/AGY\nIen7ki4F/t1DO4uAH3aGPkTEc3n5xZIeyu0vAN6ZC+/TImJrXmddZTtzSVM2ExG7gD8BnQL5gcp2\nL8mPnaSpnc8lFcxDwGJJ35Z0YUT8q4djMDM7YVwgm5mNjkeBOS3rXAeMAOeTeo4nAOTidB7wNLBW\n0vKIeD6vtwW4Blh9LDsnaRJwK7A0ImYCq4BJx7DJ/1Y3TxoLPTs/zoqI2yPicVJP8xDwDUkrardk\nZnaScYFsZjY6NgMTJV3dWSBpFvCWyjpTgL9FxMvAJ4Bxeb23AiMRsYpUCF8g6Q3AKRGxAbiRVGiW\negD4rKTxeftTOVQMPyvpNeQL+iLiBeCFPFwD0oV/Hds6v+ehFWcCu2vaux/4VN4u+Y4a0yS9Cdib\nLyJc2eMxmJmdMB6DbGY2CiIiJC0BvifpK8A+4Cngy5XVbgU2SFoO/JJDvbDzgesl7QdeBJYD04E1\nkjodGTf0sDurSUMhBvM2V0XELZJWAcPAM6RhHR1XAXdICmDTYfv7gzwk4wBwZUS8lK/tqx77Jklv\nBx7Mz70IfBw4C1gp6WVgP/C5Ho7BzOyEUUSc6H0wMzMzMztpeIiFmZmZmVmFC2QzMzMzswoXyGZm\nZmZmFS6QzczMzMwqXCCbmZmZmVW4QDYzMzMzq3CBbGZmZmZW8T9P9mRvX0mY7AAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     }
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "k3389HWmBWHS",
    "colab_type": "text"
   },
   "source": [
    "# COMBINE"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "cZLAHI8GTyYS",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine(test, svm_clfs)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "1vsQqPpwhPFw",
    "colab_type": "code",
    "outputId": "bfb8bda0-21dc-4ea9-a5cb-b75a28d4c817",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"SVM\")\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "SVM\n",
      "Voting\n",
      "0.6844190173954742\n",
      "0.7960573476702509\n",
      "Average\n",
      "0.7157183506823902\n",
      "0.8093189964157707\n",
      "Maximun\n",
      "0.7246264897256722\n",
      "0.8100358422939068\n",
      "Minimun\n",
      "0.6913066132481633\n",
      "0.7992831541218638\n",
      "Median\n",
      "0.7041830560278793\n",
      "0.8043010752688172\n",
      "Product\n",
      "0.7099291334989276\n",
      "0.8064516129032258\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "PAbKv8WyCVHi",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine(test, lr_clfs)\n"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "IrRxaur_hT6n",
    "colab_type": "code",
    "outputId": "e215eab5-209b-4a94-9d59-77add9f38cdd",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"Logistic Regression\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Logistic Regression\n",
      "Voting\n",
      "0.6634891322392161\n",
      "0.785663082437276\n",
      "Average\n",
      "0.7007152448729785\n",
      "0.8046594982078853\n",
      "Maximun\n",
      "0.7257250611783137\n",
      "0.8129032258064516\n",
      "Minimun\n",
      "0.6982068572166055\n",
      "0.8050179211469534\n",
      "Median\n",
      "0.6661083384689553\n",
      "0.7867383512544803\n",
      "Product\n",
      "0.7025877364569507\n",
      "0.8057347670250896\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "p52CSCD5UEo4",
    "colab_type": "code",
    "outputId": "d65fe1df-7785-4711-8b5d-f5c366649bf7",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    }
   },
   "source": [
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine(test, rf_clfs)"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "yDk5jCZyhZoa",
    "colab_type": "code",
    "outputId": "e42e7ab7-d0da-4f89-bf46-8d59e5b35aa7",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"RandomForest\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "RandomForest\n",
      "Voting\n",
      "0.6386654602575691\n",
      "0.7767025089605735\n",
      "Average\n",
      "0.6735967455814172\n",
      "0.7939068100358423\n",
      "Maximun\n",
      "0.6957896487123648\n",
      "0.8017921146953405\n",
      "Minimun\n",
      "0.6585667362849739\n",
      "0.7885304659498208\n",
      "Median\n",
      "0.6325298150126409\n",
      "0.7745519713261648\n",
      "Product\n",
      "0.6741421701785196\n",
      "0.7949820788530466\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "MGu1XrzQUGvV",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine(test, nb_clfs)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "PusC5H3Bhe1k",
    "colab_type": "code",
    "outputId": "0997acc6-9fcd-4a6f-e8c1-cafe7ab3c084",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"Naive Bayes\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Naive Bayes\n",
      "Voting\n",
      "0.7095484057209919\n",
      "0.7709677419354839\n",
      "Average\n",
      "0.7146150077283755\n",
      "0.771326164874552\n",
      "Maximun\n",
      "0.6684489240413125\n",
      "0.7086021505376344\n",
      "Minimun\n",
      "0.6713225177500645\n",
      "0.7132616487455197\n",
      "Median\n",
      "0.7070147481564805\n",
      "0.7738351254480287\n",
      "Product\n",
      "0.6995432175360584\n",
      "0.7480286738351255\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "s6W5gvofUIPz",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine(test, mlp_clfs)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "J_fDKfW4hifb",
    "colab_type": "code",
    "outputId": "3e3e3f87-5dfc-443d-d5fa-22e3b581dfbb",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"MLP\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "MLP\n",
      "Voting\n",
      "0.7376713858829925\n",
      "0.8146953405017922\n",
      "Average\n",
      "0.7421823341260287\n",
      "0.8168458781362007\n",
      "Maximun\n",
      "0.7558185584946902\n",
      "0.8218637992831541\n",
      "Minimun\n",
      "0.734807415905882\n",
      "0.8150537634408602\n",
      "Median\n",
      "0.7382668178904442\n",
      "0.8157706093189964\n",
      "Product\n",
      "0.7445729230450793\n",
      "0.8186379928315413\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "vVye-vYNUKXp",
    "colab_type": "code",
    "outputId": "a70fbace-fc6d-4d01-b43d-d11af819e346",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    }
   },
   "source": [
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine(test, extra_clfs)"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "08NKMhxthlZn",
    "colab_type": "code",
    "outputId": "219a3649-949b-4078-8720-2e994cb2851d",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"ExtraTrees\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "ExtraTrees\n",
      "Voting\n",
      "0.6041728541728542\n",
      "0.7645161290322581\n",
      "Average\n",
      "0.6864255055869992\n",
      "0.8003584229390681\n",
      "Maximun\n",
      "0.7180508730155312\n",
      "0.8078853046594983\n",
      "Minimun\n",
      "0.6678697228577085\n",
      "0.7899641577060932\n",
      "Median\n",
      "0.6043816554772555\n",
      "0.7659498207885305\n",
      "Product\n",
      "0.6683551166247766\n",
      "0.7917562724014336\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "dKCfnBDPDRLL",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine(test, knn_clfs)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "KyWbyl7Ehotz",
    "colab_type": "code",
    "outputId": "41d3eb65-9b4c-4cb2-b45d-ad10d98429a1",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"KNN\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "KNN\n",
      "Voting\n",
      "0.5841563513230934\n",
      "0.7508960573476703\n",
      "Average\n",
      "0.5180769722103975\n",
      "0.7422939068100358\n",
      "Maximun\n",
      "0.6006645951239001\n",
      "0.760573476702509\n",
      "Minimun\n",
      "0.4931792169162635\n",
      "0.7218637992831541\n",
      "Median\n",
      "0.5651980739173554\n",
      "0.7455197132616488\n",
      "Product\n",
      "0.4844043114772519\n",
      "0.7200716845878136\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "11MWVssOVMGg",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "cnn_clfs = {\n",
    "    'CV':cnn_cv, 'TFIDF':cnn_tfidf, 'W2V': cnn_w2v, 'GLOVE': cnn_glove, 'FAST': cnn_fast\n",
    "}\n",
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine(data_test, cnn_clfs, cnn=True)"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "SG8JyrsfhrSr",
    "colab_type": "code",
    "outputId": "e891690b-cf00-469a-aacf-18a0c88f476b",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"CNN\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "CNN\n",
      "Voting\n",
      "0.7412237801834992\n",
      "0.8146953405017922\n",
      "Average\n",
      "0.7471398331573856\n",
      "0.8186379928315413\n",
      "Maximun\n",
      "0.732525410124536\n",
      "0.8103942652329749\n",
      "Minimun\n",
      "0.690820749432793\n",
      "0.8003584229390681\n",
      "Median\n",
      "0.7492539945295474\n",
      "0.8175627240143369\n",
      "Product\n",
      "0.7473348328710535\n",
      "0.8218637992831541\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "NVkL7SOtVOwO",
    "colab_type": "code",
    "colab": {}
   },
   "source": [
    "full_clfs = {}\n",
    "for ext, clf in svm_clfs.items():\n",
    "    full_clfs['SVM-'+ext] = clf\n",
    "\n",
    "for ext, clf in lr_clfs.items():\n",
    "    full_clfs['LR-'+ext] = clf\n",
    "\n",
    "for ext, clf in rf_clfs.items():\n",
    "    full_clfs['RF-'+ext] = clf\n",
    "\n",
    "for ext, clf in nb_clfs.items():\n",
    "    full_clfs['NB-'+ext] = clf\n",
    "\n",
    "for ext, clf in mlp_clfs.items():\n",
    "    full_clfs['MLP-'+ext] = clf\n",
    "\n",
    "for ext, clf in extra_clfs.items():\n",
    "    full_clfs['EXTRA-'+ext] = clf\n",
    "\n",
    "for ext, clf in knn_clfs.items():\n",
    "    full_clfs['KNN-'+ext] = clf\n",
    "\n",
    "for ext, clf in cnn_clfs.items():\n",
    "    full_clfs['CNN-'+ext] = clf"
   ],
   "execution_count": 0,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "4C_-hLsfWSD0",
    "colab_type": "code",
    "outputId": "510febe8-e1c0-4e0f-d29f-ad4132494aa1",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    }
   },
   "source": [
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine_all(test, data_test, clfs=full_clfs)"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "G7FFXNDchuq1",
    "colab_type": "code",
    "outputId": "3f4c017e-c9ef-46aa-b63b-d8c255ff8e1a",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"Completo\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Completo\n",
      "Voting\n",
      "0.7049554323294327\n",
      "0.8043010752688172\n",
      "Average\n",
      "0.7210728056535317\n",
      "0.8118279569892473\n",
      "Maximun\n",
      "0.6267720389274155\n",
      "0.7648745519713261\n",
      "Minimun\n",
      "0.49674366182767044\n",
      "0.7168458781362007\n",
      "Median\n",
      "0.7046779873278032\n",
      "0.8053763440860215\n",
      "Product\n",
      "0.4941984593271793\n",
      "0.7172043010752688\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "_vfkrbEYb9_W",
    "colab_type": "code",
    "outputId": "2e6a040a-8f29-499f-fd9a-8053cfe98b75",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    }
   },
   "source": [
    "clfs_combine = {\n",
    "    'NB-W2V': full_clfs['NB-W2V'], 'LR-FAST': full_clfs['LR-FAST'],  'LR-CV': full_clfs['LR-CV'], \n",
    "    'RF-TFIDF': full_clfs['RF-TFIDF'], 'EXTRA-GLOVE': full_clfs['EXTRA-GLOVE']\n",
    "}\n",
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine_all(test, data_test, clfs=clfs_combine)"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "RTa28hC7hyCB",
    "colab_type": "code",
    "outputId": "22a01009-f3cb-471f-b6fb-f223ea324aee",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"Combinado dissimilar\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Combinado dissimilar\n",
      "Voting\n",
      "0.7240303033591081\n",
      "0.8100358422939068\n",
      "Average\n",
      "0.7589260616609779\n",
      "0.825089605734767\n",
      "Maximun\n",
      "0.6612900604263422\n",
      "0.6996415770609319\n",
      "Minimun\n",
      "0.6708696289442577\n",
      "0.7121863799283155\n",
      "Median\n",
      "0.7239495380852379\n",
      "0.8111111111111111\n",
      "Product\n",
      "0.7368029158924664\n",
      "0.7935483870967742\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "swDxkMfYgYJP",
    "colab_type": "code",
    "outputId": "829f7e01-63b0-4b6d-cfcb-68691cb2e70e",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    }
   },
   "source": [
    "clfs_combine = {\n",
    "    'MLP-CV': full_clfs['MLP-CV'], 'LR-FAST': full_clfs['LR-FAST'], 'SVM-W2V': full_clfs['SVM-W2V'], \n",
    "    'CNN-CV': full_clfs['CNN-CV'], 'EXTRA-W2V': full_clfs['EXTRA-W2V']\n",
    "}\n",
    "y_pred_voting, y_pred_average, y_pred_maximun, y_pred_minimun, y_pred_median, y_pred_product = get_combine_all(test, data_test, clfs=clfs_combine)"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  11 out of  11 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  13 out of  13 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  14 out of  14 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  16 out of  16 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  17 out of  17 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  19 out of  19 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  22 out of  22 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  23 out of  23 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  25 out of  25 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  26 out of  26 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  28 out of  28 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  29 out of  29 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  31 out of  31 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  32 out of  32 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  33 out of  33 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  34 out of  34 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  35 out of  35 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  37 out of  37 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  38 out of  38 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  39 out of  39 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  41 out of  41 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  42 out of  42 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  43 out of  43 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  44 out of  44 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  45 out of  45 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  46 out of  46 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  47 out of  47 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  48 out of  48 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  49 out of  49 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:    0.1s finished\n"
     ],
     "name": "stdout"
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "TKUEOLSjhJOx",
    "colab_type": "code",
    "outputId": "44094e3b-66c6-41cf-f8e3-8b1c1a77e34b",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    }
   },
   "source": [
    "print(\"Combinado Similar\")\n",
    "\n",
    "print(\"Voting\")\n",
    "print(f1_score(class_test, y_pred_voting, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_voting))\n",
    "\n",
    "print(\"Average\")\n",
    "print(f1_score(class_test, y_pred_average, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_average))\n",
    "\n",
    "print(\"Maximun\")\n",
    "print(f1_score(class_test, y_pred_maximun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_maximun))\n",
    "\n",
    "print(\"Minimun\")\n",
    "print(f1_score(class_test, y_pred_minimun, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_minimun))\n",
    "\n",
    "print(\"Median\")\n",
    "print(f1_score(class_test, y_pred_median, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_median))\n",
    "\n",
    "print(\"Product\")\n",
    "print(f1_score(class_test, y_pred_product, average='macro'))\n",
    "print(accuracy_score(class_test, y_pred_product))"
   ],
   "execution_count": 0,
   "outputs": [
    {
     "output_type": "stream",
     "text": [
      "Combinado Similar\n",
      "Voting\n",
      "0.7005746957348946\n",
      "0.7985663082437277\n",
      "Average\n",
      "0.7338053390535783\n",
      "0.8157706093189964\n",
      "Maximun\n",
      "0.76028812757396\n",
      "0.8254480286738352\n",
      "Minimun\n",
      "0.7246259399113675\n",
      "0.803584229390681\n",
      "Median\n",
      "0.7113045213676807\n",
      "0.8043010752688172\n",
      "Product\n",
      "0.7341433952895082\n",
      "0.8150537634408602\n"
     ],
     "name": "stdout"
    }
   ]
  }
 ]
}