release.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import dlib\n",
    "import cv2\n",
    "from scipy.misc import *\n",
    "import matplotlib.pyplot as plt\n",
    "from glob import glob\n",
    "from keras.engine import  Model\n",
    "from keras.layers import Input\n",
    "from keras_vggface.vggface import VGGFace\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.pipeline import Pipeline\n",
    "import itertools\n",
    "from sklearn.cluster import KMeans, SpectralClustering\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from keras.models import load_model\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def log_progress(sequence, every=None, size=None, name='Items'):\n",
    "    from ipywidgets import IntProgress, HTML, VBox\n",
    "    from IPython.display import display\n",
    "\n",
    "    is_iterator = False\n",
    "    if size is None:\n",
    "        try:\n",
    "            size = len(sequence)\n",
    "        except TypeError:\n",
    "            is_iterator = True\n",
    "    if size is not None:\n",
    "        if every is None:\n",
    "            if size <= 200:\n",
    "                every = 1\n",
    "            else:\n",
    "                every = int(size / 200)     # every 0.5%\n",
    "    else:\n",
    "        assert every is not None, 'sequence is iterator, set every'\n",
    "\n",
    "    if is_iterator:\n",
    "        progress = IntProgress(min=0, max=1, value=1)\n",
    "        progress.bar_style = 'info'\n",
    "    else:\n",
    "        progress = IntProgress(min=0, max=size, value=0)\n",
    "    label = HTML()\n",
    "    box = VBox(children=[label, progress])\n",
    "    display(box)\n",
    "\n",
    "    index = 0\n",
    "    try:\n",
    "        for index, record in enumerate(sequence, 1):\n",
    "            if index == 1 or index % every == 0:\n",
    "                if is_iterator:\n",
    "                    label.value = '{name}: {index} / ?'.format(\n",
    "                        name=name,\n",
    "                        index=index\n",
    "                    )\n",
    "                else:\n",
    "                    progress.value = index\n",
    "                    label.value = u'{name}: {index} / {size}'.format(\n",
    "                        name=name,\n",
    "                        index=index,\n",
    "                        size=size\n",
    "                    )\n",
    "            yield record\n",
    "    except:\n",
    "        progress.bar_style = 'danger'\n",
    "        raise\n",
    "    else:\n",
    "        progress.bar_style = 'success'\n",
    "        progress.value = index\n",
    "        label.value = \"{name}: {index}\".format(\n",
    "            name=name,\n",
    "            index=str(index or '?')\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "CLASS2LBL = {'Negative': 0, 'Neutral': 1, 'Positive': 2}\n",
    "LBL2CLASS = {0: 'Negative', 1: 'Neutral', 2: 'Positive'}\n",
    "\n",
    "def parse_lbl(fname):\n",
    "    _class = fname.split('/')[-2]\n",
    "    return CLASS2LBL[_class]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          cmap=plt.cm.Blues):\n",
    "    plt.figure(figsize=(15, 10))\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        cm = np.round(cm, 2)\n",
    "        print(cm)\n",
    "    \n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title('Confusion matrix')\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True labels')\n",
    "    plt.xlabel('Predicted labels')\n",
    "    plt.savefig('confusion_release.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Соберем лица и лендмарки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_PATH = '/home/_datasets_/emotiw2017/'\n",
    "TRAIN_DIR = 'train'\n",
    "VAL_DIR = 'val'\n",
    "TEST_DIR = 'test'\n",
    "IMG_TMPL = '*.jpg'\n",
    "PREDICTOR_PATH = '/home/arassadin/distr/shape_predictor_68_face_landmarks.dat'\n",
    "OUT_SIZE = (224, 224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "detector = dlib.get_frontal_face_detector()\n",
    "predictor = dlib.shape_predictor(PREDICTOR_PATH)\n",
    "face_cascade = cv2.CascadeClassifier('/home/arassadin/distr/opencv/data/haarcascades_GPU/haarcascade_frontalface_alt2.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3630 2065 772\n"
     ]
    }
   ],
   "source": [
    "files_train = glob(os.path.join(DATA_PATH, TRAIN_DIR, '*', '*'))\n",
    "files_val = glob(os.path.join(DATA_PATH, VAL_DIR, '*', '*'))\n",
    "files_test = glob(os.path.join(DATA_PATH, TEST_DIR, '*'))\n",
    "\n",
    "print len(files_train), len(files_val), len(files_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_faces_dlib(img):\n",
    "    try:\n",
    "        rects = detector(img, 1)\n",
    "    except:\n",
    "        return None\n",
    "    faces = []\n",
    "\n",
    "    for rect in rects:\n",
    "        y1 = rect.top()\n",
    "        if y1 < 0:\n",
    "            y1 = 0\n",
    "        y2 = rect.bottom()\n",
    "        if y2 > img.shape[0]:\n",
    "            y2 = img.shape[0]\n",
    "        x1 = rect.left()\n",
    "        if x1 < 0:\n",
    "            x1 = 0\n",
    "        x2 = rect.right()\n",
    "        if x2 > img.shape[1]:\n",
    "            x2 = img.shape[1]\n",
    "        faces.append(img[y1:y2, x1:x2])\n",
    "    \n",
    "    return faces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_faces_ocv(img):\n",
    "    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n",
    "    dets = face_cascade.detectMultiScale(gray, 1.3, 4)\n",
    "    faces = [img[y : y + h, x : x + w] for x, y, w, h in dets]        \n",
    "    return faces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_dst(p1, p2):\n",
    "    x1, y1 = p1\n",
    "    x2, y2 = p2\n",
    "    return np.sqrt(np.square(y2 - y1) + np.square(x2 - x1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_landmarks(face):\n",
    "    bbox = dlib.rectangle(0, 0, face.shape[1] - 1, face.shape[0] - 1)\n",
    "    shape = predictor(face, bbox)\n",
    "    landmarks = [(shape.part(i).x, shape.part(i).y) for i in range(68)]\n",
    "    leftmost = min([l[0] for l in landmarks])\n",
    "    topmost = min([l[1] for l in landmarks])\n",
    "    landmarks = [(l[0] - leftmost, l[1] - topmost) for l in landmarks]\n",
    "    dsts = []\n",
    "    for i in range(68):\n",
    "        for j in range(i + 1, 68):\n",
    "            dsts.append(get_dst(landmarks[i], landmarks[j]))\n",
    "    dsts = np.asarray(dsts)\n",
    "    dsts /= dsts.max()\n",
    "    return dsts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "905538d095db450a8c357eed410dde71"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3630 3553 3553\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b805a5af8919451f85fbf410afa0d23e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2065 2020 2020\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10eb3eaa6c3445e49ce54e2932f3b770"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Something wrong with image /home/_datasets_/emotiw2017/test/1997_Convocation.jpg!\n",
      "Something wrong with image /home/_datasets_/emotiw2017/test/2011Fall_Convocation.jpg!\n",
      "772 715 715\n",
      "No faces found on 163 photos\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists('./data'):\n",
    "    os.makedirs('./data')\n",
    "\n",
    "fault_counter = 0\n",
    "def routine(files):\n",
    "    global fault_counter\n",
    "    data_faces, data_dsts = {}, {}\n",
    "    for fname in log_progress(files):\n",
    "        img = imread(fname, mode='RGB')\n",
    "        faces = find_faces_dlib(img)\n",
    "        if faces is None:\n",
    "            print 'Something wrong with image {}!'.format(fname)\n",
    "            continue\n",
    "        if len(faces) == 0:\n",
    "            faces = find_faces_ocv(img)\n",
    "            if len(faces) == 0:\n",
    "                fault_counter += 1\n",
    "                continue\n",
    "\n",
    "        faces = [imresize(f, OUT_SIZE) for f in faces]\n",
    "        dsts = [find_landmarks(f) for f in faces]\n",
    "        \n",
    "        data_faces[fname] = faces\n",
    "        data_dsts[fname] = dsts\n",
    "    return data_faces, data_dsts\n",
    "\n",
    "faces_train, dsts_train = routine(files_train)\n",
    "# np.save('data/faces_train', faces_train); \n",
    "np.save('data/dsts_train_sized', dsts_train)\n",
    "print len(files_train), len(faces_train), len(dsts_train)\n",
    "\n",
    "faces_val, dsts_val = routine(files_val)\n",
    "# np.save('data/faces_val', faces_val); \n",
    "np.save('data/dsts_val_sized', dsts_val)\n",
    "print len(files_val), len(faces_val), len(dsts_val)\n",
    "\n",
    "faces_test, dsts_test = routine(files_test[14:])\n",
    "# np.save('data/faces_test', faces_test); \n",
    "np.save('data/dsts_test_sized', dsts_test)\n",
    "print len(files_test), len(faces_test), len(dsts_test)\n",
    "\n",
    "print 'No faces found on {} photos'.format(fault_counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 2]\n",
      "[0 1 2]\n"
     ]
    }
   ],
   "source": [
    "y_train = {}\n",
    "for fname in files_train:\n",
    "    y_train[fname] = parse_lbl(fname)\n",
    "    \n",
    "y_val = {}\n",
    "for fname in files_val:\n",
    "    y_val[fname] = parse_lbl(fname)\n",
    "    \n",
    "print np.unique(y_train.values())\n",
    "print np.unique(y_val.values())\n",
    "\n",
    "np.save('data/y_train', y_train)\n",
    "np.save('data/y_val', y_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Сбор нейросетевых скорингов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalize_vggface_1(arr):\n",
    "    arr = arr[::-1, :, :].astype(np.float32)\n",
    "\n",
    "    arr[0, :, :] -= 93.5940\n",
    "    arr[1, :, :] -= 104.7624\n",
    "    arr[2, :, :] -= 129.1863\n",
    "\n",
    "    return arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalize_vggface_2(arr):\n",
    "    arr = arr[:, :, :].astype(np.float32)\n",
    "\n",
    "    arr[0, :, :] -= 93.5940\n",
    "    arr[1, :, :] -= 104.7624\n",
    "    arr[2, :, :] -= 129.1863\n",
    "\n",
    "    return arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalize_vggface_3(arr):\n",
    "    arr = arr[::-1, :, :].astype(np.float32)\n",
    "\n",
    "    arr[0, :, :] -= 93.5940\n",
    "    arr[1, :, :] -= 104.7624\n",
    "    arr[2, :, :] -= 129.1863\n",
    "    \n",
    "    arr = arr[::-1, :, :]\n",
    "\n",
    "    return arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/keras_vggface/vggface.py:166: UserWarning: You are using the TensorFlow backend, yet you are using the Theano image data format convention (`image_data_format=\"channels_first\"`). For best performance, set `image_data_format=\"channels_last\"` in your Keras config at ~/.keras/keras.json.\n",
      "  warnings.warn('You are using the TensorFlow backend, yet you '\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "add3167719804181981ebd0f64bb3425"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2548fcfc37444660ad5e4bf2b0cd638b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b02ea3e144764f6598648d4728e74fe0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b6e10fc3f7a40b5a31c7d44002faa51"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8dfc671e1ef4050ac0a968213f24ee5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7197389117b146ad88d45f1bd0fb46c7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e31fb951e1b14805aac3f13f03f561ec"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce0a1def255340fc9800298ae3520c49"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10bb72328c5e4bdaa6d19707a974f03e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cebdfd4610147bf8683c20d4198808e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b51057a2219d40f3878c9d15a6c925cc"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa68ec01d395457296ca6882ba296c48"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def routine1(data, model, norm_f):\n",
    "    scores = {}\n",
    "    for fname in log_progress(data.keys()):\n",
    "#         _class = fname.split('/')[-2]\n",
    "#         lbl = CLASS2LBL[_class]\n",
    "#         scores_y.append(lbl)\n",
    "        \n",
    "        if len(data[fname]) < 1:\n",
    "            continue\n",
    "        faces = np.asarray([norm_f(np.moveaxis(face, 2, 0)) for face in data[fname]])\n",
    "        preds = model.predict(faces)\n",
    "        scores[fname] = np.median(preds, axis=0)\n",
    "        # break\n",
    "    return scores\n",
    "\n",
    "def routine2(fname, model, m='default'):\n",
    "    data = np.load(fname).item()\n",
    "    for i, norm_f in zip([1, 3], [normalize_vggface_1, normalize_vggface_3]):\n",
    "        scores = routine1(data, model, norm_f)\n",
    "        postfix = os.path.basename(fname).split('_')[-1]\n",
    "        np.save('data/scores_{}_norm={}_model={}'.format(postfix, i, m), scores)\n",
    "\n",
    "model = VGGFace(include_top=False, input_shape=(3, 224, 224), pooling='avg')\n",
    "routine2('data/faces_train.npy', model, m='vgg-face-avgpool')\n",
    "routine2('data/faces_val.npy', model, m='vgg-face-avgpool')\n",
    "routine2('data/faces_test.npy', model, m='vgg-face-avgpool')\n",
    "\n",
    "vgg_faces = VGGFace()\n",
    "out = vgg_faces.get_layer('fc7').output\n",
    "model = Model(vgg_faces.input, out)\n",
    "routine2('data/faces_train.npy', model, m='vgg-face-fc7')\n",
    "routine2('data/faces_val.npy', model, m='vgg-face-fc7')\n",
    "routine2('data/faces_test.npy', model, m='vgg-face-fc7')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2065\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c8349b97a1f4303be5aba96f9a2f5e0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "frames_val = np.load('data_phase3/frames_val.npy').item()\n",
    "print len(frames_val)\n",
    "\n",
    "model = load_model('nn_raw/model.h5')\n",
    "\n",
    "fscores_val = {}\n",
    "for fname in log_progress(frames_val.keys()):\n",
    "    fscores_val[fname] = model.predict(np.expand_dims(frames_val[fname], axis=0))[0]\n",
    "#     break\n",
    "np.save('data/probs_frames_val', fscores_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "770\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6bb1775adcf49ec8c4d910359d12760"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "frames_test = np.load('data_phase3/frames_test.npy').item()\n",
    "print len(frames_test)\n",
    "\n",
    "model = load_model('nn_raw/model_release.h5')\n",
    "\n",
    "fscores_test = {}\n",
    "for fname in log_progress(frames_test.keys()):\n",
    "    fscores_test[fname] = model.predict(np.expand_dims(frames_test[fname], axis=0))[0]\n",
    "#     break\n",
    "np.save('data/probs_frames_test', fscores_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Обучение слабых классификаторов и сбор их скорингов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_X_y_flat(d):\n",
    "    X, y = [], []\n",
    "    for fname in log_progress(d.keys()):\n",
    "        for el in d[fname]:\n",
    "            y.append(parse_lbl(fname))\n",
    "            X.append(el)\n",
    "    X = np.asarray(X)\n",
    "    y = np.asarray(y, np.uint8)\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_X_flat(d):\n",
    "    X = []\n",
    "    for fname in log_progress(d.keys()):\n",
    "        for el in d[fname]:\n",
    "            X.append(el)\n",
    "    X = np.asarray(X)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_X_y(d):\n",
    "    X, y = [], []\n",
    "    for fname in log_progress(d.keys()):\n",
    "        X.append(d[fname])\n",
    "        y.append(parse_lbl(fname))\n",
    "    X = np.asarray(X)\n",
    "    y = np.asarray(y, np.uint8)\n",
    "    return X, y, d.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_X(d):\n",
    "    X = []\n",
    "    for fname in log_progress(d.keys()):\n",
    "        X.append(d[fname])\n",
    "    X = np.asarray(X)\n",
    "    return X, d.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def probs_deflate(probs_pred, d):\n",
    "    probs_deflated = {}\n",
    "\n",
    "    cur_position = 0\n",
    "    for fname in log_progress(d.keys()):\n",
    "        q = len(d[fname])\n",
    "        score = np.sum(probs_pred[cur_position : cur_position + q], axis=0)\n",
    "        probs_deflated[fname] = score\n",
    "        cur_position += q\n",
    "    #     break\n",
    "\n",
    "    return probs_deflated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_scores(y_true, probs_pred):\n",
    "    accuracy = 0.0\n",
    "    for f in log_progress(probs_pred.keys()):\n",
    "        if y_true[f] == np.argmax(probs_pred[f]):\n",
    "            accuracy += 1\n",
    "    return accuracy / len(probs_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train / val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Лендмарки:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa9993cf15b64c61afe1c27243b527a4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a86da368b567475eafebd35276c0d027"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12826, 2278) (12826,)\n",
      "(7247, 2278) (7247,)\n"
     ]
    }
   ],
   "source": [
    "dsts_X_train, dsts_y_train = parse_X_y_flat(np.load('data/dsts_train_sized.npy').item())\n",
    "dsts_val = np.load('data/dsts_val_sized.npy').item()\n",
    "dsts_X_val, dsts_y_val = parse_X_y_flat(dsts_val)\n",
    "\n",
    "print dsts_X_train.shape, dsts_y_train.shape\n",
    "print dsts_X_val.shape, dsts_y_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictor_dsts = OneVsRestClassifier(RandomForestClassifier(n_estimators=5000, n_jobs=16)).fit(dsts_X_train, dsts_y_train)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "predictor_dsts = Pipeline([\n",
    "  ('feature_selection', SelectFromModel(RandomForestClassifier(n_estimators=100))),\n",
    "  ('classification', OneVsRestClassifier(GradientBoostingClassifier(n_estimators=1000)))\n",
    "])\n",
    "predictor_dsts.fit(dsts_X_train, dsts_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7247, 3)\n"
     ]
    }
   ],
   "source": [
    "tmp_probs = predictor_dsts.predict_proba(dsts_X_val)\n",
    "print tmp_probs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f8d9f4d7bb6d49feb1b2f31945f25e80"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020\n"
     ]
    }
   ],
   "source": [
    "dsts_probs_pred = probs_deflate(tmp_probs, dsts_val)\n",
    "print len(dsts_probs_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc8dfe7fe3094229bfffc1d004705a32"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.647524752475\n"
     ]
    }
   ],
   "source": [
    "print get_scores(np.load('data/y_val.npy').item(), dsts_probs_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('data/probs_dsts_val_sized', dsts_probs_pred)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "scorings_all = probs_deflate(predictor_dsts.predict_proba(dsts_X_train), np.load('data/dsts_train.npy').item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Нейронки:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nn_scores_train_fnames = ['data/scores_train.npy_norm=1_model=vgg-face-avgpool.npy',\n",
    "                          'data/scores_train.npy_norm=1_model=vgg-face-fc7.npy',\n",
    "                          'data/scores_train.npy_norm=3_model=vgg-face-avgpool.npy',\n",
    "                          'data/scores_train.npy_norm=3_model=vgg-face-fc7.npy']\n",
    "nn_scores_val_fnames = ['data/scores_val.npy_norm=1_model=vgg-face-avgpool.npy',\n",
    "                        'data/scores_val.npy_norm=1_model=vgg-face-fc7.npy',\n",
    "                        'data/scores_val.npy_norm=3_model=vgg-face-avgpool.npy',\n",
    "                        'data/scores_val.npy_norm=3_model=vgg-face-fc7.npy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 4\n"
     ]
    }
   ],
   "source": [
    "nn_scores_train = [np.load(x).item() for x in nn_scores_train_fnames]\n",
    "nn_scores_val = [np.load(x).item() for x in nn_scores_val_fnames]\n",
    "\n",
    "print len(nn_scores_train), len(nn_scores_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a620b2b2ec9475789f3c93618442026"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1d86e03bd8940a5a452004ba0093216"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e506b0ef2884481a32b428840444e7c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f96d7a65cf794be3bf9061129a9a7695"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a05c82c0bf3644f3881282671394b861"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "preds_decl = ['OneVsRestClassifier(RandomForestClassifier(n_estimators=5000, n_jobs=16))',\n",
    "              'OneVsRestClassifier(RandomForestClassifier(n_estimators=1000, n_jobs=16))',\n",
    "              'OneVsRestClassifier(RandomForestClassifier(n_estimators=1000, n_jobs=16))',\n",
    "              'OneVsRestClassifier(RandomForestClassifier(n_estimators=5000, n_jobs=16))']\n",
    "\n",
    "nn_predictors= []\n",
    "for scores, decl in log_progress(zip(nn_scores_train, preds_decl)):\n",
    "    tmp_X, tmp_y, _ = parse_X_y(scores)\n",
    "    nn_predictors.append(eval(decl).fit(tmp_X, tmp_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da184ff9a70c40869051d171ea2fc284"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dfa7a21ebd3439382299336df6b41eb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.69603960396\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e5935a312f945c3aa4ac4e6d1e9e304"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.683168316832\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "610009ec3e494247afbf0009a5c3ea14"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.698514851485\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f925d7b9e0704a5f92e1ebdc5ee0f281"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.676732673267\n"
     ]
    }
   ],
   "source": [
    "nn_probs_pred = []\n",
    "\n",
    "for scores, predictor in log_progress(zip(nn_scores_val, nn_predictors)):\n",
    "    tmp_X, tmp_y, fnames = parse_X_y(scores)\n",
    "    tmp_probs = predictor.predict_proba(tmp_X)\n",
    "    nn_probs_pred.append(dict(zip(fnames, tmp_probs)))\n",
    "    print np.mean(tmp_y == np.argmax(tmp_probs, axis=1))\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('data/probs_nn_val', nn_probs_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Стэкинг**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5bf18b8f5c5415cbbcb6a32360d9140"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "380709739122444a9f99b6d9a08603dd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3881a3079089456a8bdb6df4638c3cfd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13dac33d2f5d4892905aa7d957bf2da4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020\n"
     ]
    }
   ],
   "source": [
    "scorings_all_val = np.load('data/probs_dsts_val.npy').item()\n",
    "for d in np.load('data/probs_nn_val.npy'):\n",
    "    for fname in log_progress(d.keys()):\n",
    "        scorings_all_val[fname] = np.append(scorings_all_val[fname], d[fname])\n",
    "\n",
    "print len(scorings_all_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ceb5b24b20b4b19968c44b570963852"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2020, 15) (2020,)\n"
     ]
    }
   ],
   "source": [
    "X, y, _ = parse_X_y(scorings_all_val)\n",
    "print X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   2.4s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    2.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................................. , total=   2.2s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   2.3s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   2.1s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   2.0s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   2.1s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   2.3s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   2.0s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   2.3s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   1.7s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  10 out of  10 | elapsed:   21.5s finished\n"
     ]
    }
   ],
   "source": [
    "res = cross_val_score(OneVsRestClassifier(RandomForestClassifier(n_estimators=100, n_jobs=16)), X, y, cv=10, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.70935961,  0.70935961,  0.74384236,  0.68965517,  0.72772277,\n",
       "        0.72772277,  0.70149254,  0.69154229,  0.65671642,  0.76119403])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train + val / test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Лендмарки:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78dd0d8e6c53497e87772c66c7ad7a2a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17a6dcbb2c7740238a62d61e158fe12a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20073, 2278) (20073,)\n",
      "(4317, 2278)\n"
     ]
    }
   ],
   "source": [
    "dsts_X_train, dsts_y_train = parse_X_y_flat( dict( np.load('data/dsts_train_sized.npy').item(), \n",
    "                                                   **np.load('data/dsts_val_sized.npy').item() \n",
    "                                                 )\n",
    "                                           )\n",
    "dsts_test = np.load('data/dsts_test_sized.npy').item()\n",
    "dsts_X_test = parse_X_flat(dsts_test)\n",
    "\n",
    "print dsts_X_train.shape, dsts_y_train.shape\n",
    "print dsts_X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictor_dsts = OneVsRestClassifier(RandomForestClassifier(n_estimators=5000, n_jobs=16)).fit(dsts_X_train, dsts_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tmp_probs = predictor_dsts.predict_proba(dsts_X_test)\n",
    "print tmp_probs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dsts_probs_pred = probs_deflate(tmp_probs, dsts_test)\n",
    "print len(dsts_probs_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('data/probs_dsts_test_sized', dsts_probs_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Нейронки:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nn_scores_train_fnames = [\n",
    "    ('data/scores_train.npy_norm=4_model=vgg-face-avgpool.npy', 'data/scores_val.npy_norm=4_model=vgg-face-avgpool.npy'),\n",
    "    ('data/scores_train.npy_norm=4_model=vgg-face-fc7.npy', 'data/scores_val.npy_norm=4_model=vgg-face-fc7.npy'),\n",
    "    ('data/scores_train.npy_norm=3_model=vgg-face-avgpool.npy', 'data/scores_val.npy_norm=3_model=vgg-face-avgpool.npy'),\n",
    "    ('data/scores_train.npy_norm=3_model=vgg-face-fc7.npy', 'data/scores_val.npy_norm=3_model=vgg-face-fc7.npy')\n",
    "]\n",
    "nn_scores_test_fnames = ['data/scores_test.npy_norm=4_model=vgg-face-avgpool.npy',\n",
    "                         'data/scores_test.npy_norm=4_model=vgg-face-fc7.npy',\n",
    "                         'data/scores_test.npy_norm=3_model=vgg-face-avgpool.npy',\n",
    "                         'data/scores_test.npy_norm=3_model=vgg-face-fc7.npy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 4\n"
     ]
    }
   ],
   "source": [
    "nn_scores_train = [dict(np.load(x1).item(), **np.load(x2).item()) for x1, x2 in nn_scores_train_fnames]\n",
    "nn_scores_test = [np.load(x).item() for x in nn_scores_test_fnames]\n",
    "\n",
    "print len(nn_scores_train), len(nn_scores_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a75db237813a44f786c54929b5f74170"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99b228112fe34c4da8e9e4b8217f5921"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0dc7e0b3a194806bc988a2b1c908465"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2daafe0a592476b939074e04314808d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e528382065594040b94167a2235fe702"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "preds_decl = ['OneVsRestClassifier(RandomForestClassifier(n_estimators=7000, n_jobs=16))',\n",
    "              'OneVsRestClassifier(RandomForestClassifier(n_estimators=2000, n_jobs=16))',\n",
    "              'OneVsRestClassifier(RandomForestClassifier(n_estimators=2000, n_jobs=16))',\n",
    "              'OneVsRestClassifier(RandomForestClassifier(n_estimators=7000, n_jobs=16))']\n",
    "\n",
    "nn_predictors= []\n",
    "for scores, decl in log_progress(zip(nn_scores_train, preds_decl)):\n",
    "    tmp_X, tmp_y, _ = parse_X_y(scores)\n",
    "    nn_predictors.append(eval(decl).fit(tmp_X, tmp_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf1d672d9be74b2ab5d8cfc1c8ae8a43"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d34b313d7c224f6d9d9a0df118bb06a1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cf3a9224bcd485ea8aeca61aadc509a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "afe9ec521cee451b9094a71784820737"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2617b9cbeb30480fa94facf2b9bef158"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn_probs_pred = []\n",
    "\n",
    "for scores, predictor in log_progress(zip(nn_scores_test, nn_predictors)):\n",
    "    tmp_X, fnames = parse_X(scores)\n",
    "    tmp_probs = predictor.predict_proba(tmp_X)\n",
    "    nn_probs_pred.append(dict(zip(fnames, tmp_probs)))\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('data/probs_nn_test_x2', nn_probs_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Тест ансамбля"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "probs_dsts_fname = 'data/probs_dsts_val_sized.npy'\n",
    "probs_nn_fname = 'data/probs_nn_val.npy'\n",
    "probs_frames_fname = 'data/probs_frames_val.npy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "probs_dsts = np.load(probs_dsts_fname).item()\n",
    "probs_nn = np.load(probs_nn_fname)\n",
    "probs_frames = np.load(probs_frames_fname).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0141784866741 1.19587458746 0.863800733867 21.3868224982\n"
     ]
    }
   ],
   "source": [
    "print np.min(probs_dsts.values()), np.mean(probs_dsts.values()), np.median(probs_dsts.values()), np.max(probs_dsts.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0138257575758 0.333333333333 0.317539199658 0.944839071475\n"
     ]
    }
   ],
   "source": [
    "print np.min(probs_nn[0].values()), np.mean(probs_nn[0].values()), np.median(probs_nn[0].values()), np.max(probs_nn[0].values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2065\n"
     ]
    }
   ],
   "source": [
    "ys_true = np.load('data/y_val.npy').item()\n",
    "print len(ys_true)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "y_true = []\n",
    "y_pred = []\n",
    "\n",
    "weight_dsts = 0.5\n",
    "weights_nn = [0.7, 0.7, 0.7, 0.7]\n",
    "\n",
    "for fname in log_progress(probs_dsts.keys()):\n",
    "    preds = np.zeros((3,))\n",
    "    preds += probs_dsts[fname] * weight_dsts\n",
    "    for tmp_probs, w in zip(probs_nn, weights_nn):\n",
    "        preds += tmp_probs[fname] * w\n",
    "    y_pred.append(np.argmax(preds))\n",
    "    y_true.append(ys_true[fname])\n",
    "#     break\n",
    "    \n",
    "y_true = np.asarray(y_true)\n",
    "y_pred = np.asarray(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e8038d5b0b649fab14cb53d87475b48"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.75396039604\n"
     ]
    }
   ],
   "source": [
    "y_true = []\n",
    "y_pred = []\n",
    "\n",
    "weight_dsts = 0.6\n",
    "weigths_frames = 0.86\n",
    "weights_nn = [0.7, 0.7, 0.7, 0.7]\n",
    "\n",
    "for fname in log_progress(probs_dsts.keys()):\n",
    "    preds = np.zeros((3,))\n",
    "    if probs_dsts[fname].max() < 3.6:\n",
    "        preds += probs_dsts[fname] * weight_dsts\n",
    "    else:\n",
    "        preds += probs_dsts[fname] * weight_dsts / np.max(probs_dsts[fname])\n",
    "    preds += probs_frames[fname] * weigths_frames\n",
    "    for tmp_probs, w in zip(probs_nn, weights_nn):\n",
    "        preds += tmp_probs[fname] * w\n",
    "    y_pred.append(np.argmax(preds))\n",
    "    y_true.append(ys_true[fname])\n",
    "#     break\n",
    "    \n",
    "y_true = np.asarray(y_true)\n",
    "y_pred = np.asarray(y_pred)\n",
    "\n",
    "print np.mean(y_true == y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.8   0.13  0.07]\n",
      " [ 0.26  0.66  0.07]\n",
      " [ 0.06  0.14  0.8 ]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAALWCAYAAACKtTbEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmUXWWVN+DfroQwi0gQSAIyiAwiCgYUtBVt0YAITgyK\nY2vb2uKEfgraIk7tQOPQNg44t9JAABUQBJxQcQxCRAbFyCBJAJnEAQQS3u+PuomVQJICUqm6p57H\ndVfqnPPe9+wbVy3Y7H32rdZaAAAAumBgtAMAAABYWSQ4AABAZ0hwAACAzpDgAAAAnSHBAQAAOkOC\nAwAAdIYEB6BPVNWaVXV6Vd1aVSc9gH0OrqpzVmZso6Wq/qmqfjvacQAwdpTvwQFYuarqhUkOTbJt\nkr8kmZ3k/a218x7gvi9O8roku7fWFjzgQMe4qmpJtm6tzRntWADoHyo4ACtRVR2a5GNJ/jPJRkk2\nS/LJJPuthO0fluTy8ZDcDEdVTRztGAAYeyQ4ACtJVa2X5D1JXtta+1pr7W+ttbtaa6e31v5fb83q\nVfWxqprfe32sqlbvXdujquZW1Zur6o9VdW1Vvbx37d1JjkhyYFX9tapeUVVHVtVXh9x/86pqi/7F\nv6peVlVXVNVfqurKqjp4yPnzhrxv96qa1Wt9m1VVuw+5dm5Vvbeqftzb55yqmryMz78o/rcOif/Z\nVbV3VV1eVTdX1duHrN+1qn5aVX/qrf2fqprUu/bD3rJf9T7vgUP2f1tVXZfki4vO9d6zVe8eO/eO\np1TVDVW1xwP6PxaAviLBAVh5dkuyRpKvL2fNO5I8Psljkjw6ya5J/mPI9Y2TrJdkapJXJDmmqtZv\nrb0rg1WhE1tr67TWPr+8QKpq7ST/nWSv1tq6SXbPYKvc0usekuSM3toNknwkyRlVtcGQZS9M8vIk\nD00yKclblnPrjTP4dzA1gwnZZ5O8KMljk/xTkndW1Ra9tQuTvCnJ5Az+3f1zkn9Pktbak3prHt37\nvCcO2f8hGaxmvWrojVtrv0/ytiRfraq1knwxyZdba+cuJ14AOkaCA7DybJDkxhW0kB2c5D2ttT+2\n1m5I8u4kLx5y/a7e9btaa2cm+WuSbe5nPHcn2aGq1mytXdtau+Re1jwzye9aa19prS1orR2f5DdJ\nnjVkzRdba5e31m5PMjODydmy3JXB543uSnJCBpOXj7fW/tK7/6UZTOzSWvtla+1nvfteleQzSZ48\njM/0rtbaHb14ltBa+2ySOUl+nmSTDCaUAIwjEhyAleemJJNX8GzIlCRXDzm+undu8R5LJUi3JVnn\nvgbSWvtbkgOTvDrJtVV1RlVtO4x4FsU0dcjxdfchnptaawt7Py9KQK4fcv32Re+vqkdU1Ter6rqq\n+nMGK1T32v42xA2ttb+vYM1nk+yQ5BOttTtWsBaAjpHgAKw8P01yR5JnL2fN/Ay2Vy2yWe/c/fG3\nJGsNOd546MXW2tmttT0zWMn4TQb/xX9F8SyKad79jOm++FQG49q6tfagJG9PUit4z3JHf1bVOhkc\n8vD5JEf2WvAAGEckOAArSWvt1gw+d3JM7+H6tapqtaraq6o+3Ft2fJL/qKoNew/rH5Hkq8vacwVm\nJ3lSVW3WG3Bw+KILVbVRVe3Xexbnjgy2ut19L3ucmeQRVfXCqppYVQcm2T7JN+9nTPfFukn+nOSv\nverSa5a6fn2SLe/jnh9Pcn5r7ZUZfLbo0w84SgD6igQHYCVqrR2dwe/A+Y8kNyS5JskhSb7RW/K+\nJOcnuSjJr5Nc0Dt3f+717SQn9vb6ZZZMSgZ6ccxPcnMGn21ZOoFIa+2mJPskeXMGW+zemmSf1tqN\n9yem++gtGRxg8JcMVpdOXOr6kUm+3JuydsCKNquq/ZLMyD8+56FJdl40PQ6A8cEXfQIAAJ2hggMA\nAHSGBAcAABgVVTWjqn5bVXOq6rB7ub5ZVX2/qi6sqouqau8V7qlFDQAAWNWqakKSy5PsmWRukllJ\nXtBau3TImmOTXNha+1RVbZ/kzNba5svbVwUHAAAYDbsmmdNau6K1dmcGvyB6v6XWtCQP6v28Xobx\n1QrL+zI6emrimq0mrTvaYUAnPGqbTUc7BOiMCQMr+tog4L648IJf3tha23C041iZJjzoYa0tuH3F\nC0dAu/2GS5IM/XLmY1trxw45nprBaaOLzE3yuKW2OTLJOVX1uiRrJ3naiu4rwRmGmrRuVt9mhRNK\ngWE483tHj3YI0BkPXnvSaIcAnbLO6gNXj3YMK1tbcPuo/Xvs32cf8/fW2vQHuM0LknyptXZ0Ve2W\n5CtVtUNr7d6+2y2JFjUAAGB0zEsytLVjWu/cUK9IMjNJWms/TbJGksnL21SCAwAAnVVJDYzOa8Vm\nJdm6qraoqklJDkpy2lJr/pDkn5OkqrbLYIJzw/I2leAAAACrXGttQZJDkpyd5LIkM1trl1TVe6pq\n396yNyf516r6VZLjk7ysrWAMtGdwAACAUdFaOzPJmUudO2LIz5cmecJ92VOCAwAAXVVJanxNXNSi\nBgAAdIYKDgAAdNnwHvjvjPH1aQEAgE6T4AAAAJ2hRQ0AALrMkAEAAID+pIIDAACdVYYMAAAA9CsJ\nDgAA0Bla1AAAoMsMGQAAAOhPKjgAANBVFUMGAAAA+pUEBwAA6AwtagAA0FllyAAAAEC/UsEBAIAu\nM2QAAACgP0lwAACAztCiBgAAXWbIAAAAQH9SwQEAgM4qQwYAAAD6lQQHAADoDC1qAADQVRVDBgAA\nAPqVCg4AAHSZIQMAAAD9SYIDAAB0hhY1AADoLN+DAwAA0LdUcAAAoMsGjIkGAADoSxIcAACgM7So\nAQBAV1UMGQAAAOhXKjgAANBlZcgAAABAX5LgAAAAnaFFDQAAOqsMGQAAAOhXKjgAANBlhgwAAAD0\nJwkOAADQGVrUAACgywwZAAAA6E8qOAAA0FVVhgwAAAD0KwkOAADQGVrUAACgywwZAAAA6E8qOAAA\n0GWGDAAAAPQnCQ4AANAZWtQAAKCzypABAACAfqWCAwAAXWbIAAAAQH+S4AAAAJ2hRQ0AALqqYsgA\nAABAv1LBAQCAzjImGgAAoG9JcAAAgM7QogYAAF3me3AAAAD6kwoOAAB0mSEDAAAA/UkFBwAAuswz\nOAAAAP1JggMAAHSGFjUAAOiqKkMGAAAA+pUKDgAAdJkhAwAAAP1JggMAAHSGBAcAADqsqkblNczY\nZlTVb6tqTlUddi/XP1pVs3uvy6vqTyva0zM4AADAKldVE5Ick2TPJHOTzKqq01prly5a01p705D1\nr0uy04r2leAAAEBHVTLsasoo2DXJnNbaFUlSVSck2S/JpctY/4Ik71rRplrUAACAkTC5qs4f8nrV\nUtenJrlmyPHc3rl7qKqHJdkiyfdWdFMVHAAAYCTc2FqbvpL2OijJya21hStaKMEBAICuqt5rbJqX\nZNMhx9N65+7NQUleO5xNtagBAACjYVaSratqi6qalMEk5rSlF1XVtknWT/LT4WyqggMAAJ01/JHN\nq1prbUFVHZLk7CQTknyhtXZJVb0nyfmttUXJzkFJTmitteHsK8EBAABGRWvtzCRnLnXuiKWOj7wv\ne2pRAwAAOkMFBwAAOmystqiNFBUcAACgM1RwAACgw1RwAAAA+pQEBwAA6AwtagAA0GFa1GCM23P3\n7fKrr78zF5/6rrzl5Xve4/qmG6+fs459fX56/NvyixMPzzOeuP0oRAlj1/e/c06etOuj8oTHbp//\n+dhR97j+s5/8KDP2eHwetuHa+eapX1t8fu41V2fGHo/P05+0a5662075yhc/uyrDhjHp22eflZ12\n2DY7brd1jj7qg/e4fscdd+QlBx+UHbfbOns88fG5+qqrkiQnHn9cdttlp8WvddeYkIt+NXsVRw/d\nJMGhrwwMVD522AHZ75BPZqfnvS/7z3hstt1y4yXWvO2VM3LKty/Ibi/4UF5y+Bfz8cMPHKVoYexZ\nuHBh/uOtb8hXZp6a7/90dk49ZWYu/81lS6yZOm3TfOSYz+bZz1/yd+ehG22SU8/+Qc754S9y+rd/\nlGM+dlSuu3b+qgwfxpSFCxfm0Dcckq+ddmbO/9UlOenEE3LZZZcusebLX/x8HvzgB+eiy36X177+\njXnnOw5Lkhz4goPz01kX5qezLsxnv/i/2XzzLbLjox8zGh+DrqtRfI0SCQ59ZZcdNs/vr7kxV827\nKXctWJiTzr4g++yx4xJrWmt50NprJEnWW2fNXHvDraMRKoxJs385K5tvsVUetvmWmTRpUvZ77v45\n51unL7Fm0802z/aPfFQGBpb8R8SkSZOy+uqrJ0nuvPOO3H333assbhiLzp/1i2y51cOzxZaDv0/P\nP+DAnHH6qUusOeP003Lwi1+aJHnOc5+fc7//3bTWllhz8onH53kH+I9xsLJIcOgrUx66XuZef8vi\n43nX35KpG663xJr3f+bMHLT3rplz1nvz9U+8Jod+6KRVHSaMWddeOz+bTJ22+HjjKVNz7X2owsyf\ne02e9sTp2eVRD8+/v+Et2XiTKSMRJvSF+fPnZdqm//h9mjp1WubPm3fPNdM2TZJMnDgx6z1ovdx0\n001LrDnlpJnZ/8AXjHzAME6MWIJTVa2qjh5y/JaqOnIE7vP2pY5/srLvQX85YMb0fPX0n+XhM96Z\n57zuU/n8+14y7h6ug5EyZdqm+c555+e88y/JSSd8NTf88frRDgn62qxf/DxrrrVWHvnIHUY7FDqq\nUqkanddoGckKzh1JnltVk0fwHkmyRILTWtt9hO/HKJr/x1szbaP1Fx9P3Wj9zFuqBe2lz94tp5xz\nQZLk5xddmTUmrZbJD157lcYJY9Umm0zJtfPmLj6+bv68bHI/qjAbbzIl2267fX7+0x+vzPCgr0yZ\nMjVzr/nH79O8eXMzZerUe66Ze02SZMGCBbn1z7dmgw02WHz95JknZP8DD1o1AcM4MZIJzoIkxyZ5\n09IXqmrDqjqlqmb1Xk8Ycv7bVXVJVX2uqq5elCBV1Teq6pe9a6/qnftgkjWranZVHdc799fenydU\n1TOH3PNLVfX8qppQVUf17ntRVf3bCP4dsJKdf8nVefhmG+ZhUzbIahMnZP9n7Jwzzr1oiTXXXHdz\n9th1myTJNltslDVWXy033PLX0QgXxpxH7zw9V14xJ3+4+srceeedOfVrJ2XPGfsM673z583N7bff\nniT5059uyS9+/pNstfUjRjJcGNMeO32X/H7O73LVlYO/TyfPPDF777PvEmv23udZOe4rX06SfP1r\nJ+fJezx18X/Zvvvuu/O1U07K8/eX4DCyxlsFZ6S/B+eYJBdV1YeXOv/xJB9trZ1XVZslOTvJdkne\nleR7rbUPVNWMJK8Y8p5/aa3dXFVrJplVVae01g6rqkNaa/c2duTEJAckOaOqJiX55ySv6e15a2tt\nl6paPcmPq+qc1tqVQ9/cS6JelSRZbZ0H9rfASrNw4d1504dm5vRPvjYTBipfPvVnueyK6/LO1zwz\nF1z6h5zxg1/nsI98PZ985wvyuhc9Ja0l/3rEV0Y7bBgzJk6cmPd++GM5+PnPyt0LF+bAg1+abbbb\nPkf957vz6J0em6fvtU9mX3B+XvniA3Prrbfk22edmY988L353k8vzJzLf5P3vPOwVFVaa/m3174x\n222vrYbxa+LEiTn6Y5/Is/eZkYULF+bFL3t5tt/+kXnvu4/IzjtPzzOftW9e+vJX5JUvf0l23G7r\nrP+Qh+RLXzl+8fvP+9EPM23aptliyy1H8VNA99TSkzxW2sZVf22trVNV70lyV5Lbk6zTWjuyqv6Y\nZOhTrRsm2SbJeUmesyjZqKqbkzyitXZj7/md5/TWb57kGa21ny26z73cd40klyfZOsmMJAe01g6u\nqpOT7Jjktt5b1kvyb621c5b1WQbWemhbfZsDHthfCJAkmfO9o1e8CBiWB689abRDgE5ZZ/WBX7bW\npo92HCvTxA22bOvu9d5RufefjnvRqPx9jnQFJ0k+luSCJF8ccm4gyeNba38funBZpayq2iPJ05Ls\n1lq7rarOTbLG8m7aWvt7b90zkhyY5IRF2yV5XWvt7Pv6QQAAoN+Mt2FLIz4murV2c5KZWbLd7Jwk\nr1t0UFWLWsx+nMG2slTV05Msepp8vSS39JKbbZM8fshed1XVasu4/YlJXp7kn5Kc1Tt3dpLXLHpP\nVT2iqjyBDgAAHbCqvgfn6CRDp6m9Psn03kP+lyZ5de/8u5M8vaouTrJ/kuuS/CWDycnEqrosyQeT\n/GzIXsdm8Dmf4+7lvuckeXKS77TW7uyd+1ySS5Nc0LvPZ7JqKlkAALDKGTKwkgx9Lqa1dn2StYYc\n35jBtrGl3ZrBZ2sWVNVuSXZprd3Ru7bXMu7ztiRvW8Z970rykKXW353B0dJLjJcGAAD631irXGyW\nZGZVDSS5M8m/jnI8AABAHxlTCU5r7XdJdhrtOAAAoBOq9xpHVtUzOAAAACNuTFVwAACAlcuYaAAA\ngD4lwQEAADpDixoAAHRUZXS/k2Y0qOAAAACdoYIDAAAdpoIDAADQpyQ4AABAZ2hRAwCALhtfHWoq\nOAAAQHeo4AAAQFeVIQMAAAB9S4IDAAB0hhY1AADoMC1qAAAAfUoFBwAAOkwFBwAAoE9JcAAAgM7Q\nogYAAB1VKS1qAAAA/UoFBwAAumx8FXBUcAAAgO6Q4AAAAJ2hRQ0AALqqfA8OAABA31LBAQCADlPB\nAQAA6FMSHAAAoDO0qAEAQIdpUQMAAOhTKjgAANBl46uAo4IDAAB0hwQHAADoDC1qAADQYYYMAAAA\n9CkVHAAA6KiqUsEBAADoVxIcAACgM7SoAQBAh2lRAwAA6FMqOAAA0GEqOAAAAH1KggMAAHSGFjUA\nAOiy8dWhpoIDAAB0hwoOAAB0mCEDAAAAfUqCAwAAdIYWNQAA6KrSogYAANC3VHAAAKCjKsk4K+Co\n4AAAAN0hwQEAADpDixoAAHRWGTIAAACwKlTVjKr6bVXNqarDlrHmgKq6tKouqar/W9GeKjgAANBh\nY7WAU1UTkhyTZM8kc5PMqqrTWmuXDlmzdZLDkzyhtXZLVT10Rfuq4AAAAKNh1yRzWmtXtNbuTHJC\nkv2WWvOvSY5prd2SJK21P65oUxUcAADosFF8BmdyVZ0/5PjY1tqxQ46nJrlmyPHcJI9bao9HJElV\n/TjJhCRHttbOWt5NJTgAAMBIuLG1Nv0B7jExydZJ9kgyLckPq+pRrbU/LesNWtQAAIDRMC/JpkOO\np/XODTU3yWmttbtaa1cmuTyDCc8ySXAAAKCranDIwGi8hmFWkq2raouqmpTkoCSnLbXmGxms3qSq\nJmewZe2K5W0qwQEAAFa51tqCJIckOTvJZUlmttYuqar3VNW+vWVnJ7mpqi5N8v0k/6+1dtPy9vUM\nDgAAdFQlGRgYo3Oik7TWzkxy5lLnjhjyc0tyaO81LCo4AABAZ0hwAACAztCiBgAAHTZ6X4MzOlRw\nAACAzlDBAQCADqtxVsJRwQEAADpDggMAAHSGFjUAAOiqMmQAAACgb6ngAABAR1UMGQAAAOhbEhwA\nAKAztKgBAEBnlRY1AACAfqWCAwAAHTbOCjgqOAAAQHdIcAAAgM7QogYAAB1myAAAAECfUsEBAICu\nKkMGAAAA+pYEBwAA6AwtagAA0FEVQwYAAAD6lgoOAAB02Dgr4KjgAAAA3SHBAQAAOkOLGgAAdJgh\nAwAAAH1KBQcAADpsnBVwVHAAAIDukOAAAACdoUUNAAC6qgwZAAAA6FsqOMOw9ZZT8pkT3j3aYUAn\nPObN3xjtEKAzrvr0AaMdAjDGVQwZAAAA6FsSHAAAoDO0qAEAQGeVIQMAAAD9SgUHAAA6bJwVcFRw\nAACA7pDgAAAAnaFFDQAAOsyQAQAAgD6lggMAAF1VhgwAAAD0LQkOAADQGVrUAACgoyqGDAAAAPQt\nFRwAAOgwFRwAAIA+JcEBAAA6Q4saAAB02DjrUFPBAQAAukMFBwAAOsyQAQAAgD4lwQEAADpDixoA\nAHRVGTIAAADQt1RwAACgoyplyAAAAEC/kuAAAACdoUUNAAA6bJx1qKngAAAA3aGCAwAAHTYwzko4\nKjgAAEBnSHAAAIDO0KIGAAAdNs461FRwAACA7lDBAQCAjqpKapyVcFRwAACAzpDgAAAAnaFFDQAA\nOmxgfHWoqeAAAADdoYIDAAAdZsgAAABAn5LgAAAAnSHBAQCADhv8LpxV/xpebDWjqn5bVXOq6rB7\nuf6yqrqhqmb3Xq9c0Z6ewQEAAFa5qpqQ5JgkeyaZm2RWVZ3WWrt0qaUnttYOGe6+EhwAAOioSlIZ\ns0MGdk0yp7V2RZJU1QlJ9kuydIJzn2hRAwAARsLkqjp/yOtVS12fmuSaIcdze+eW9ryquqiqTq6q\nTVd0UxUcAABgJNzYWpv+APc4PcnxrbU7qurfknw5yVOX9wYJDgAAdNjAmO1Qy7wkQysy03rnFmut\n3TTk8HNJPryiTbWoAQAAo2FWkq2raouqmpTkoCSnDV1QVZsMOdw3yWUr2lQFBwAAuqoqNdyZzatY\na21BVR2S5OwkE5J8obV2SVW9J8n5rbXTkry+qvZNsiDJzUletqJ9JTgAAMCoaK2dmeTMpc4dMeTn\nw5Mcfl/2lOAAAECHjdECzojxDA4AANAZEhwAAKAztKgBAEBHVZKBcdajpoIDAAB0hgoOAAB02Dgr\n4KjgAAAA3SHBAQAAOkOLGgAAdFiNsx41FRwAAKAzVHAAAKCjqgwZAAAA6FsSHAAAoDO0qAEAQIcN\njLMeNRUcAACgM1RwAACgw8ZX/UYFBwAA6BAJDgAA0Bla1AAAoMPKkAEAAID+pIIDAAAdVUkGxlcB\nRwUHAADoDgkOAADQGVrUAACgq6oMGQAAAOhXKjgAANBh46yAo4IDAAB0hwQHAADoDC1qAADQYYYM\nLKWqnltV6/Z+PqyqZlbVY0Y+NAAAgPtmOC1qR7bW/lJVuyfZO8lxST49smEBAAAPVCUZqNF5jZbh\nJDgLe3/uk+QzrbVTk6w+ciEBAADcP8N5BufaqjomyYwk06tqUgwnAAAAxqDhJDgHZLA17ROttVuq\nakqSw0Y2LAAAYGUYb0MGlpngVNWDhhyeNeTcX5P8eITjAgAAuM+WV8G5JEnL4LNJiyw6bkk2G8G4\nAACAlWB81W+Wk+C01jZdlYEAAAA8UMMaFlBVB1XV23s/T6uqx45sWAAAAPfdCocMVNX/JFktyZOS\n/GeS2zL4PTi7jGxoAADAA1GVDBgycA+7t9Z2rqoLk6S1dnNvVDQAAMCYMpwE566qGsjgYIFU1QZJ\n7h7RqAAAgJVinBVwhvUMzjFJTkmyYVW9O8l5ST40olEBAADcDyus4LTW/reqfpnkab1T+7fWLh7Z\nsAAAAO674bSoJcmEJHdlsE1tWJPXAACA0VfjrEdthclKVb0jyfFJpiSZluT/qurwkQ4MAADgvhpO\nBeclSXZqrd2WJFX1/iQXJvnASAYGAAA8cOOsgDOsdrNrs2QiNLF3DgAAYExZZoJTVR+tqo8kuTnJ\nJVX1uar6bJJfJ7lxVQUISfKLH303L5nxuBz89F3yf8d+/B7XZ37xk3nZM3fPK/Z9Ug592XNy3bxr\nFl+7fv7c/L9/eX5euvduedkzd891c/+wKkOHMeepO2ycn/7nXvnFB/bO6/fe9l7X7LfLpjnvfTPy\no/fOyKdf9fjF56c+ZK3MPPTJ+fH79sp575uRTTdYa1WFDWPSt88+KzvtsG123G7rHH3UB+9x/Y47\n7shLDj4oO263dfZ44uNz9VVXJUlOPP647LbLTotf664xIRf9avYqjh66aXktaosmpV2S5Iwh5382\ncuHAPS1cuDAff8/bctQXTs6GG03Jq/ffM7s/dUY2f/g2i9dsvd2j8umTv5M11lwrpx7/hXzmv47M\nuz76+STJB97273nRqw/N9Cfskdv/9tfUgDkZjF8DVfngix6b/Y8+N/Nvvj3nHLFnzpo9P5fP//Pi\nNVs+dJ28Ye/t8sz//G5uve2uTF539cXXjnnl4/LRb16aH1x6fdZefWLubm00PgaMCQsXLsyhbzgk\np515TqZOm5Yn7b5r9t5n32y33faL13z5i5/Pgx/84Fx02e9y0swT8s53HJb/Pe6EHPiCg3PgCw5O\nklx88a/zguc/Jzs++jGj9VHosEplYJz1qC0zwWmtfX5VBgLL8puLLsiUzbbIlE03T5I8de/n5Mff\n/dYSCc5Oj/+nxT9v/+jp+fZpJydJrprz2yxcuDDTn7BHkmTNtddZZXHDWLTzlg/JVX/8S66+4W9J\nkm/8/A/Z6zFTl0hwXvTkLfOF783JrbfdlSS58S93JEkeMeVBmTih8oNLr0+S/O2OBas4ehhbzp/1\ni2y51cOzxZZbJkmef8CBOeP0U5dIcM44/bS8/Z3vSpI857nPz5vf+Lq01paYanXyicfneQccuGqD\nhw5b4ZCBqtoqyfuTbJ9kjUXnW2uPGMG4YLEbr782D91kyuLjDTeekst+9ctlrj/z5OPyuCf9c5Jk\n7lW/zzrrPihHvO6luXbuH/LY3Z6Uf33zEZkwYcKIxw1j0SYPXjPzbr598fH8W27LY7fcYIk1W220\nbpLkjMP/OQMDlaNOvTjfu/i6bLXRurn1trvyxdc+IQ+bvHZ+cOn1ee/JF6niMG7Nnz8v0zadtvh4\n6tRpmfWLn99zzbRNkyQTJ07Meg9aLzfddFMmT568eM0pJ83MCad8Y9UEzfhThgzcmy8l+WKSSrJX\nkplJTnygN66qVlVHDzl+S1UdeT/3enBV/fv9fO9VVTV5xSvpB98+bWZ+e8nsHPiKQ5IkCxcsyK9/\n+bO8+q3vzqdP+nbmX3N1zvr68aMcJYxtEycMZMuN1s1+H/5e/u0zP81HXrZLHrTmapk4UHn81pNz\n5MzZ2fO9387mG66dFzxx89EOF/rarF/8PGuutVYe+cgdRjsU6IzhJDhrtdbOTpLW2u9ba/+RwUTn\ngbojyXNXUnLx4CT3muBU1XC/zJQxavJGm+SP185ffHzDdfMzeaNN7rHulz/5Qb766Y/m/Z/8aiZN\nGnxmYMNDn+LeAAAgAElEQVSNp2SrbXfIlE03z4SJE/PEp+2d31160SqLHcaaa/90e6Y+ZM3Fx1PW\nXyvX3nL7Emvm33xbzp49PwsWtvzhxr/l99f9JVtutG7m33J7Lr7mT7n6hr9l4d0tZ144Lzs+bP1V\n/RFgzJgyZWrmXjN38fG8eXMzZerUe66ZOzj4ZsGCBbn1z7dmgw3+UTU9eeYJ2f/Ag1ZNwDBODCfB\nuaOqBpL8vqpeXVXPSrLuSrj3giTHJnnT0heqasOqOqWqZvVeT+idP7Kq3jJk3cVVtXmSDybZqqpm\nV9VRVbVHVf2oqk5Lcmlv7Teq6pdVdUlVvWolxM8qsu2jdsq8q6/ItXOvzl133pnvnfn17P7UGUus\n+d2lF+Uj73pz3v/Jr2b9DTZcfH6bR+2Uv/7lz/nTzYOD/y782Y/ysK22CYxXF155c7bYaN1sNnnt\nrDZhIM9+3GY5a/a8JdZ868J5ecI2g79HD1lnUrbaeN1cfcNfc+GVN+dBa03KBr2hA/+03Ub57ZBn\nd2C8eez0XfL7Ob/LVVdemTvvvDMnzzwxe++z7xJr9t7nWTnuK19Oknz9ayfnyXs8dfHzN3fffXe+\ndspJef7+EhxGVlWNymu0DKe68aYkayd5fQafxVkvyb+spPsfk+SiqvrwUuc/nuSjrbXzqmqzJGcn\n2W45+xyWZIfW2mOSpKr2SLJz79yVvTX/0lq7uarWTDKrqk5prd20kj4HI2jCxIl5/Ts/mLe+Yv/c\nfffd2et5L8wWW2+bL/z3B7LNDo/JE566Vz591JG5/ba/5cg3viJJstEmU/P+Tx2XCRMm5DVvfXfe\n/LLnprWWRzzy0dln/xeP8ieC0bPw7pbDv3pBZh765AwMVI4/74r8dv6f87Zn75DZV92cs2fPz/cu\nvi57PHLjnPe+GVl4d8uRM2fnlr/dmSQ58sTZOeUte6QqueiqW/KVH1wxyp8IRs/EiRNz9Mc+kWfv\nMyMLFy7Mi1/28my//SPz3ncfkZ13np5nPmvfvPTlr8grX/6S7Ljd1ln/IQ/Jl77yjzbp8370w0yb\ntuniIQXAylFtlB4Oraq/ttbWqar3JLkrye1J1mmtHVlVf0wyf8jyDZNsk+QtSf7aWvuv3h4XJ9mn\nt+abrbUdeuf3SPKu1tpThtzvyCTP6R1unuQZrbWfVdVVSaa31pb4bp9eledVSbLRlGmPPeF7ZtPD\nyrD/h74z2iFAZ1z16QNGOwTolHVWH/hla236aMexMj304Tu0A486aVTu/T/P3X5U/j6XWcGpqq8n\nWWb201p77kqK4WNJLsjgIINFBpI8vrX296ViWpAl2+rWyLL9bcj79kjytCS7tdZuq6pzV/DetNaO\nzWALXbbZ4TFGBAEAQB9YXova/6yKAHptYzOTvCLJF3qnz0nyuiRHJUlVPaa1NjvJVelVbKpq5yRb\n9Nb/Jct/Lmi9JLf0kpttkzx+OWsBAIA+tbwv+vzuKozj6CSHDDl+fZJjquqiDMb4wySvTnJKkpdU\n1SVJfp7k8l6sN1XVj3sta99KcsZS+5+V5NVVdVmS3yb52Uh+GAAAGAsqGdUH/kfDqI1Qbq2tM+Tn\n65OsNeT4xiT3+Erf1trtSZ6+jP1euNSpc4dcuyPLGG3dWtv8PoQNAACMYb4jBgAAOmxgfBVwhvU9\nOEmSqlp9JAMBAAB4oFaY4FTVrlX16yS/6x0/uqo+MeKRAQAA3EfDaVH77wxOLvtGkrTWflVVT1n+\nWwAAgLFAi9q9rGmtXb3UuYUjEQwAAMADMZwKzjVVtWuSVlUTMvj9NJePbFgAAMADVTX+xkQPp4Lz\nmiSHJtksyfUZ/JLM14xkUAAAAPfHCis4rbU/JjloFcQCAADwgKwwwamqzyZpS59vrb1qRCICAABW\nmvE2ZGA4z+B8Z8jPayR5TpJrRiYcAACA+284LWonDj2uqq8kOW/EIgIAAFaacTZjYFhDBpa2RZKN\nVnYgAAAAD9RwnsG5Jf94Bmcgyc1JDhvJoAAAAO6P5SY4NTg0+9FJ5vVO3d1au8fAAQAAYOypJAPj\nrEdtuS1qvWTmzNbawt5LcgMAAIxZw5miNruqdmqtXTji0QAAACvV/Xnovp8tM8GpqomttQVJdkoy\nq6p+n+RvGax0tdbazqsoRgAAgGFZXgXnF0l2TrLvKooFAAAYR6pqRpKPJ5mQ5HOttQ8uY93zkpyc\nZJfW2vnL23N5CU4lSWvt9/cvXAAAYLSN1RkDVTUhyTFJ9kwyN4NdY6e11i5dat26Sd6Q5OfD2Xd5\nCc6GVXXosi621j4ynBsAAADci12TzGmtXZEkVXVCkv2SXLrUuvcm+VCS/zecTZeX4ExIsk56lRwA\nAKC/VNVYHhM9Nck1Q47nJnnc0AVVtXOSTVtrZ1TVA05wrm2tvec+hwkAAJBMrqqhz8sc21o7drhv\nrqqBJB9J8rL7ctMVPoMDAABwP9zYWpu+nOvzkmw65Hha79wi6ybZIcm5NViF2jjJaVW17/IGDSwv\nwfnnFYYMAACMaWO3Qy2zkmxdVVtkMLE5KMkLF11srd2aZPKi46o6N8lbVjRFbZnf+9Nau/kBBgwA\nAHCvet+5eUiSs5NclmRma+2SqnpPVd3vr6pZXgUHAADocwNjt4KT1tqZSc5c6twRy1i7x3D2XGYF\nBwAAoN9IcAAAgM7QogYAAB1VyVj+HpwRoYIDAAB0hgoOAAB02Dgr4KjgAAAA3aGCAwAAXVVje0z0\nSFDBAQAAOkOCAwAAdIYWNQAA6LDK+OpRU8EBAAA6QwUHAAA6avCLPkc7ilVLBQcAAOgMCQ4AANAZ\nWtQAAKDDtKgBAAD0KRUcAADosKrxVcJRwQEAADpDggMAAHSGFjUAAOgo34MDAADQx1RwAACgqyoZ\nZzMGVHAAAIDukOAAAACdoUUNAAA6bGCc9aip4AAAAJ2hggMAAB1lTDQAAEAfk+AAAACdoUUNAAA6\nbJzNGFDBAQAAukMFBwAAOqsykPFVwlHBAQAAOkOCAwAAdIYWNQAA6KiKIQMAAAB9SwUHAAC6qpIB\nFRwAAID+JMEBAAA6Q4saAAB02MA4mzKgggMAAHSGCg4AAHSUMdEAAAB9TIIDAAB0hhY1AADoMEMG\nAAAA+pQKDgAAdNg4K+Co4AAAAN0hwQEAADpDixoAAHRUZfxVNMbb5wUAADpMBQcAALqqkhpnUwZU\ncAAAgM6Q4AAAAJ2hRQ0AADpsfDWoqeAAAAAdooIDAAAdVUkGDBkAAADoTxIcAACgM7SoAQBAh42v\nBjUVHAAAoENUcAAAoMPG2YwBFRwAAKA7JDgAAEBnaFEDAIDOqtQ461FTwQEAADpDBQcAADqqMv4q\nGuPt8wIAAB0mwQEAADpDixoAAHSYIQMAAAB9SgUHAAA6bHzVb1RwAACADpHgAAAAnaFFbRjWWX1i\nHr/VBqMdBnTCL//r2aMdAnTG5Me9brRDAMa6MmQAAACgb6ngAABAR1XGX0VjvH1eAABgjKiqGVX1\n26qaU1WH3cv1V1fVr6tqdlWdV1Xbr2hPCQ4AALDKVdWEJMck2SvJ9klecC8JzP+11h7VWntMkg8n\n+ciK9tWiBgAAHTaGhwzsmmROa+2KJKmqE5Lsl+TSRQtaa38esn7tJG1Fm0pwAACA0TA1yTVDjucm\nedzSi6rqtUkOTTIpyVNXtKkWNQAA6LAapVeSyVV1/pDXq+5P/K21Y1prWyV5W5L/WNF6FRwAAGAk\n3Nham76c6/OSbDrkeFrv3LKckORTK7qpCg4AADAaZiXZuqq2qKpJSQ5KctrQBVW19ZDDZyb53Yo2\nVcEBAIAOG6szBlprC6rqkCRnJ5mQ5AuttUuq6j1Jzm+tnZbkkKp6WpK7ktyS5KUr2leCAwAAjIrW\n2plJzlzq3BFDfn7Dfd1TggMAAB1VSQYyRks4I8QzOAAAQGdIcAAAgM7QogYAAB02VocMjBQVHAAA\noDNUcAAAoLMqZcgAAABAf5LgAAAAnaFFDQAAOsyQAQAAgD6lggMAAB1VSQYMGQAAAOhPEhwAAKAz\ntKgBAEBXlSEDAAAAfUsFBwAAOkwFBwAAoE+p4AAAQIeVMdEAAAD9SYIDAAB0hhY1AADoqEoyML46\n1FRwAACA7lDBAQCADjNkAAAAoE9JcAAAgM7QogYAAB1W46tDTQUHAADoDhUcAADoMEMGAAAA+pQE\nBwAA6AwtagAA0FGVZGB8daip4AAAAN2hggMAAJ1VhgwAAAD0KwkOAADQGVrUAACgqyqp8dWhpoID\nAAB0hwoOAAB02Dgr4KjgAAAA3SHBAQAAOkOLGgAAdFQlGRhnUwZUcAAAgM5QwQEAgA4bX/UbFRwA\nAKBDJDgAAEBnaFEDAIAuG2c9aio4AABAZ6jgAABAh9U4K+Go4AAAAJ0hwQEAADpDixoAAHRYja8O\nNRUcAACgO1RwAACgw8ZZAUcFBwAA6A4JDgAA0Bla1AAAoMvGWY+aCg4AANAZKjgAANBRlaTGWQlH\nBQcAAOgMCQ4AANAZWtQAAKCrKqnx1aGmggMAAHSHCg4AAHTYOCvgqOAAAADdIcEBAAA6Q4saAAB0\n2TjrUVPBAQAAOkMFBwAAOqtS46yEo4IDAAB0hgQHAADoDC1qAADQYTW+OtRUcAAAgO5QwQEAgI6q\njLsp0So4AABAd0hwAACAztCiBgAAXTbOetRUcAAAgM6Q4AAAQIfVKP1vWLFVzaiq31bVnKo67F6u\nH1pVl1bVRVX13ap62Ir2lOAAAACrXFVNSHJMkr2SbJ/kBVW1/VLLLkwyvbW2Y5KTk3x4RftKcAAA\ngNGwa5I5rbUrWmt3JjkhyX5DF7TWvt9au613+LMk01a0qSEDAADQYTV2hwxMTXLNkOO5SR63nPWv\nSPKtFW0qwQEAAEbC5Ko6f8jxsa21Y+/PRlX1oiTTkzx5RWslOAAA0GGjWMC5sbU2fTnX5yXZdMjx\ntN65JVTV05K8I8mTW2t3rOimnsEBAABGw6wkW1fVFlU1KclBSU4buqCqdkrymST7ttb+OJxNJTgA\nAMAq11pbkOSQJGcnuSzJzNbaJVX1nqrat7fsqCTrJDmpqmZX1WnL2G4xLWoAANBVlVHtUVuR1tqZ\nSc5c6twRQ35+2n3dUwUHAADoDBUcAADosBrLJZwRoIJDXzjn7LOy4yO3ySO3fXiO+vAH73H9jjvu\nyIteeGAeue3D80+7Py5XX3XV4mu/vuiiPPmJu2XnRz8y0x/zqPz9739fhZHD2HPud8/JU3Z9VJ40\nfft88mNH3eP6z3/yo+z9lMdny4eunTNO+9o9rv/lz3/O43bYKu986xtXRbjQN/bcfbv86uvvzMWn\nvitvefme97i+6cbr56xjX5+fHv+2/OLEw/OMJy79he3AyiDBYcxbuHBh3vj61+bU07+VCy+6NCed\ncHwuu/TSJdZ86Qufz/oPXj+X/GZOXveGN+Udb39bkmTBggX5l5e+KJ845tO54FeX5OzvnpvVVltt\nND4GjAkLFy7MO9/6hnx55qn5zk9m57Svzczlv7lsiTVTpm2ao//ns9nveQfe6x5Hf+Dd2XX3J6yK\ncKFvDAxUPnbYAdnvkE9mp+e9L/vPeGy23XLjJda87ZUzcsq3L8huL/hQXnL4F/Pxw+/9dwx4YCQ4\njHmzfvGLbLXVw7PFlltm0qRJ2f/Ag/LN009dYs03Tz81B7/4pUmS5z7v+Tn3e99Nay3f+fY52eFR\nO2bHRz86SbLBBhtkwoQJq/wzwFgx+4JZ2XyLrbLZ5oO/T896zv759rdOX2LNppttnu0e+agMDNzz\nHxG/nn1Bbrzhj3nSHvf5mU/otF122Dy/v+bGXDXvpty1YGFOOvuC7LPHjkusaa3lQWuvkSRZb501\nc+0Nt45GqIwzlaRqdF6jRYLDmDd//rxMm/aP74CaOnVa5s2bd881mw6umThxYh603nq56aab8rvL\nL09V5Vl7PyO77bJzjv6vD6/S2GGsue7a+dlk6rTFx5tMmZrrrp0/rPfefffded8Rb8s73v2BkQoP\n+taUh66Xudffsvh43vW3ZOqG6y2x5v2fOTMH7b1r5pz13nz9E6/JoR86aVWHCePCKk9wqmphb4b1\nxVV1UlWtdT/2+FxVbd/7+e1LXfvJyoqV/rdg4YL85Cfn5Yv/e1y++4Pzcto3vp7vf++7ox0W9KX/\n/fxn8pSnzVgiQQKG74AZ0/PV03+Wh894Z57zuk/l8+97SWo0/zM340aN0mu0jEYF5/bW2mNaazsk\nuTPJq+/rBq21V7bWFj2E8falru2+EmJkDJkyZWrmzr1m8fG8eXMzderUe665ZnDNggUL8udbb80G\nG2yQqVOn5YlPfFImT56ctdZaKzP22jsXXnjBKo0fxpKNN5mSa+fNXXx87fx52XiTKcN67wXn/yxf\n/tyn8oTHPCLvf9fh+dqJx+WD7/6PkQoV+sr8P96aaRutv/h46kbrZ95SLWgvffZuOeWcwX8G/fyi\nK7PGpNUy+cFrr9I4YTwY7Ra1HyV5eJJU1aG9qs7FVfXG3rm1q+qMqvpV7/yBvfPnVtX0qvpgkjV7\nFaHjetf+2vvzhKp65qIbVdWXqur5VTWhqo6qqllVdVFV/duq/tDcN9N32SVz5vwuV115Ze68886c\ndOIJeeY++y6x5pn77JvjvvLlJMnXTjk5T37KU1NV2fPpz8glF/86t912WxYsWJAf/fAH2W47U2sY\nvx690/RcecWc/OHqwd+n079+Uvbca59hvfe/P/Pl/PSiOfnx7Mvzjnd/IM898OAc9q73jXDE0B/O\nv+TqPHyzDfOwKRtktYkTsv8zds4Z5160xJprrrs5e+y6TZJkmy3+f3v3HrfZXO9//PVmnDJisJFS\nlEM5xDbGIbuihHHIdCCnGOQUqk3t7F/xw27vDjrsVCr1i51sHfZPETKKX9KBaJwrlFIqP4dQzozP\n/mOtaV/uRmbMsK57Xa/n43E/5r7XWtdan/t+WK7rsz6f7/e7Iosvtgi333VvF+FKvdbZOjhJJgBT\ngfOSTAb2ATahqWhdmuQi4IXA76tq+/Y1j2tmraojkxxaVRvM4RJfAXYBzkmyKPBq4GBgP+CeqpqS\nZDHgB0nOr6pfjYnvAOAAgFWe//wF9ntr3k2YMIGPffyT7Lj9NsyaNYu9p+/L2uusw3HHHM2Gkzdi\nhx1fy/R992Pf6W9mnRevzqRJy3LqaV8GYNKkSbztHYfzD5tNIQnbbLsdU7fb/kmuKPXXhAkTOO6D\n/85eO+/IrFmz2GX3vVnzxWvzkfcfy0s3mMxrpu7AVTMv54C93sQ999zFd2acy8c+8C9854dXdB26\nNNRmzXqMf/zgV/nmiYew8ELhP868hJ/ddCtHHbw9M3/6G8656BqO/OjXOfGo3Thszy2pgv2PPrXr\nsDUqRqwTMlX1zF4wmQVc0/54MXAETeKxXFUd3R7zL8DtwHnA+TTJytlVdXG7/7vAO6vq8iT3VtXE\ngfPfW1UTkywO3ACsAWwL7FJVeyT5L+ClwP3tS5YGDqyq858o5smTN6ofXHr5gvkDSCPutj891HUI\nUm+s9eojug5B6pUHr/zUT6pqo67jWJDWXX/D+tp5F3dy7bVXntjJ37OLCs4DYysuTzTArqpuSLIh\nsB3wviQXVNVxc3ORqnqwTYS2Ad4EfHn25YDDqmrGU4xfkiRJGjcyYiWcrsfgzHYxMC3Js5IsCbwO\nuDjJysD9VfUl4Hhgwzm89pEkT7Ry41doWt9eTlMNApgBHDz7NUnWbK8pSZIkaZzrbAzOoKqameQU\n4Mftps9X1RVJtgGOT/IY8AhNK9tYJwFXJ5lZVXuM2Xc+cCpwZlU9PPvcwKrAzDSlo9uBaQv0F5Ik\nSZLUiWc8wRkcLzNm+0eBj47ZNoOm4jL22C0Gvn838O45nb+qHgGWHfPax2imln7c9NKSJElSH43a\nckvD0qImSZIkSfNtKFrUJEmSJD09RqyAYwVHkiRJUn+Y4EiSJEnqDVvUJEmSpD4bsR41KziSJEmS\nesMKjiRJktRTATJiJRwrOJIkSZJ6wwqOJEmS1FdxoU9JkiRJGrdMcCRJkiT1hi1qkiRJUo+NWIea\nFRxJkiRJ/WEFR5IkSeqzESvhWMGRJEmS1BsmOJIkSZJ6wxY1SZIkqbdCRqxHzQqOJEmSpN6wgiNJ\nkiT1WEargGMFR5IkSVJ/mOBIkiRJ6g1b1CRJkqSeCiO3DI4VHEmSJEn9YQVHkiRJ6rMRK+FYwZEk\nSZLUGyY4kiRJknrDFjVJkiSpxzJiPWpWcCRJkiT1hhUcSZIkqccyWgUcKziSJEmS+sMER5IkSVJv\n2KImSZIk9diIdahZwZEkSZLUH1ZwJEmSpL6KkwxIkiRJ0rhlgiNJkiSpN2xRkyRJknpttHrUrOBI\nkiRJ6g0rOJIkSVJPBScZkCRJkqRxywRHkiRJUm/YoiZJkiT12Ih1qFnBkSRJktQfVnAkSZKkHnOS\nAUmSJEkap0xwJEmSJPWGLWqSJElSj2XEphmwgiNJkiSpN6zgSJIkSX02WgUcKziSJEmS+sMER5Ik\nSVJv2KImSZIk9diIdahZwZEkSZLUH1ZwJEmSpJ5Kmq9RYgVHkiRJUm+Y4EiSJEnqDVvUJEmSpB7L\niE0zYAVHkiRJUm9YwZEkSZL6bLQKOFZwJEmSJPWHCY4kSZKk3rBFTZIkSeqxEetQs4IjSZIkqRtJ\ntk1yfZJfJDlyDvtfkWRmkkeTvHFuzmkFR5IkSeqxDGkJJ8nCwKeA1wC3AJclOauqfjpw2G+A6cA7\n5/a8JjiSJEmSurAx8IuqugkgyZeBnYC/JDhV9et232Nze1Jb1CRJkiR14bnAbwd+vqXdNl+s4EiS\nJEm9FdLdNAPLJ7l84OeTquqkp/uiJjiSJEmSng53VNVGf2P/74BVBn5+XrttvpjgSJIkST0VhneS\nAeAyYI0kq9EkNrsCu8/vSR2DI0mSJOkZV1WPAocCM4CfAV+tquuSHJfktQBJpiS5BdgZ+GyS657s\nvFZwJEmSJHWiqs4Fzh2z7eiB7y+jaV2ba1ZwJEmSJPWGCY4kSZKk3rBFTZIkSeqxIZ5k4GlhBUeS\nJElSb5jgSJIkSeoNW9QkSZKkHguj1aNmBUeSJElSb1jBkSRJkvoqTjIgSZIkSeOWCY4kSZKk3rBF\nTZIkSeqptF+jxAqOJEmSpN6wgiNJkiT12YiVcKzgSJIkSeoNExxJkiRJvWGLmiRJktRjGbEeNSs4\nkiRJknrDCo4kSZLUYxmtAo4VHEmSJEn9YYIjSZIkqTdsUZMkSZJ6bMQ61KzgSJIkSeoPKziSJElS\nn41YCccKjiRJkqTeMMGRJEmS1Bu2qEmSJEk9lhHrUbOCI0mSJKk3rOBIkiRJPRUgo1XAsYIjSZIk\nqT9SVV3HMPSS3A7c3HUcelLLA3d0HYTUE95P0oLlPTU+vKCq/q7rIBakJOfR/PfXhTuqattn+qIm\nOOqNJJdX1UZdxyH1gfeTtGB5T0nPHFvUJEmSJPWGCY4kSZKk3jDBUZ+c1HUAUo94P0kLlveU9Axx\nDI4kSZKk3rCCI0mSJKk3THAkSZIk9YYJjiRJkqTeMMGRJM2zJEu1/6brWCRJGmSCI0maa2m8ALg8\nyeSqKpMcaf54D0kLlgmORs7sN5Ikz0myctfxSONJNW4GTgFOTrKBSY701CVJtVPaJtkqyeuTPDfJ\nwl3HJo1XJjgaOe2HsWnA6cCnk3wwyfO6jksadm31ZiGAqno/cCpwepK/N8mRnpqB5ObtwLHAJsCF\nwMZdxiWNZyY4GjlJ1gMOB3YAfgxsCdzTaVDSkJv9lLmqHksyCaCqjgc+h0mONF+SrAm8sqo2B34N\n/Aa4dGC/95U0D0xwNIpmAWcDOwPbA7tW1Z+TrNNtWNLwGnjK/I/Ax5KclmS1qvoocCLwxSRTZh8n\nae4kWQ74PXB1klOAacDU9mHC3kmW9r6S5o0JjkZGkrWT7Aw8DLwceCuwV1XdlGQq8LkkK3UapDTE\nkhwCvJbm3tmI5p7ZrKpOAE4DPplksS5jlMaTJJsA/0zz4G0lYHVgv6p6NMmewBHAUh2GKI1L8aGA\nRkWS/YF9quplSd5B0998IXA/8B7g3VV1dpcxSsNkcPBz+/PRwMnAG4FXAT8FpgKHVNXFSSZV1V3d\nRCsNt7bNLFX12MC21YALgLfQtKV9CLgLWBj4e2CPqrq2g3Clcc0ER701+8NZkglV9Wi77TTgkqr6\nRJK3AC8AlgXOrKrzx36gkwRJ/glYDHgfsAbw6ap6dbvvRuB84IiqerC7KKXhNma2tOWAh6rq3iRv\nALasqkOTrEFTyVkRuKydsVDSPJrQdQDSgtYO1ly/qr6WZDKwZZJfVNU3aJ4+bwNQVZ9vj1+kqh5p\nt5ncSAOSvA7YFDisfWDwx3b7NJo25yuAD5ncSHPWVm7WA44Cdm7fl44Efp3kCzSTCeyUZM2qugG4\nsbtopX5wDI76aCHgtnal9VuARYFDknwCeBSYmuTNA8c/2kGM0lAaHEOT5LnAq2laZf5/u/kBmvE2\n+wDHAMf6lFl6Yu3sg1cDhybZAriSJtm5DTgD2Bx4EfDhJIt2FqjUI7aoqZeSTADuoBlX89kkSwAf\nAW4GDgN+Dkyrqns7DFMaKkmWBKbTtJy9BFgLOBf4V+APNFWcR9v7aSFgyaq6raNwpaGXZImqeqD9\nfmHgCzTjPzeoqoeSbE0zscAbgecDG1XV3Z0FLPWECY56IcmzgNdU1ZntrDQPAwHOA/61qj7eLlC4\nErALcGNVndNdxNJwSrI98EXgTuDF7VS169HMnPYIzVibR7qMURoPkixOMwvauTRJzHpVdXTblrYZ\n/5PkTACWBJarqpu6i1jqDxMc9Ua7fsBGwIPA/lV1RZINge8A762qE8cc74QC0hhJXgL8B83UtLtW\n1VXtB7A1gXcCd1fV4V3GKA27JMtX1R1JXg5cBPyCJsF5qN1/Mk3r56aOX5MWPMfgaNwbWOH5/TQz\notdSA2QAAAqgSURBVD1aVVcAVNVMYCvg40nePvg6kxvp8ZK8nma2tM1oxgicmmTLdhbCZwOfBT7Q\nYYjSUEtjFeB9SSbSTKV+JvAcmgdwAFTVPsB1wPc6CVTqOSs4GtcGpoJeCJgITKLpcX6kqrYdOG4N\nYNWq+nZHoUpDL8kxNKuo719VlyXZhybROYP/WZPj1g5DlMaFJM8G1qUZp/btJK8CvgHsXlVnJ9m0\nqi5JsoLj2KQFzwqOxq2B5GZr4L007TQ3t+tzLJrkm0k2SXIRcGf7JpO/fVZp9CR5AUBVHQOcCnwq\nyZSqOplmUo6JwKEmN9ITG3x/qao/AesDRyfZtqouBPYEvpbkI8AXkjzP5EZ6ergOjsatNrnZlmZ2\ntEOB05OsDxxVVa9KcjpwLPCRqvrj7Nd0F7E0fNpxavsn+VZVnVVVH2mnqj0zyc5VdU6S86pqVtex\nSsNqzCKeuwP3VNWnkzwCvKvdf1aS1wCvpJnF85YuY5b6zBY1jUttS9pSNIOhj6JZ9fl44HfA3TTT\n2d6VZJmqutsJBaTG2HuhXVH9QGB54ILZswsmuRS4HXjD7IHRkv62JIcAbwF2qaob2227A/sCJ7RJ\nju9H0tPMCo7GlYE3hsWr6p4k+9FMLHAczQDOJYBbgd8mOW72egK+mUh/9ZR5b5oJBe4FPkgzQ9qW\nSZahWfz2apop1k1upCfRtqetDuwFbA/cmuR1wCrAl4BFgP2SXFBV93UXqTQaTHA0bgyMudkEODHJ\n9Kq6JskKNOveTAKWAy4Ezpi9uJqkx0tyELAHcCRwMXAXcDKwE7AdsDawZ1X9uqsYpWE3+MCg/ffG\ndsznl4HrgaVp7q23VdUxSc40uZGeGbaoaVxpx9y8kaZaswKwTZvkfIhmQOfqwFurakaHYUpDJcnz\naSbauK9tSTsBeBuwM/A6YIfBxTuTLDt73JqkvzamGvoymgdsV9IsJr0hcGFV/TLJATQLer7V1jTp\nmWOCo3EjyWrAecA+VfXDJEcD02naAX5Jk/Q8WlU/7i5KabgkWRH4X8Bvgc9U1b1J/p2mnXMFmmlr\nH0hyBPCTqvpud9FKw20OY9jeCexKM17tTuD7wGlV9ee2hfpgYHpVXdtJwNKIcppojSd3ApcCNwFU\n1XE0byYzgBWr6ocmN9JfuR24DFgZ2KcdK/AHYG/gzW1yswtNy9rN3YUpjQt/ae1PshKwDfDyqpoK\nfB14MbBOkhfRrB21j8mN9MyzgqOhNTDmZmmAdlKBbwDnV9WJ7THbAP8GBHhFVd3bXcTS8GgXt12o\nqq5vk5odgKnAVVX12SQnAuvQVHZWp1nc85ruIpaGWzvF877AVTTtaBcA3wOOmd0WneTTwN1V9c9J\nFnOSDqkbTjKgodUmNzsChwN3JbmEZlD06UmeBzwAvB7Yh2aa2yVpZoSSRlo7zuZ64I4kxwKzgJNo\nBj2vnuTAdkzAujTvA3e4Jof0xNrxn8fRLIS7ArAbzQQC/wlsnOSutoPgJ8CaSRY2uZG6Y4KjoTJm\n4OamNGMHdqZZAXr/qvpQkjcBWwHPp5mSc3lgc+CxbqKWhktV3ZlkK+A7NK3I6wNfoXkA8DCwblvV\nOaWqHuwuUmn4JVkWOBfYqaq+mWQV4MM0s3aeRzMG52NJrgNe1R7nwrhSh2xR09BI8nfANOD0diD0\nK4BlaNbqOJxmMPSvkqw6e/radvaaL9KsCm2fszSgbak5gSbBWZHmw9euwMY043A2r6p7uotQGh+S\nbA98CNisqv6U5DTgoqo6KckkYDVgVZqJOhzLJnXMBEdDI8k0mnECVwKnAFOAT9JMLvDaqrq7/cB2\nUPt1J/AcYIJvKNKctR/MPgZsWlV/bD+MLQI8y3VupLmXZCrNA4MZNJN27FlV93cblaQ5sUVNnWt7\nlWcB3wQWBragmd3p00nOoFmn4znthAJHA/9UVbe3L/9dFzFL40VVnZPkMeCSJJtV1Z1dxySNR1X1\nrSQHA+cDK1XV/UkWt81TGj5WcNSpJGsBb6F5w/heVT3UPiWbCvy0qj6T5BiaSs0ywBeqaoYLpknz\nJslOwDHA5KpyvJr0FLXvUR8Gtqyq27qOR9JfM8FRp5K8Evh/wI3AV4EXAscDrwEWBX5PMxC6fFIm\nzZ8kE51KXZp/7QOD/02zwHT5wE0aLiY46lySfwDOBjYB3gBMomlLu4VmfY5jgC8A+ORZkjQMfGAg\nDS/H4KhzVfX9JLsB/wW8rKr+nORsYD3gAOBXJjaSpGFiciMNLys4GhpJtgM+AUypqj+229K2pznm\nRpIkSU/KCo6GRlWd28729PMka1XVXbOTGpMbSZIkzQ0rOBo67bod91XVd7uORZIkSeOLCY6Glm1p\nkiRJmlcmOJIkSZJ6Y6GuA5AkSZKkBcUER5IkSVJvmOBIkiRJ6g0THEnqSJJZSa5Mcm2SryV51nyc\na4t2gVySvDbJkX/j2GWSvPUpXOOYJO+c2+1jjjklyRvn4VqrJrl2XmOUJMkER5K680BVbVBV6wIP\nAwcN7kxjnv8/XVVnVdUH/sYhywDznOBIkjQemOBI0nC4GFi9rVxcn+SLwLXAKkm2TvKjJDPbSs9E\ngCTbJvl5kpnA62efKMn0JJ9sv18xydeTXNV+vQz4APCitnp0fHvcu5JcluTqJMcOnOs9SW5I8n1g\nrSf7JZLs357nqiT/d0xVaqskl7fn26E9fuEkxw9c+8A5nHOdJD9u4706yRrz/ueVJI0KExxJ6liS\nCcBU4Jp20xrAiVW1DnAf8F5gq6raELgcODzJ4sDngB2BycBKT3D6E4CLqmp9YEPgOuBI4Jdt9ehd\nSbZur7kxsAEwOckrkkwGdm23bQdMmYtf54yqmtJe72fAfgP7Vm2vsT3wmfZ32A+4p6qmtOffP8lq\nY855EPDxqtoA2Ai4ZS7ikCSNqAldByBJI2yJJFe2318M/B9gZeDmqrqk3b4psDbwgyQAiwI/Al4M\n/KqqbgRI8iXggDlc41XAXgBVNQu4J8mkMcds3X5d0f48kSbhWQr4elXd317jrLn4ndZN8j6aNriJ\nwIyBfV+tqseAG5Pc1P4OWwMvHRifs3R77RsGXvcj4D1JnkeTQN04F3FIkkaUCY4kdeeBtirxF20S\nc9/gJuDbVbXbmOMe97r5FOD9VfXZMdd4x1M41ynAtKq6Ksl0YIuBfWNXlq722odV1WAiRJJV/3JQ\n1X8muZSm8nNukgOr6sKnEJskaQTYoiZJw+0SYPMkqwMkWTLJmsDPgVWTvKg9brcneP0FwMHtaxdO\nsjTwZ5rqzGwzgH0HxvY8N8kKwPeAaUmWSLIUTTvck1kK+EOSRYA9xuzbOclCbcwvBK5vr31wezxJ\n1kyy5OCLkrwQuKmqTgDOBF46F3FIkkaUFRxJGmJVdXtbCTk9yWLt5vdW1Q1JDgDOSXI/TYvbUnM4\nxduBk5LsB8wCDq6qHyX5QTsN87facTgvAX7UVpDuBfasqplJvgJcBdwGXDYXIR8FXArc3v47GNNv\ngB8DzwYOqqoHk3yeZmzOzDQXvx2YNuacuwBvTvIIcCvwb3MRhyRpRKVqbMeAJEmSJI1PtqhJkiRJ\n6g0THEmSJEm9YYIjSZIkqTdMcCRJkiT1hgmOJEmSpN4wwZEkSZLUGyY4kiRJknrjvwH418J52Ria\n+AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff500576a50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cmtx = confusion_matrix(y_true, y_pred)\n",
    "plot_confusion_matrix(cmtx, ['Negative', 'Neutral', 'Positive'], normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 5. Сабмишн"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "probs_dsts_fname = 'data/probs_dsts_test_x2_sized.npy'\n",
    "weight_dsts = 0.6\n",
    "\n",
    "probs_frames_fname = 'data/probs_frames_test.npy'\n",
    "weight_frames = 0.86\n",
    "\n",
    "probs_nn_fname = 'data/probs_nn_test_x2.npy'\n",
    "weights_nn = [0.7, 0.7, 0.7, 0.7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "715 (4,) 770\n"
     ]
    }
   ],
   "source": [
    "probs_dsts = np.load(probs_dsts_fname).item()\n",
    "probs_nn = np.load(probs_nn_fname)\n",
    "probs_frames = np.load(probs_frames_fname).item()\n",
    "\n",
    "print len(probs_dsts), probs_nn.shape, len(probs_frames)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "predictions = {}\n",
    "\n",
    "for fname in log_progress(probs_dsts.keys()):\n",
    "    preds = np.zeros((3,))\n",
    "    preds += probs_dsts[fname] * weight_dsts\n",
    "    for tmp_probs, w in zip(probs_nn, weights_nn):\n",
    "        preds += tmp_probs[fname] * w\n",
    "    predictions[fname] = np.argmax(preds)\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd2364f7a8ac473aa4f568fc326b70b7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions = {}\n",
    "\n",
    "for fname in log_progress(probs_dsts.keys()):\n",
    "    preds = np.zeros((3,))\n",
    "    if probs_dsts[fname].max() < 3.6:\n",
    "        preds += probs_dsts[fname] * weight_dsts\n",
    "    else:\n",
    "        preds += probs_dsts[fname] * weight_dsts / np.max(probs_dsts[fname])\n",
    "    preds += probs_frames[fname] * weight_frames\n",
    "    for tmp_probs, w in zip(probs_nn, weights_nn):\n",
    "        preds += tmp_probs[fname] * w\n",
    "    predictions[fname] = np.argmax(preds)\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SUBMISSION_OUT_DIR = 'submission5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d303d91531b645aaa3e8341bb01c3873"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if not os.path.exists(SUBMISSION_OUT_DIR):\n",
    "    os.makedirs(SUBMISSION_OUT_DIR)\n",
    "    \n",
    "for fname in log_progress(probs_frames.keys()):\n",
    "    if predictions.get(fname, None) is None:\n",
    "        predicted = np.argmax(probs_frames[fname])\n",
    "    else:\n",
    "        predicted = predictions[fname]\n",
    "    fname = os.path.basename(fname)\n",
    "    if fname == 'family-get-together-002.png':\n",
    "        fname = 'family-get-together-002_1.png'\n",
    "    fname = '.'.join(fname.split('.')[:-1])\n",
    "    if fname == 'Prime_Minister_Narendra_Modi_with_French_President_FranЗois_Hollande_at_the_G20_Summit':\n",
    "        fname = 'Prime_Minister_Narendra_Modi_with_French_President_Francis_Hollande_at_the_G20_Summit'\n",
    "    f = open(os.path.join(SUBMISSION_OUT_DIR, fname + '.txt'), 'w')\n",
    "    f.write('{}'.format(LBL2CLASS[predicted]))\n",
    "    f.flush()\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8423c7bdf3ae46088d6138e48f7fc184"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "f = open('data/subm.list', 'w')\n",
    "for fname in log_progress(predictions.keys()):\n",
    "    f.write('{}\\n'.format(os.path.basename(fname)))\n",
    "f.flush()\n",
    "f.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}