sim_new_votes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "IMPORTANT: submissions in test set should be different from train set, i.e., `train_test_different_submissions = True`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "为了使用邮件提醒功能，请设置环境变量MY_QQ_EMAIL（QQ邮箱地址）与MY_QQ_EMAIL_PWD（QQ邮箱授权码）\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:13 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "GPU ready...\n",
      "Smart using cuda:3\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:14 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['peer_vote_option'], at \u001b[0m\u001b[1;32m<ipython-input-1-1b7eb2607882>:6 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 1.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mpeer_vote_option\u001b[0m\u001b[1;33m str len 4: real\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:14 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "Finetune the model using peer votes\n"
     ]
    }
   ],
   "source": [
    "CONFIG_PATH = \"configs/finetune_CURIO_test_diff.yml\"\n",
    "from utils import get_config\n",
    "config = get_config(CONFIG_PATH, \"_sim\", print_config = False)\n",
    "peer_vote_option = \"real\" # \"real\", \"real_user_same_vote\", \"similar_user_same_vote\", \"real_user_opposite_vote\", \"similar_user_opposite_vote\"\n",
    "from superdebug import debug\n",
    "debug(peer_vote_option=peer_vote_option)\n",
    "use_voted_users_feature = config[\"use_voted_users_feature\"]\n",
    "use_voted_users_feature = False\n",
    "if use_voted_users_feature:\n",
    "    print(\"Use peer votes as model input feature\")\n",
    "else:\n",
    "    print(\"Finetune the model using peer votes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Version 7.5.0 of praw is outdated. Version 7.6.0 was released Tuesday May 10, 2022.\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter, defaultdict\n",
    "import os\n",
    "import random\n",
    "import shutil\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from process_data import get_model_input\n",
    "from model import get_best_model, TransformerVoter\n",
    "from utils import get_user_reps, get_bool_vec, record_existing_votes\n",
    "from train import evaluate_model, train_model\n",
    "import sklearn.metrics\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:20 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m at \u001b[0m\u001b[1;32m/home/TableSense/largedisk/wanrong/Curation-Modeling/process_data.py:364 get_model_input\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;36mLoading prepared data...\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:20 \u001b[0m\u001b[1;33m------------------\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "target, original_feature_map, categorical_features, string_features, train_data, test_data, test_data_info, train_submission_upvote_df, num_all_users = get_model_input(config)\n",
    "extra_input = (categorical_features, string_features, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:22 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['original_token_num'], at \u001b[0m\u001b[1;32m/home/TableSense/largedisk/wanrong/Curation-Modeling/model.py:251 get_tokenizer\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 2.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36moriginal_token_num\u001b[0m\u001b[1;33m num val: 30522\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:22 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:22 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m 1 vars: ['latest_token_num'], at \u001b[0m\u001b[1;32m/home/TableSense/largedisk/wanrong/Curation-Modeling/model.py:263 get_tokenizer\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m0 / 3.\u001b[0m \u001b[1;33m \u001b[0m\u001b[1;36mlatest_token_num\u001b[0m\u001b[1;33m num val: 30576\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:05:22 \u001b[0m\u001b[1;33m------------------\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at prajjwal1/bert-mini were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "model, _ = get_best_model(config, categorical_features, string_features, original_feature_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SUBMISSION_ID</th>\n",
       "      <th>SUBREDDIT</th>\n",
       "      <th>CREATED_TIME</th>\n",
       "      <th>USERNAME</th>\n",
       "      <th>VOTE</th>\n",
       "      <th>group_id</th>\n",
       "      <th>TITLE</th>\n",
       "      <th>author_id</th>\n",
       "      <th>AUTHOR</th>\n",
       "      <th>NSFW</th>\n",
       "      <th>LINK</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>#_COMMENTS</th>\n",
       "      <th>SUBMISSION_TEXT</th>\n",
       "      <th>SUBMISSION_URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62e3007451803b00160f2bd1</td>\n",
       "      <td>r/Team Mignon</td>\n",
       "      <td>Thu Jul 28 2022</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62d1ccae5631c64c66b3c4c3</td>\n",
       "      <td>I think it's less the number of years of free ...</td>\n",
       "      <td>62d6fb4c84f3b100160e615c</td>\n",
       "      <td>14</td>\n",
       "      <td>false</td>\n",
       "      <td></td>\n",
       "      <td>1.659044e+09</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>62fd4c8b0804680016acda57</td>\n",
       "      <td>r/LOLCHI</td>\n",
       "      <td>Wed Aug 17 2022</td>\n",
       "      <td>15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>619c6bb8762f45190c34df31</td>\n",
       "      <td>He's doing his best to minimize self-cites, wh...</td>\n",
       "      <td>62fabdaf73e60a001685c0fe</td>\n",
       "      <td>10</td>\n",
       "      <td>false</td>\n",
       "      <td></td>\n",
       "      <td>1.660767e+09</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              SUBMISSION_ID      SUBREDDIT     CREATED_TIME  USERNAME  VOTE  \\\n",
       "0  62e3007451803b00160f2bd1  r/Team Mignon  Thu Jul 28 2022         2   0.0   \n",
       "1  62fd4c8b0804680016acda57       r/LOLCHI  Wed Aug 17 2022        15   0.0   \n",
       "\n",
       "                   group_id  \\\n",
       "0  62d1ccae5631c64c66b3c4c3   \n",
       "1  619c6bb8762f45190c34df31   \n",
       "\n",
       "                                               TITLE  \\\n",
       "0  I think it's less the number of years of free ...   \n",
       "1  He's doing his best to minimize self-cites, wh...   \n",
       "\n",
       "                  author_id  AUTHOR   NSFW LINK     timestamp  #_COMMENTS  \\\n",
       "0  62d6fb4c84f3b100160e615c      14  false       1.659044e+09           0   \n",
       "1  62fabdaf73e60a001685c0fe      10  false       1.660767e+09           0   \n",
       "\n",
       "  SUBMISSION_TEXT SUBMISSION_URL  \n",
       "0                                 \n",
       "1                                 "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(model.device)\n",
    "test_data.iloc[:2]\n",
    "# !nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define functions to simulate new peer votes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_new_vote_pred(peer_votes_num_acc_df, target_vote, pred_target_vote_score, peer_vote_num, peer_votes_data):\n",
    "    peer_votes_num_acc_df.at[peer_vote_num, \"Total\"] += 1\n",
    "    pred_target_vote = int(pred_target_vote_score >= 0.5)\n",
    "    peer_votes_num_acc_df.at[peer_vote_num, \"Acc\"] += int(pred_target_vote == target_vote)\n",
    "    peer_votes_num_acc_df.at[peer_vote_num, \"Confidence\"] += abs(1- target_vote - pred_target_vote_score)\n",
    "        \n",
    "    if peer_vote_num > 0:\n",
    "        majority_vote = -1\n",
    "        if sum(peer_votes_data[\"VOTE\"][:peer_vote_num])/peer_vote_num > 0.5:\n",
    "            majority_vote = 1\n",
    "        elif sum(peer_votes_data[\"VOTE\"][:peer_vote_num])/peer_vote_num < 0.5:\n",
    "            majority_vote = 0\n",
    "        if pred_target_vote == majority_vote:\n",
    "            peer_votes_num_acc_df.at[peer_vote_num, \"Same as majority\"] += 1\n",
    "\n",
    "def pred_acc_peer_feature(peer_votes_num_acc_df, target_vote, target_vote_data, peer_votes_data):\n",
    "    all_peer_users = peer_votes_data[\"USERNAME\"].to_numpy()\n",
    "    all_peer_votes = peer_votes_data[\"VOTE\"].to_numpy()\n",
    "    upvoted_users, downvoted_users = [], []\n",
    "    for new_user_i in range(-1, len(all_peer_users)):\n",
    "        peer_vote_num = new_user_i + 1\n",
    "        if new_user_i >= 0:\n",
    "            new_user = all_peer_users[new_user_i]\n",
    "            new_vote = all_peer_votes[new_user_i]\n",
    "            if new_vote == 1:\n",
    "                upvoted_users.append(int(new_user))\n",
    "            elif new_vote == 0:\n",
    "                downvoted_users.append(int(new_user))\n",
    "        target_vote_data.at[0, \"UPVOTED_USERS\"] = upvoted_users\n",
    "        target_vote_data.at[0, \"DOWNVOTED_USERS\"] = downvoted_users\n",
    "        pred_target_vote_score = evaluate_model(config, model, data = target_vote_data, weights = None, batch_size=1, sample_voted_users=False, extra_input = extra_input, ret=\"prediction\", disable_tqdm=True)[0,0]\n",
    "        count_new_vote_pred(peer_votes_num_acc_df, target_vote, pred_target_vote_score, peer_vote_num, peer_votes_data)\n",
    "\n",
    "def pred_acc_peer_finetune(peer_votes_num_acc_df, target_vote, target_vote_data, peer_votes_data, n_step_per_sample=1):\n",
    "    tmp_model = copy.deepcopy(model)\n",
    "    # 0 peer votes\n",
    "    pred_target_vote_score = evaluate_model(config, tmp_model, data = target_vote_data, weights = None, batch_size=1, sample_voted_users=False, extra_input = extra_input, ret=\"prediction\", disable_tqdm=True)[0,0]\n",
    "    count_new_vote_pred(peer_votes_num_acc_df, target_vote, pred_target_vote_score, 0, peer_votes_data)\n",
    "\n",
    "    if len(peer_votes_data) > 0:\n",
    "        for new_user_i, tmp_model in enumerate(train_model(config, tmp_model, data=peer_votes_data, weights = None, batch_size=1, epochs=1, verbose=2, validation_split=False, step_generator=True, n_step_per_sample=n_step_per_sample, extra_input = extra_input)):\n",
    "            if tmp_model is None:\n",
    "                print(\"tmp_model is\", tmp_model)\n",
    "                break\n",
    "            peer_vote_num = new_user_i + 1\n",
    "            pred_target_vote_score = evaluate_model(config, tmp_model, data = target_vote_data, weights = None, batch_size=1, sample_voted_users=False, extra_input = extra_input, ret=\"prediction\", disable_tqdm=True)[0,0]\n",
    "            count_new_vote_pred(peer_votes_num_acc_df, target_vote, pred_target_vote_score, peer_vote_num, peer_votes_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_step_per_sample = 1\n",
    "if not use_voted_users_feature:\n",
    "    model.optim = torch.optim.Adam(model.parameters(), lr = 1.2 * config[\"learning_rate\"]) # 2 还不错，5不行\n",
    "    n_step_per_sample = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "if peer_vote_option == \"similar_user_opposite_vote\" or peer_vote_option == \"similar_user_same_vote\":\n",
    "    existing_votes, existing_user_votes, existing_user_updown_votes, existing_submission_votes, existing_user_subreddits = record_existing_votes(train_data)\n",
    "\n",
    "    all_users = list(range(max(max(train_data[\"USERNAME\"]), max(test_data[\"USERNAME\"])) + 1))\n",
    "    all_username_tokens = [f\"USERNAME_{user_i}\" for user_i in all_users]\n",
    "    all_username_token_ids = torch.tensor(model.tokenizer.convert_tokens_to_ids(all_username_tokens))\n",
    "    all_username_token_ids = all_username_token_ids.to(model.device); model = model.to(model.device)\n",
    "    with torch.no_grad():\n",
    "        user_embedding = model.lm_encoder.embeddings.word_embeddings(all_username_token_ids)\n",
    "    # debug(all_username_tokens=all_username_tokens, all_username_token_ids=all_username_token_ids, user_embedding=user_embedding)\n",
    "    all_users_reps, all_user_i_user_map = get_user_reps(all_users, all_user_embedding=user_embedding, train_data=train_data, selected_submissions = {key: None for key in set(train_data[\"SUBMISSION_ID\"])}, user_grouping_method = \"votes\", do_PCA = False)\n",
    "\n",
    "    user_similarity = sklearn.metrics.pairwise.cosine_similarity(all_users_reps)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simulate new votes coming and calculate the accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "test_submissions = list(set(test_data[\"SUBMISSION_ID\"]))\n",
    "max_voted_users = 1000\n",
    "xlim = 50\n",
    "peer_votes_num_acc_df = pd.DataFrame(np.zeros((max_voted_users + 1,4)), columns=['Acc', 'Confidence', 'Total', \"Same as majority\"])\n",
    "model = model.to(model.device)\n",
    "assert len(peer_votes_num_acc_df) == 1001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Acc</th>\n",
       "      <th>Confidence</th>\n",
       "      <th>Total</th>\n",
       "      <th>Same as majority</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1001 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Acc  Confidence  Total  Same as majority\n",
       "0     0.0         0.0    0.0               0.0\n",
       "1     0.0         0.0    0.0               0.0\n",
       "2     0.0         0.0    0.0               0.0\n",
       "3     0.0         0.0    0.0               0.0\n",
       "4     0.0         0.0    0.0               0.0\n",
       "...   ...         ...    ...               ...\n",
       "996   0.0         0.0    0.0               0.0\n",
       "997   0.0         0.0    0.0               0.0\n",
       "998   0.0         0.0    0.0               0.0\n",
       "999   0.0         0.0    0.0               0.0\n",
       "1000  0.0         0.0    0.0               0.0\n",
       "\n",
       "[1001 rows x 4 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "peer_votes_num_acc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▏         | 1/76 [00:00<00:28,  2.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 2/76 [00:01<01:06,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 3/76 [00:01<00:46,  1.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 4/76 [00:03<00:57,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 5/76 [00:03<01:00,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 6/76 [00:04<01:01,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 7/76 [00:06<01:13,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 8/76 [00:06<00:57,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 9/76 [00:07<01:03,  1.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 10/76 [00:08<00:51,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 11/76 [00:09<00:53,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 12/76 [00:09<00:43,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 13/76 [00:09<00:37,  1.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 14/76 [00:10<00:44,  1.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|█▉        | 15/76 [00:11<00:38,  1.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 16/76 [00:11<00:32,  1.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 17/76 [00:12<00:30,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▎       | 18/76 [00:12<00:28,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 19/76 [00:12<00:25,  2.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▋       | 20/76 [00:13<00:31,  1.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 21/76 [00:15<00:41,  1.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 22/76 [00:16<00:48,  1.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 23/76 [00:16<00:38,  1.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 24/76 [00:17<00:43,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 25/76 [00:18<00:47,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 26/76 [00:19<00:37,  1.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 27/76 [00:19<00:32,  1.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 28/76 [00:20<00:28,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 29/76 [00:20<00:31,  1.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 30/76 [00:21<00:26,  1.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 31/76 [00:22<00:30,  1.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 32/76 [00:22<00:26,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 33/76 [00:22<00:22,  1.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▍     | 34/76 [00:23<00:20,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 35/76 [00:23<00:19,  2.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 36/76 [00:24<00:18,  2.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▊     | 37/76 [00:25<00:23,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 38/76 [00:25<00:20,  1.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████▏    | 39/76 [00:25<00:17,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 40/76 [00:26<00:21,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 41/76 [00:27<00:25,  1.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 42/76 [00:28<00:28,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 43/76 [00:29<00:23,  1.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 44/76 [00:30<00:25,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 45/76 [00:31<00:26,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 46/76 [00:32<00:28,  1.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 47/76 [00:33<00:27,  1.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 48/76 [00:34<00:28,  1.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 49/76 [00:34<00:21,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 50/76 [00:35<00:21,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 51/76 [00:36<00:16,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 52/76 [00:36<00:13,  1.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|██████▉   | 53/76 [00:37<00:17,  1.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 54/76 [00:38<00:18,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 55/76 [00:39<00:14,  1.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▎  | 56/76 [00:39<00:11,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 57/76 [00:40<00:13,  1.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▋  | 58/76 [00:40<00:11,  1.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 59/76 [00:41<00:09,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 60/76 [00:42<00:11,  1.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 61/76 [00:42<00:09,  1.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 62/76 [00:43<00:07,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 63/76 [00:43<00:06,  2.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 64/76 [00:43<00:05,  2.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 65/76 [00:44<00:07,  1.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 66/76 [00:45<00:06,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 67/76 [00:45<00:05,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 68/76 [00:46<00:03,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 69/76 [00:46<00:03,  2.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 70/76 [00:47<00:03,  1.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 71/76 [00:48<00:03,  1.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 72/76 [00:49<00:03,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 73/76 [00:50<00:02,  1.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 74/76 [00:52<00:02,  1.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▊| 75/76 [00:52<00:00,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 76/76 [00:52<00:00,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tmp_model is None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# test_data[\"UPVOTED_USERS\"] = test_data[\"UPVOTED_USERS\"].astype('object')\n",
    "# test_data[\"DOWNVOTED_USERS\"] = test_data[\"DOWNVOTED_USERS\"].astype('object')\n",
    "\n",
    "for submission_id in tqdm(test_submissions[0:2000]):\n",
    "    submission_votes_data = test_data[test_data[\"SUBMISSION_ID\"] == submission_id]\n",
    "    submission_votes_data = submission_votes_data.sample(frac=1).reset_index(drop=True) # shuffle\n",
    "    target_vote_data = submission_votes_data.iloc[-1:].reset_index(drop=True)\n",
    "    target_vote = int(target_vote_data[\"VOTE\"])\n",
    "    peer_votes_data = submission_votes_data.iloc[:-1]\n",
    "    # peer_votes_data = peer_votes_data[:xlim]\n",
    "    if peer_vote_option == \"real\":\n",
    "        pass\n",
    "    elif peer_vote_option == \"real_user_opposite_vote\":\n",
    "        peer_votes_data.loc[:,\"VOTE\"] = [1 - target_vote] * len(peer_votes_data)\n",
    "    elif peer_vote_option == \"real_user_same_vote\":\n",
    "        peer_votes_data.loc[:,\"VOTE\"] = [target_vote] * len(peer_votes_data)\n",
    "    elif peer_vote_option == \"similar_user_opposite_vote\":\n",
    "        target_user = int(target_vote_data[\"USERNAME\"])\n",
    "        similar_users = user_similarity[target_user].argsort()[::-1][:xlim]\n",
    "        peer_votes_data = pd.concat([target_vote_data] * xlim, axis = 0)\n",
    "        peer_votes_data[\"USERNAME\"] = similar_users\n",
    "        peer_votes_data[\"VOTE\"] = [1 - target_vote] * xlim\n",
    "    elif peer_vote_option == \"similar_user_same_vote\":\n",
    "        target_user = int(target_vote_data[\"USERNAME\"])\n",
    "        similar_users = user_similarity[target_user].argsort()[::-1][:xlim]\n",
    "        peer_votes_data = pd.concat([target_vote_data] * xlim, axis = 0)\n",
    "        peer_votes_data[\"USERNAME\"] = similar_users\n",
    "        peer_votes_data[\"VOTE\"] = [target_vote] * xlim\n",
    "    \n",
    "    if use_voted_users_feature:\n",
    "        pred_acc_peer_feature(peer_votes_num_acc_df, target_vote, target_vote_data, peer_votes_data)\n",
    "        # pred_acc_peer_feature_finetune(peer_votes_num_acc_df, target_vote, target_vote_data, peer_votes_data, n_step_per_sample=n_step_per_sample)\n",
    "    else:\n",
    "        pred_acc_peer_finetune(peer_votes_num_acc_df, target_vote, target_vote_data, peer_votes_data, n_step_per_sample=n_step_per_sample)\n",
    "    # print(peer_votes_num_acc_df)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/TableSense/anaconda3/envs/cr4/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/home/TableSense/anaconda3/envs/cr4/lib/python3.6/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/home/TableSense/anaconda3/envs/cr4/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n",
      "/home/TableSense/anaconda3/envs/cr4/lib/python3.6/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Acc</th>\n",
       "      <th>Confidence</th>\n",
       "      <th>Total</th>\n",
       "      <th>Same as majority</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Same as majority rate</th>\n",
       "      <th>Total scaled</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>54.0</td>\n",
       "      <td>0.704521</td>\n",
       "      <td>76.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.710526</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>55.0</td>\n",
       "      <td>0.705271</td>\n",
       "      <td>76.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.723684</td>\n",
       "      <td>0.539474</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>57.0</td>\n",
       "      <td>0.714322</td>\n",
       "      <td>76.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.434211</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55.0</td>\n",
       "      <td>0.732842</td>\n",
       "      <td>76.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.723684</td>\n",
       "      <td>0.710526</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60.0</td>\n",
       "      <td>0.745499</td>\n",
       "      <td>76.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>0.789474</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>57.0</td>\n",
       "      <td>0.752187</td>\n",
       "      <td>76.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.855263</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>40.0</td>\n",
       "      <td>0.730781</td>\n",
       "      <td>55.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.854545</td>\n",
       "      <td>0.723684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>41.0</td>\n",
       "      <td>0.734224</td>\n",
       "      <td>54.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.759259</td>\n",
       "      <td>0.907407</td>\n",
       "      <td>0.710526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>29.0</td>\n",
       "      <td>0.811898</td>\n",
       "      <td>35.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.828571</td>\n",
       "      <td>0.914286</td>\n",
       "      <td>0.460526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28.0</td>\n",
       "      <td>0.824407</td>\n",
       "      <td>35.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.942857</td>\n",
       "      <td>0.460526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>28.0</td>\n",
       "      <td>0.831898</td>\n",
       "      <td>35.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.942857</td>\n",
       "      <td>0.460526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>29.0</td>\n",
       "      <td>0.840483</td>\n",
       "      <td>35.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.828571</td>\n",
       "      <td>0.971429</td>\n",
       "      <td>0.460526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>31.0</td>\n",
       "      <td>0.848285</td>\n",
       "      <td>35.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.885714</td>\n",
       "      <td>0.971429</td>\n",
       "      <td>0.460526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>31.0</td>\n",
       "      <td>0.853363</td>\n",
       "      <td>35.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.885714</td>\n",
       "      <td>0.971429</td>\n",
       "      <td>0.460526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>31.0</td>\n",
       "      <td>0.859117</td>\n",
       "      <td>35.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.885714</td>\n",
       "      <td>0.971429</td>\n",
       "      <td>0.460526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>31.0</td>\n",
       "      <td>0.863464</td>\n",
       "      <td>35.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.885714</td>\n",
       "      <td>0.971429</td>\n",
       "      <td>0.460526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>18.0</td>\n",
       "      <td>0.834637</td>\n",
       "      <td>22.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.289474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>15.0</td>\n",
       "      <td>0.845748</td>\n",
       "      <td>18.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.236842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>11.0</td>\n",
       "      <td>0.876182</td>\n",
       "      <td>13.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.171053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>11.0</td>\n",
       "      <td>0.876035</td>\n",
       "      <td>13.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.846154</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.171053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.912417</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.078947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.917482</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.078947</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Acc  Confidence  Total  Same as majority  Accuracy  \\\n",
       "0   54.0    0.704521   76.0               0.0  0.710526   \n",
       "1   55.0    0.705271   76.0              41.0  0.723684   \n",
       "2   57.0    0.714322   76.0              33.0  0.750000   \n",
       "3   55.0    0.732842   76.0              54.0  0.723684   \n",
       "4   60.0    0.745499   76.0              57.0  0.789474   \n",
       "5   57.0    0.752187   76.0              65.0  0.750000   \n",
       "6   40.0    0.730781   55.0              47.0  0.727273   \n",
       "7   41.0    0.734224   54.0              49.0  0.759259   \n",
       "8   29.0    0.811898   35.0              32.0  0.828571   \n",
       "9   28.0    0.824407   35.0              33.0  0.800000   \n",
       "10  28.0    0.831898   35.0              33.0  0.800000   \n",
       "11  29.0    0.840483   35.0              34.0  0.828571   \n",
       "12  31.0    0.848285   35.0              34.0  0.885714   \n",
       "13  31.0    0.853363   35.0              34.0  0.885714   \n",
       "14  31.0    0.859117   35.0              34.0  0.885714   \n",
       "15  31.0    0.863464   35.0              34.0  0.885714   \n",
       "16  18.0    0.834637   22.0              22.0  0.818182   \n",
       "17  15.0    0.845748   18.0              18.0  0.833333   \n",
       "18  11.0    0.876182   13.0              13.0  0.846154   \n",
       "19  11.0    0.876035   13.0              13.0  0.846154   \n",
       "20   6.0    0.912417    6.0               5.0  1.000000   \n",
       "21   6.0    0.917482    6.0               5.0  1.000000   \n",
       "\n",
       "    Same as majority rate  Total scaled  \n",
       "0                0.000000      1.000000  \n",
       "1                0.539474      1.000000  \n",
       "2                0.434211      1.000000  \n",
       "3                0.710526      1.000000  \n",
       "4                0.750000      1.000000  \n",
       "5                0.855263      1.000000  \n",
       "6                0.854545      0.723684  \n",
       "7                0.907407      0.710526  \n",
       "8                0.914286      0.460526  \n",
       "9                0.942857      0.460526  \n",
       "10               0.942857      0.460526  \n",
       "11               0.971429      0.460526  \n",
       "12               0.971429      0.460526  \n",
       "13               0.971429      0.460526  \n",
       "14               0.971429      0.460526  \n",
       "15               0.971429      0.460526  \n",
       "16               1.000000      0.289474  \n",
       "17               1.000000      0.236842  \n",
       "18               1.000000      0.171053  \n",
       "19               1.000000      0.171053  \n",
       "20               0.833333      0.078947  \n",
       "21               0.833333      0.078947  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "peer_votes_num_acc_df = peer_votes_num_acc_df[peer_votes_num_acc_df[\"Total\"] > 0]\n",
    "peer_votes_num_acc_df[\"Accuracy\"] = peer_votes_num_acc_df[\"Acc\"]/peer_votes_num_acc_df[\"Total\"]\n",
    "peer_votes_num_acc_df[\"Confidence\"] = peer_votes_num_acc_df[\"Confidence\"]/peer_votes_num_acc_df[\"Total\"]\n",
    "peer_votes_num_acc_df[\"Same as majority rate\"] = peer_votes_num_acc_df[\"Same as majority\"]/peer_votes_num_acc_df[\"Total\"]\n",
    "peer_votes_num_acc_df[\"Total scaled\"] = peer_votes_num_acc_df[\"Total\"]/max(peer_votes_num_acc_df[\"Total\"])\n",
    "peer_votes_num_acc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:06:29 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m at \u001b[0m\u001b[1;32m<ipython-input-13-754ebc79e6f7>:6 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;36mConfidence and accuracy\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:06:29 \u001b[0m\u001b[1;33m------------------\u001b[0m\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEOCAYAAABhOhcDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAzrklEQVR4nO3deVxU9f7H8dfMwICAsikIoqKohLnjkgupuGCGQtfMLmllhq36y1v3pi0uuaQtVpplWto11AqXSLSyMjXNXMANcRcVFEFEBEZkhpnz+4Mc5aKCOAR4Ps/Ho0cw853v+czX4T1nzpzz/WoURVEQQgihKtqqLkAIIcTfT8JfCCFUSMJfCCFUSMJfCCFUSMJfCCFUSMJfCCFUqMzwnzVrFqGhoQQGBnLkyJEbtjGbzUyZMoW+ffvSr18/YmNjbV6oEEII27Erq0GfPn14/PHHeeyxx27aZs2aNZw+fZr169eTk5NDZGQkXbt2xc/Pr1xFWCwWDAYD9vb2aDSa8lcvhBAqpigKJpMJZ2dntNrbO5BTZvh37NixzE7WrVvH0KFD0Wq1eHh40LdvX3788UeefvrpchVhMBhu+qlCCCHErbVo0YLatWvf1mPKDP/ySE9Px9fX1/q7j48P586dK/fj7e3tgeInoNfrbVFSjZaUlESrVq2quowq9+vO02zadQJnF5dS99lpNfyzfyC+9Urfd7eS18U1MhbFjEYjR44csWbo7bBJ+N+pq4d6ZO//mqSkpKouocotW5+OxaLgacwrdZ+dFg4fOUp2xu2/6GsyeV1cI2NxTUUOl9sk/H18fDh79ixt2rQBSn8SKK9WrVrh4OBgi5JqtISEBIKDg6u6jCp1Kb+QS4Y0+rd3ZczwXlVdTrUgr4trZCyKFRYWVvhN0Caneg4YMIDY2FgsFgvZ2dn88ssvhIWF2aJroVJHU3MA8PWQw4BCVIYy9/ynTZvG+vXrycrKYuTIkbi5ubF27Vqio6MZO3YsrVu3JiIigr1799K/f38AXnjhBRo2bFjpxYu717G0HDQa8PGomYd1LBYLaWlpGAwGm/VpZ2fHwYMHbdZfTaa2sXB2dsbPz++2z+i5FU11mNL56kcXOexTTD7SwrRF2zmblc+oPm41ciwyMzMpLCykQYMGNvuDNRgMODs726Svmk5NY2GxWDhz5gwODg54eXmVuO9OslOu8BXV0tHUHJr5uVV1GRWWk5ODt7e3TffUhDpptVq8vb25dOmSbfu1aW9C2MCFSwVk516hWUO3qi6lwsxmc4VOvxPiRuzt7SkqKrJpnxL+oto59teXvc393Ku2kDskV6sLW6mM15KEv6h2jqbloNVqaNKgTlWXcle5dOkSbdq0Ydq0aVVdiqgGJPxFtXM0NYdG3rVx1FeLaxDvGvHx8bRt25a1a9diNBordVu2PkQhbE/+ukS1oigKx1Jz6HJv/aou5a6zcuVK/v3vf/PZZ5/x66+/8sADD5CRkcG0adM4efIkAOHh4TzzzDPk5eUxY8YMkpKS0Gg0dOzYkYkTJzJ+/HhatWrF8OHDAUr8Pn78eHQ6HSkpKRgMBuLi4nj55ZdJSUnBZDLRqFEjZsyYgaurKwArVqxgyZIlQPEx7c8++4x58+bRoEED67xgycnJjBs3jh9//FEOo9mYhL+oVs5fLCDXYKzRX/b+rw27TvPzjtN33I/ZbEan05W4rV/nRoR2bFTmYw8dOkROTg733Xcf58+fZ+XKlTzwwAO88sor9OzZk7lz5wKQnZ0NwIwZM3ByciIuLg6tVmu9vSwHDx4kJiYGJycnAF5//XU8PDwA+OCDD1i4cCGvvPIK27dv57PPPmPZsmXUq1cPg8GAnZ0dw4cP59lnn2XUqFFoNBpiYmKIioqS4K8EEv6iWjmalgNA87so/KuDFStWEBERgUajoX///kybNo0zZ86we/duFi9ebG13Nah/++03Vq1aZT1V9ertZRkwYIA1+AHi4uJYs2YNJpOJy5cv4+/vD8DGjRuJiIigXr16ANZz9gMCAmjYsCGbN2+mXbt2bNiwgQkTJtzx8xelSfiLauXo6YvY6TT4+9w9X/aGdizf3nlZKnphk9FoJD4+Hr1eT1xcHAAmk4nVq1ffdl86nQ6LxWL9vbCwsMT91wf/rl27WL58OV9//TUeHh6sWbOGb7/9tsxtjBgxguXLl3P8+HH69+9/21MVi/KRL3xFtXIsLQd/nzrY2+nKbizK5ddff6VJkyZs3ryZDRs2sGHDBhYtWsT3339P+/bt+fLLL61trx7e6d27N1988QVXJwC4envjxo3Zv38/UHwV8/bt22+63dzcXFxcXHBzc8NoNLJy5Urrfb169SIuLo6srCyg+I3t6htJz549SUlJYfHixURFRdluIEQJEv6i2rj6ZW+zhjX7/P7qZuXKlQwaNKjEbe3bt8disTBmzBgSExMJDw9n8ODBrFixAoAJEyZgMBist3/yyScADB06lHPnzjFw4EAmT55sncn3RkJCQmjUqBFhYWEMHz6cli1bWu/r0qULo0ePZuTIkQwePJgnnniCvLziqbu1Wi2RkZH4+flxzz332Ho4xF/ksI+oNtIvGDBcKSrzeL9iMWM25GJXW94kyuPzzz+/4e2//PILAJ07dy51X506dZg5c2ap293d3fnqq69u2N//tre3t+fDDz+8aV1Dhw5l6NChN7xvx44dPP744zd9rLhzEv6i2jh6Oge49mWvYi5Cl5tB/oEtGLPSMF1IK/5/djqYi/B++FWcA0sHl6i59u/fz7hx42jZsqVMC1/JJPxFtXEsLQe9nZaG3rWxFBZwNmYSdc4dJxNAo8XOzQt9XT+cmgWjr9cQp2YdqrpkYWOtW7e2fiIRlUvCX1QbR1NzaNLAFZ1G4dzq9zFmpHC5ZRjNu/XH3tMXrZ0s7CKErcgXvqJaMFsUjqfl0NzPlQs/fUHB8d3UHRBNYaNgHLz9JfiFsDEJf1EtnMnM44rRTLBlD7mJP+HaNZI6HfpXdVlC3LXksI+oFo6l5dDW/hQehzfjHNQVj96PVXVJQtzVJPzvAkVmC3/sO0tBoe1mUtTb67i/XQN0ur/nw2HG4SRGuGzBoUFz6g0ag0YjH0pt7dKlS4SEhPDII4/wxhtvVHU5VW7ZsmUsWbIER0dH5s2bx/jx4296GmtgYCCJiYl31dKREv53ga/XH+abX47YvN+CwiIGdmti837/lykng9anllGgc6HZ0PFo7WUd58pw/ZTO//nPf9DrbfM9SlFREXZ2NS9KvvrqK9555x3rhWo3C/67Vc37FxMlXLhUwOpNx+nexpfoyFY263fGlztYvfEYYV0a3/Hev6IoFBxPxFyQBxotGjSgAdCARkP25m/AYiY58HHaObvapH5R2o2mdH799ddp0aIFTzzxBABHjhzhueee45dffsFgMPD2229z+PBhCgsL6dKlCxMmTECn0zFixAjuuece9u7di6urK59++inPPPMMFy9epLCwkDZt2jBlyhT0ej1Go5GpU6eyY8cOPDw8CAoKIisrizlz5gCwYMEC1q9fj9lsxtvbm6lTp1onfLve7t27eeeddzAYDFgsFsaPH0+PHj3Yt28f06dP5/Llyzg5OfH666/Tpk0b0tLSGDJkCI8++iibNm2ioKCA6dOn07FjR1566SVSU1P5z3/+w7333su4ceMYMmSIdbqK9evXM3v2bBwcHOjfv+R3T3v37uW9997DYDAAMHbsWHr16nXL7UHxZHlz586lqKgIrVbLzJkzrWN4o/4qm4R/Dbf0x0NYLBaeDG+Jp2stm/U7pHdz3v7vTv7Yl05I+wYV7kdRLGT9uJC8xPU3b6Sz4/O8UB4KaFbh7VRnefs2krd3wx33YzabufQ/UzrXbhtK7Ta9ynzszaZ0fuihh5g+fbo1/FetWsVDDz2ERqPh7bffplOnTkyfPh2LxcIrr7zCypUreeSRRwBITU1l2bJl2NnZoSgK7733Hu7u7iiKwquvvsrKlSv55z//yTfffMPZs2dZu3YtZrOZESNGUL9+8XoNcXFxpKam8u2336LValm2bBkzZ87k/fffL1F/Tk4OL774InPnzqVDhw7k5uaiKApGo5GxY8fy9ttv07VrV/744w/Gjh3L+vXrrY9r164d48aN4/vvv+e9997j66+/5sMPPyQ0NJQ5c+bQokUL0tLSrNvKysrizTffZPny5TRt2pSFCxda78vNzWXSpEksWLAALy8vMjMzefjhh4mPj7/l9lJSUnjjjTdYunQp/v7+GI1GjEbjLfurU6dyJzeU8K/BTqbn8uvO0wwKCaC+p22PRXZp5UODes6s+O0oPdr5Vmg+dcVi5vza+eTv21B89k67voBC8VxhCijF/20+lMvx747JNM6V6EZTOmdkZNCxY0cMBgOHDx8mICCA+Ph4vvnmGwA2bNjAvn37rFM+X7lyBW9vb2ufgwYNsh7usVgsLFq0iM2bN2OxWLh06RKOjo4AbN++nYiICOzs7LCzs+PBBx8kISHBuo2kpCQeeughoPgNzsXFpVT9e/bsISAggA4dii/s0+l0ODs7c/jwYezt7enatSsA3bp1w97enpSUFJydnXFycqJ3794AtGvXjlmzZpU5Vnv37qVly5Y0bdoUgGHDhvHee+8BxZ8+0tLSiI6OtrbXaDScOnUKd3f3m27vjz/+4P7777dOaa3X69Hr9WzatOmm/bVu3brMWu+EhH8N9mX8AWo52jOsX4s76kdRLJjzLmK6eA7TxXMUXTyHpfAyQ7t05MP40+w5cp72gV6316fFzPnv55J/4HfcQ4bhFjL0pm8ghzftxdnRDp+6d8+Xader3aZXufbOy2LrKZ1XrVrFc889R2RkJKtXr6Zz584EBATQoEHxJz1FUfjkk09o2LDhDfu9fvrmNWvWkJCQwNKlS3FxcWH+/PnW1cFuRVEUnnvuOR5++OHbfl7lcf33Glqt9o6Xl1QUhcDAQJYuXVrqvrS0tNve3q36q2xySkUNtffIeRIOZfJInxbUdirfF3eKYsF08RyGQ9vJ3vwN51a8Q+qClzj5zmOcnjua9JiJZK39hJxt35G75xea7ZtHYB0DK387elu1KWYTmatnk3/gdzx6P4b7/Y/c8pPDsdSLNGvoJqs1VZKbTel8dT7/yMhI4uPjiY2N5R//+If1caGhoSxYsACz2QwUT+ucmpp6w23k5eXh7u6Oi4sLeXl51sMgUDxx3Jo1aygqKqKwsJAffvihxDaWLVvGpUuXgOI3qkOHDpXqv127dhw/fpzdu3cDfx0Cu3SJJk2aYDKZ+PPPPwHYtm0bRUVFNGlS8RMV2rVrR3JysvXNKzY21npf+/btOXXqlHV7APv27bNOfX0z3bt3Z/PmzdY+jUYj+fn5Fe7PFmTPvwayWBQWxR/Ay70W4T1Kv8gVRcFsuITpQhqmrDSMmacpzDyFMfMUirHgr1Ya7D18sK/bAKem7bB3r4+de/3i/7vWw5SVRvryqYx2XMfclJ4cS723XEsrWoqMZK56n8tHd+HZbySuncNv2d5UZOZkei4R9wdUYCREedxqSucdO3bQuXNnmjVrxo4dO5g9e7a1zWuvvca7775rPVxkb2/Pa6+9dsNPApGRkfz6668MGDAAT09PgoODrfPzP/rooxw6dIgHH3wQd3d36+GUq4/LycmxrgmsKAr//Oc/S03l7Obmxty5c5k5cyaXL18Giqed7tatG3PmzCnxhe9HH310R2cyeXp6MnXqVJ599lkcHR1LfOHr6urKJ598wrvvvsuMGTMwmUw0bNiQ+fPn37JPf39/pk6dyrhx46zLcc6cOZPAwMCb9lfZO0Ma5e94iylDYWEhSUlJtGrVCgcHOc0vISGB4ODgm97/W0Iqs5cl8nJUB3q296Uw/QSFZw5jPJ9qnf3SUpBvba9xcMLBqzF6r8bovf3Re/mj92pU5imVppxMzi6dQsHF8/xZ7x88+cyjt2xvMRWSsWIWBSf2UndANHWCB5T5XI+mXuRfH25m/OOd6N7Wt9T9ZY1FdXXw4EGCgoJs2mdFD/tUB/n5+bi4uGA0GnnuuecYMGDATadzLo+aPBYVdaPX1J1kp+z51zCFVwrZ8MNGHvHOIPDIPk5uOIRivAKAtlZt9HX9cL6nK/q6ftjX9UNf1w9dbc8K7UXYu3nh9+QMDnz2Ot3PryB1mwcNu5aecsFsuER+8lby9vyMMTOVug8+T512fQA4dS6XH/84yeY9Z7hiNJd67NUlAe+mBdtFaSNHjsRoNFJYWEi3bt2sX/CKqiPhX80pioLp/Gkun9hLQcpe8k8mM1JjBBMU5fpRu1VPHBvfi2PDlpWyuInO2RW/x98iYd4bBGz4jEt2Jlw7PYilsADDke3kJ/1OQco+UCzovfzxHvIK+mad2JiYxo/bTnLgxAXsdFq6tvahntuNT0X1cq+Ft4fTDe8Td4frj5uL6kHCvxrSFBrIT/qdyyl7KDixF3P+RQB07r7sKAzAWLc5j4+MRPc3XRDlUdeDo/c8xeWDS2m9fhGGQ9spPHsUpciInasXbl0jcWkVQrbWk9htJ/l52XpyDUZ8PJ0ZGd6SPp0a4eoih/OEqE4k/CvRxdwruLo4oNXe+pBLUe4FrqQe/Ou/ZNwyT5MJaGu5UKtJW2o1aYtT07Ys/i2d+BMnmPNsb3TOlXsByP+KDL2H53fez7+bHcbvwlFqtw3FpVUIDg0CSc3IY/HPR9mcuBs0GrrcW58HuvrTtnm9Mp/73UxRFDmDSdhEZXw1K+FfSU6dy+X/3t9IxyBv/j2iIw72167MNF/OxXBkB1dOF4d9UU4mABp7Rxz9Ailo3oRmPQair98Ejbb4cRt2nSZ+Swp9Ozemcf2/N/gBfOu5cF9rP+Ye0bPozZdxcrTneFoO3y7Zybb96ejtdQy+P4CI+wOoe5PDO2qi0+kwmUw2mz9HqJvJZLL5/EkS/pXku43H0WhgR/I5Ji3YxusjO2OXeZjc3T9jOLwdzEVonerg2DCIOh0HUqthkDXszyYk4OB7baqDVb8dY3H8Ado2r8uowfdW2XMaEtqMrfvOsmjNAS5cusKugxk4OdrxSJ8WDAppKod2ruPm5kZGRgYNGjRAq5XLaUTFWSwWMjIycHW17WFeCf9KkJ17hY2JqYTd509rXz274ldx5MNFuCuX0Dq6UKdDGLXbhqL3anzLwwIWi8KXa5NZvfEYIe0aMO6f7bG30920fWVr3tCdts3r8tOfp6jtpGfEA0E82L0JzrXsq6ym6qpu3bqkpaVx+PBhm/VpNBrlk8Rf1DYWzs7O1K1b16Z9liv8U1JSGD9+PDk5Obi5uTFr1izrHBVXXbhwgQkTJpCenk5RURFdunThjTfeqJFTvd6p+C0nwGLmAfMGzL9tY1CtIlKKvNmsC2Zo1D+p6+NRZh9FZgtzv93Dhl2phHdvQnRk62px/PzFoe3YfyyLkHYNcHRQ379teWm1Who1amTTPhMSEmjbtq1N+6ypZCzuXLk+j06aNImoqCh++uknoqKimDhxYqk28+fPJyAggDVr1vD9999z4MAB68x6alJQWMS6P04yJCAf85Hfqd22N37PfESjkdNJMDZl/PztHEvNuWUfxiIL0xfvYMOuVB4bcA+jH6oewQ9Q39OZfl0aS/ALUcOVGf4XLlwgOTmZ8PDiy/TDw8NJTk4mOzu7RDuNRmOdZ9toNGIymUrMAKgWP+84haHAxH36Q9i51qPugGj0df1o3tCdWS+G4GCv47VPt7A9KZ3M7Mul/kvLzGPJhiwSD2XwwsNtebRfoJwxIoSwuTJ339LT0/H29kb31zziOp0OLy8v0tPT8fC4dvji+eefZ8yYMfTo0YOCggIee+yxGnlZ/p0wmy3EbT5B10YatOcOUbtXlPVsHYAG9Vx4Z0wIkxf+ybTFO27aj04Lrz7eiW5tSk93IIQQtmCzz+4//vgjgYGB/Pe//8VgMBAdHc2PP/7IgAFlz+9yVVJSkq3KqRJJp4r33kd6HkAxaEnR1OXEX/OWX+/RHi4cTtPx18wGpfh66nEwpZOQkF7JFdcMCTcYQ7WSsbhGxuLOlBn+Pj4+ZGRkWGeiM5vNZGZm4uPjU6JdTEwMM2bMQKvVUrt2bUJDQ9m+fftthX9lT+ymKAoffr0bf586PNTLtqtGKYrC0t8309DTAT/DYWoFdiGgW8+btu9+i75q6mRmlUHG4hoZi2tkLIpdnditIso85u/p6UlQUJB1fu74+HiCgoJKHPIB8PPzY/PmzUDxaVjbtm2jefPmFSqqsuw+cp4Nu1JZtOYAW/edtWnfB05c4GhqDlFBBixX8qnTofQEaEIIUV2U62yfyZMnExMTQ1hYGDExMUyZMgWA6Oho9u/fDxTP/Z2QkMCgQYOIjIzE39/futZndRH76xE8XR0JbOTOh8sTOXUu12Z9r954nDrOevxzE7Fzr4+jv+0WUxdCCFsr1zH/gICAG87Kd/3Cxo0aNbKu9VkdJadcIOn4BaIjWtG9rS8vfbCJGYt38P5LPXG5w4uUUjPy2JF8jlEhbhgPHMKjz+NoNHJVpxCi+lJNQsX+epQ6znr6d2mMp2stxj/eiYzsy8xeloDFcmeTJsVtPo7eTktnu8Ogs6N2m942qloIISqHKsL/xJlL7DqYweD7m1ovTrq3qSfREa3YmZzB1z9X/BL8i3lX2LArlX7BPhgP/Y7LPV3ROf39E68JIcTtUEX4x/56BCdHOx7s3rTE7QO7N6FPp4YsX3+Y7UkVO61y7ZYUiswWBvpmYim8TO0O/WxRshBCVKq7PvzTMvPYuu8sD3ZvUurYvkaj4fkhbWnm58rs5YmkZebdVt/b9qezYsNRurXxRXd0M/Z1/XBs2NKW5QshRKWoseFvKDDxy47TXDEW3bLdyg3HsNdpGRwSUOo+s+ESppRE/hV0mlGOP5G5cCwZv69GUW5y9dV1diSf452vdtKsoRvP9XSl8OxR6nToL1MxCCFqhBo5O9e5Cwbe+mI7qRl5rP0jhTdGdsbTtfQCIpkXL/NbQioPdPPHrbYD5isGDAd+p+DUAQrPHqXo0vnihhotTTwacCrLEbfNMRw7uB3/oS9h717/htvfdTCDt7/cSRNfV6ZEd+Xyb4vQ2OlxaXXzi7qEEKI6qXHhf+DEBaYv3oGiKIx4IIjYX4/w8kebeeOpLjTzcyvRdvXGY4BCRJCGzDXzMCRv+Wvd2Xo4+DanTseBODZojt67CVq9I5rTF1kbs5TQzC2cnD+Oev2exDW45N584uFMZny5g8Y+tXlrdFdqaYvIPLAZ55bd0dVy+XsHQwghKqhGhf8vO04zb8UevD2ceHPUfTSo50Knlt5MXbSdVz/ewstRHayToWVnZZOX8BOTvFK4vDIDjb0jLq17Uqd9fxx8mt6w/2aN3Hn65Wf56ttWNDwei+6nBVxK3kaDyDHY1fFk79HzTF+0HT8vF6Y+0w3nWvbkJmxAMV6RK3qFEDVKjQh/s0VhydpkVm08RtvmdRn/eCdcnIpX8Wni68r7/3c/0xfv4O3/7uSJfk3obb+H7B0/8I9aRjS1G+HZaTQurULQOjiVua1aDnaMHtGLzYlNiYv7hgdO7+Tkp/9HUVB/tu86yRNuhbSvZ0fOktVk5V1AMV5B7+WPg2/1mspCCCFupdqHf0FhEe8vTWD7gXM80M2f0ZGtsdOV/J7avbYj05/tRtziGAJ2xnJJW8C+ogAu+HYn+unBFfoS9v4OjWjR+HkWfvUL3XJ+oMn+OEIdNOhquaM1e2JXrxG1AtpjV9sTl5bd5IteIUSNUm3D/4qxiE2JZ1i98RjpWfk881BrHuze5IYhW5h+gqz1n9Mx6zD5tf14/2wbTpvrMie61x2Fcn1PZyaMGcQ365uz7UQazz/WHQ835zt5WkIIUS1Uu/A/m5XPuq0n+WXnaQwFJvx96jA5uivtA71KtTVfziV743Lydv+M1qk2dR98niZte/PEwUwysy/TxPfOV7u302l57IGWgJy/L4S4e1Sr8P/g60S27stEp9XQrY0vD3ZvQssmHiX23i1XDFw+uY+CY4kYDm/HUlhAnU4Dcb9/GDrH4r3yzi1vfIqmEEKIYtUq/NMy8ojqH0hYV3886jgCxYukGDNPc/l4IpePJXIl7RBYzGgdnKgV0B737g+j92pUxZULIUTNUq3C/5X6W9GdWofhRBH55iKUIhOWIiNK4WUA9F6NcbtvMLUCOuDoF1hifVwhhBDlV63C387FDXtnF9DZodHZodHZo9HZoffyxymgPXZ1PKu6RCGEuCtUq/Cv9+DzlbqGrxBCiGI1dmI3IYQQFSfhL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKiThL4QQKlSu8E9JSWHYsGGEhYUxbNgwTp48ecN269atY9CgQYSHhzNo0CCysrJsWasQQggbKddKXpMmTSIqKoqIiAji4uKYOHEiS5YsKdFm//79fPzxx/z3v/+lXr165OXlodfrK6VoIYQQd6bMPf8LFy6QnJxMeHg4AOHh4SQnJ5OdnV2i3ZdffslTTz1FvXr1AKhdu7YsySiEENVUmeGfnp6Ot7c3Op0OAJ1Oh5eXF+np6SXaHT9+nNTUVB577DEeeughPvnkExRFqZyqhRBC3BGbLeBuNps5fPgwixcvxmg08vTTT+Pr60tkZGS5+0hKSrJVOTVeQkJCVZdQbchYXCNjcY2MxZ0pM/x9fHzIyMjAbDaj0+kwm81kZmbi4+NTop2vry8DBgxAr9ej1+vp06cP+/btu63wb9WqlRwqovhFHRwcXNVlVAsyFtfIWFwjY1GssLCwwjvNZR728fT0JCgoiPj4eADi4+MJCgrCw8OjRLvw8HC2bNmCoiiYTCb+/PNP7rnnngoVJYQQonKV61TPyZMnExMTQ1hYGDExMUyZMgWA6Oho9u/fD8CDDz6Ip6cnAwcOJDIykmbNmvHwww9XXuVCCCEqrFzH/AMCAoiNjS11+8KFC60/a7VaJkyYwIQJE2xXnRBCiEohV/gKIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKSfgLIYQKlSv8U1JSGDZsGGFhYQwbNoyTJ0/etO2JEydo27Yts2bNslWNQgghbKxc4T9p0iSioqL46aefiIqKYuLEiTdsZzabmTRpEn379rVpkUIIIWyrzPC/cOECycnJhIeHAxAeHk5ycjLZ2dml2i5YsIBevXrh7+9v80KFEELYTpnhn56ejre3NzqdDgCdToeXlxfp6ekl2h06dIgtW7bw5JNPVkqhQgghbMfOFp2YTCbefPNN3n77beubREUkJSXZopy7QkJCQlWXUG3IWFwjY3GNjMWdKTP8fXx8yMjIwGw2o9PpMJvNZGZm4uPjY21z/vx5Tp8+zejRowHIzc1FURTy8/OZOnVquYtp1aoVDg4OFXgad5eEhASCg4OruoxqQcbiGhmLa2QsihUWFlZ4p7nM8Pf09CQoKIj4+HgiIiKIj48nKCgIDw8PaxtfX1+2b99u/X3u3LlcvnyZV199tUJFCSGEqFzlOttn8uTJxMTEEBYWRkxMDFOmTAEgOjqa/fv3V2qBQgghbK9cx/wDAgKIjY0tdfvChQtv2H7MmDF3VpUQQohKJVf4CiGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECkn4CyGECtmVp1FKSgrjx48nJycHNzc3Zs2ahb+/f4k28+bNY926dWi1Wuzt7Rk3bhwhISGVUbMQQog7VK7wnzRpElFRUURERBAXF8fEiRNZsmRJiTZt2rThqaeeolatWhw6dIjhw4ezZcsWHB0dK6VwIYQQFVfmYZ8LFy6QnJxMeHg4AOHh4SQnJ5OdnV2iXUhICLVq1QIgMDAQRVHIycmxfcVCCCHuWJnhn56ejre3NzqdDgCdToeXlxfp6ek3fcx3331Ho0aNqF+/vu0qFUIIYTPlOuxzO3bs2MFHH33EokWLbvuxSUlJti6nxkpISKjqEqoNGYtrZCyukbG4M2WGv4+PDxkZGZjNZnQ6HWazmczMTHx8fEq13b17N//+97/55JNPaNq06W0X06pVKxwcHG77cXebhIQEgoODq7qMakHG4hoZi2tkLIoVFhZWeKe5zMM+np6eBAUFER8fD0B8fDxBQUF4eHiUaLdv3z7GjRvHnDlzuPfeeytUjBBCiL9Huc7znzx5MjExMYSFhRETE8OUKVMAiI6OZv/+/QBMmTKFK1euMHHiRCIiIoiIiODw4cOVV7kQQogKK9cx/4CAAGJjY0vdvnDhQuvPK1eutF1VQgghKpVc4SuEECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECpUrvBPSUlh2LBhhIWFMWzYME6ePFmqjdlsZsqUKfTt25d+/foRGxtr61qFEELYSLnCf9KkSURFRfHTTz8RFRXFxIkTS7VZs2YNp0+fZv369XzzzTfMnTuXtLQ0mxcshBDizpUZ/hcuXCA5OZnw8HAAwsPDSU5OJjs7u0S7devWMXToULRaLR4eHvTt25cff/yxcqoWQghxR8oM//T0dLy9vdHpdADodDq8vLxIT08v1c7X19f6u4+PD+fOnbNxuUIIIWzBrqoLuF5SUlJVl1BtJCQkVHUJ1YaMxTUyFtfIWNyZMsPfx8eHjIwMzGYzOp0Os9lMZmYmPj4+pdqdPXuWNm3aAKU/CdyKoigAtGjRAr1ef7vP4a6TlJREq1atqrqMakHG4hoZi2tkLIoZjUaOHDlizdDbUWb4e3p6EhQURHx8PBEREcTHxxMUFISHh0eJdgMGDCA2Npb+/fuTk5PDL7/8wtKlS8tVhMlkAuDIkSO3/QTuVvIp6BoZi2tkLK6RsbjGZDLh6Oh4W4/RKOV4yzh+/Djjx48nNzeXOnXqMGvWLJo2bUp0dDRjx46ldevWmM1m3nrrLbZu3QpAdHQ0w4YNK1cRFosFg8GAvb09Go3mtp6AEEKolaIomEwmnJ2d0Wpv77KtcoW/EEKIu4tc4SuEECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECok4S+EECpU5eFfnrUC7lazZs0iNDSUwMDAElc3q3FMLl68SHR0NGFhYQwaNIgXX3zROnPsnj17GDx4MGFhYTz11FNcuHChiqutfM8//zyDBw8mMjKSqKgoDh48CKjztQHw8ccfl/g7UeNrAiA0NJQBAwYQERFBREQEv//+O1DB8VCq2IgRI5TvvvtOURRF+e6775QRI0ZUcUV/n507dypnz55VevfurRw+fNh6uxrH5OLFi8qff/5p/X3mzJnKhAkTFLPZrPTt21fZuXOnoiiKMm/ePGX8+PFVVebfJjc31/rzzz//rERGRiqKos7XRlJSkjJq1Cjr34laXxOKopTKCkVRKjweVbrnX961Au5WHTt2LDVBnlrHxM3NjS5dulh/b9euHWfPniUpKQkHBwc6duwIwKOPPqqKdSJq165t/Tk/Px+NRqPK14bRaOStt95i8uTJ1tvU+pq4mYqOR5VO6XyrtQL+d+I4tZAxKZ7rafny5YSGhpaaHdbDwwOLxUJOTg5ubm5VV+Tf4PXXX2fr1q0oisLnn3+uytfGRx99xODBg/Hz87PepubXBMArr7yCoigEBwfzr3/9q8LjUeXH/IX4X1OnTsXJyYnhw4dXdSlVavr06WzcuJFx48bxzjvvVHU5f7vdu3eTlJREVFRUVZdSbSxdupTvv/+elStXoigKb731VoX7qtLwv36tAOCmawWoidrHZNasWZw6dYoPP/wQrVZrXSfiquzsbLRarSr28K6KjIxk+/bt1K9fX1WvjZ07d3L8+HH69OlDaGgo586dY9SoUZw6dUq1r4mr/9Z6vZ6oqCgSExMr/DdSpeF//VoBwE3XClATNY/J7NmzSUpKYt68edZFfVq1asWVK1fYtWsXAF9//TUDBgyoyjIrncFgKLFM6oYNG3B1dVXda2P06NFs2bKFDRs2sGHDBurXr88XX3zB008/rbrXBMDly5fJy8sDiqdyXrduHUFBQRX+G6nyKZ1vtlaAGkybNo3169eTlZWFu7s7bm5urF27VpVjcvToUcLDw/H397cuSuHn58e8efNITExk0qRJFBYW0qBBA959913q1q1bxRVXnqysLJ5//nkKCgrQarW4urry6quvcu+996rytXFVaGgo8+fPp0WLFqp7TQCkpqYyZswYzGYzFouFgIAA3njjDby8vCo0HlUe/kIIIf5+8oWvEEKokIS/EEKokIS/EEKokIS/EEKokIS/EEKokIS/qHbGjx/PBx98UCXbVhSFCRMm0KlTJx5++OEqqUGIv4OEvyhTaGgoXbt25fLly9bbYmNjGTFiRBVWVTkSEhLYunUrmzZtYsWKFVVdjs2MGDGC2NjYqi5DVCMS/qJcLBYLS5YsqeoybtvVqRDK68yZMzRo0AAnJ6dKqqh8ioqKqnT74u4n4S/KZdSoUSxatIjc3NxS96WlpREYGFgisK7f01y1ahWPPvooM2bMoGPHjvTp04fExERWrVpFz5496dq1K6tXry7R58WLFxk5ciTt27dn+PDhnDlzxnrf8ePHGTlyJJ07dyYsLIx169ZZ7xs/fjyTJk0iOjqadu3asX379lL1ZmRk8Oyzz9K5c2f69evHt99+CxR/mnnjjTfYs2cP7du3Z86cOaUee/W5vPXWWwQHBzNgwAC2bdtmvT8vL4/XXnuNHj16EBISwgcffFDiDWjFihU88MADdOrUiVGjRpV4XoGBgSxdupT+/fvTv3//Utt++umniYmJKXHb4MGDWb9+PQCJiYkMGTKE4OBghgwZQmJiIgAffPABu3bt4q233qJ9+/bWycBuNY6bNm1i4MCBtG/fnpCQEL744otS9YgazpYLDYi7U+/evZWtW7cqL7zwgjJ79mxFURTl22+/VYYPH64oiqKkpqYqLVq0UEwmk/Uxw4cPV7799ltFURRl5cqVSlBQkLJixQqlqKhImT17ttKzZ09l8uTJSmFhofL7778r7dq1U/Lz8xVFUZRXX31VadeunbJjxw6lsLBQmTp1qvLoo48qiqIoBoNBuf/++5UVK1YoJpNJOXDggNK5c2fl6NGj1sd26NBB2bVrl2I2m5UrV66Uej5RUVHKpEmTlCtXrijJyclKly5dlD/++MNa69Vt3cjV57J48WLFaDQqa9euVTp06KBcvHhRURRFef7555U333xTMRgMSlZWljJkyBBl+fLliqIUL8rSt29f5dixY4rJZFLmzZunDBs2zNp3ixYtlCeffFK5ePGiUlBQUGrbq1evLtH+6NGjSnBwsFJYWKhcvHhR6dixo7J69WrFZDIpa9asUTp27KhkZ2eX+vcozzh2797dujhITk6OkpSUdNMxETWT7PmLchs7diwxMTEVWjzEz8+PIUOGoNPpGDhwIOnp6bzwwgvo9Xp69OiBXq/n9OnT1va9evWiU6dO6PV6xo0bx549e0hPT2fjxo00aNCAIUOGYGdnR8uWLQkLCyuxeEWfPn0IDg5Gq9Xi4OBQoo709HQSExN55ZVXcHBwICgoiKFDhxIXF1fu5+Lh4cETTzyBvb09AwcOpEmTJmzcuJGsrCw2bdrEa6+9hpOTE56enjz55JOsXbsWKJ5wa/To0QQEBGBnZ8ezzz7LwYMHS+z9jx49Gjc3N+v8Rtfr27cvhw4dsrZfs2YN/fr1Q6/Xs3HjRho3bkxkZCR2dnaEh4fTtGlTfvvttxs+h7LG0c7OjmPHjpGfn4+rqyv33ntvucdH1AxVupiLqFlatGhBr169WLBgAQEBAbf1WE9PT+vPV4Pt+omnHBwcMBgM1t/r169v/dnZ2RlXV1cyMzM5c+YM+/bts65aBMXH9QcPHmz9/VZTHGdmZuLq6oqLi4v1Nl9fX5KSksr9XLy9vdFoNCUen5mZydmzZykqKqJHjx7W+ywWi7Wes2fPMmPGDGbNmmW9X1EUMjIyaNCgQZm1u7i40LNnT9auXcvo0aOJj49n2rRp1ud1/YIeV+vKyMi4YV9ljeOcOXP49NNPef/99wkMDOTll1+mffv25RofUTNI+IvbMnbsWB566CGeeuop621Xvxy9cuWKNVTPnz9/R9s5d+6c9WeDwcClS5fw8vLCx8eHTp06sXjx4gr16+XlxaVLl8jPz7fWenWFrPLKyMhAURTrG0B6ejqhoaHUr18fvV7Pn3/+iZ1d6T8tHx8fnn322RJvVP/r+jeVGwkPD+fjjz+mU6dOFBYWWpe+9PLyKjGn+9W6QkJCbthPWePYpk0bPv30U0wmE0uXLuWll15i06ZNt6xN1Cxy2EfclsaNGzNw4EC++uor620eHh54e3sTFxeH2WxmxYoVpKam3tF2Nm3axK5duzAajXz00Ue0bdsWHx8fevXqxcmTJ/nuu+8wmUyYTCb27dvH8ePHy9Wvj48P7du3Z/bs2RQWFnLo0CFWrFhxy0D+X9nZ2SxZsgSTycQPP/zA8ePH6dmzJ15eXnTv3p2ZM2eSn5+PxWLh9OnT7NixAyheW3XBggUcPXoUKP5y+IcffritcenZsydnz55lzpw5DBw4EK1Wa7395MmTrFmzhqKiItatW8exY8fo1asXUPwp6/p/k1uNo9Fo5PvvvycvLw97e3ucnZ2t2xF3D/kXFbfthRdeKHHOPxQvvfjFF1/QpUsXjh07dseHCMLDw5k3bx5dunThwIEDvPvuu0DxoY8vvviCdevWERISQo8ePXjvvfcwGo3l7nv27NmcOXOGkJAQXnzxRcaMGUO3bt3K/fg2bdpw6tQp7rvvPj788EPmzJmDu7s7AO+88w4mk4mBAwfSqVMnxo4da/0U1K9fP55++mn+9a9/0aFDB8LDw9m8efNtjErxCk79+vXjjz/+sC7kDuDu7s78+fNZvHgxXbp04fPPP2f+/PnWhV4ef/xxfvrpJzp16sS0adPKHMe4uDhCQ0Pp0KEDX3/9tXX8xd1D5vMX4jasWrWK2NhYli9fXtWlCHFHZM9fCCFUSMJfCCFUSA77CCGECsmevxBCqJCEvxBCqJCEvxBCqJCEvxBCqJCEvxBCqJCEvxBCqND/Ax/RGUpYmQwkAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def convert_df_to_dict(df: pd.DataFrame):\n",
    "    return {x:list(df[x]) for x in df}\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "peer_votes_num_acc_df_part = peer_votes_num_acc_df.iloc[:xlim]\n",
    "debug(\"Confidence and accuracy\")\n",
    "sns.set_theme(style=\"whitegrid\")\n",
    "ax = sns.lineplot(data=peer_votes_num_acc_df_part, x=peer_votes_num_acc_df_part.index, y='Accuracy', label = \"Accuracy\")\n",
    "ax = sns.lineplot(data=peer_votes_num_acc_df_part, x=peer_votes_num_acc_df_part.index, y='Confidence', label = \"Average confidence\")\n",
    "# sns.lineplot(data=peer_votes_num_acc_df_part, x=peer_votes_num_acc_df_part.index, y='Total scaled')\n",
    "ax.legend(loc='upper right')\n",
    "ax.set_ylim([-0.01, 1.01])\n",
    "ax.set_xlim([0, xlim])\n",
    "plt.xlabel(\"Number of peer votes\")\n",
    "plt.ylabel(\"\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Number of peer votes')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEOCAYAAACn00H/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAApnElEQVR4nO3deVQT594H8C8Jm6CCoEAQV1RKpSqCehVQiyCoUex1vaitteLbV6tdbr0uteLeYm+t2mo9WpdafG3FFq241ipa1KqArSIugAsoAZRFFllCMu8fXkO5uISRhBC+n3M8J5k8mfnlMfBl5pmZx0QQBAFERES1JKnvAoiIqGFigBARkSgMECIiEoUBQkREojBAiIhIFAYIERGJopcAiYiIgL+/P9zc3HD9+vUntlGpVFi8eDECAgIQGBiIqKgofZRGREQimepjI4MGDcLrr7+OCRMmPLXNvn37kJ6ejiNHjqCgoAAjR45E37594eLiotU21Go1SkpKYGZmBhMTk7oqnYjIqAmCAKVSCWtra0gktdun0EuAeHt7P7fNgQMHMGbMGEgkEtjZ2SEgIACHDh3C1KlTtdpGSUnJU/duiIjo2bp06YJmzZrV6j16CRBtKBQKODs7a57LZDJkZWVp/X4zMzMAjzrB3Ny8zutraJKSkuDh4VHfZdS7X8+n40T8DVg3bVrjNVOJCf4x2A3OrWq+Zqz4vajCvnikoqIC169f1/wOrQ2DCZAX9fiwFfdCqiQlJdV3CfXu/44ooFYLsK8oqvGaqQS4dj0Fedm1/8FpyPi9qMK+qCLm0L/BBIhMJkNmZia6desGoOYeibY8PDxgYWFR1+U1OAkJCfDy8qrvMurVg+JyPCi5g8GeNpg5cWB9l2MQ+L2owr54pLy8XHSQGsxpvMHBwYiKioJarUZeXh6OHj2KoKCg+i6LGrCUjAIAgLMdD2kS6YJeAmTZsmXo378/srKy8Oabb2LYsGEAgLCwMFy6dAkAEBISAhcXFwwePBhjx47FjBkz0KZNG32UR0Yq9U4BTEwAmV3jOkRFpC96OYS1YMECLFiwoMbyTZs2aR5LpVIsXrxYH+VQI5GaUQAXh6awMDOYHW0io8KfLDJaKRkF6ORiW99lEBktBggZpdwHpcgrLEOnNrb1XQqR0WKAkFFK/c8AemeXFvVbCJERY4CQUUq5UwCJxAQdWjev71KIjBYDhIxSSkYB2jo2g6W5wVzqRGR0GCBkdARBQGpGATpz/INIpxggZHTu5ZeisKSCA+hEOsYAIaOTcqcAALgHQqRjDBAyOinp+TCVmqC9jAPoRLrEACGjk3qnAO1lzWFmKq3vUoiMGgOEjMrjAfRObXj9B5GuMUDIqChyS1BSVsnxDyI9YICQUUlJLwDAAXQifWCAkFFJvVMAc1MJ2jjWbm5nIqo9BggZlZSMAnRobQNTKb/aRLrGnzIyGiq1gLQ7vAKdSF8YIGQ07uYUoaxCxQAh0hMGCBmN1P9cgc5JpIj0g7cqJQBApUqN0xczUVpeWWfrNDeTon+P1pDqaTwiJaMAluZStHbgADqRPjBACADw/ZFr+OHo9Tpfb2l5JYb261Dn632SlIwCuLrYQiox0cv2iBo7Bggh90Epok+kwaebM8JGetTZeldsO4fo2FQE9Wmn872QSpUaN+8+wFAf/YQVETFACMCOQ1ehVqsxWf4y7G2a1Nl6R73aGZ98ex6nLyrg59m6ztb7JBnZRaioVHP8g0iPOIjeyN1SFOLX8+kY5tMRTvbWdbruPh4ytG5ljd3HUyAIQp2u+7+lPJ4DnWdgEekNA6SR2xZzGU0szTAusEudr1sqMcFrAzvjxt0H+OP6vTpf/1+lZBTA2tIUspZ1G4JE9HQMkEbsz+v3kHA1B2MHdUEzK3OdbMPf2wV2zS3w4/EUnaz/sdSMfHRqYwsTEw6gE+kLA6SRUqsFbIm5DIcWTSD31d3As5mpFCH9XfFnyn2k/ucwU11TVqpwS1HI8Q8iPWOANFInLtzBjbsPMGmIO8zNdDvxUnDf9rC2NMVuHe2F3FIUolIloDPnACHSK56F1QhVKFX47uAVuLrYoL+ni863Z2VphiH9OuCn4ynIvF8M55ZNa/X+21mFOHT6Fk7+cRdlFaoar6vVagBAJw6gE+kVA6QR2vfbDdzLL8V74z0h0dNFdyP8OmLvyTREx6Zhxujuz22vrFTh1EUFDp25hcs3cmEqlaDvKzK0sn3yacYOLZrA0c6qrssmomdggDQyhSUViPr1OrzdHdGtUyu9bbdFc0v4e7fBr+fTETrYDS2aWz6xXVZuCQ6duYVfzqWjsKQCMntrvCl/GYN6tYVNUwu91UtEz8cAMXD5hWWwaWpRZ3sKPxy9htLySkyWv1wn66uNvw/shCNnb+Pn327gjWHVt5+eVYioYyk4mXgHMDFBn65OGNK3Pbp3bqW3vSQiqh0GiAG7nVWIdz+Phbe7I2ZP8obFCw52H4tPR0zcTQT0bod2Ts3rqErtObdqin6vOOPg6ZsYM6gzrCzNkHanALt+vY4zlxQwN5NiRH9XhPR3RcunHKoiIsPBADFge2LTYGICnEvOQvjGM1gwpQ+aNjETta6fjqdia8xldO/cEm+N6FrHlWpvlH8nnLqYiS37LiP3QRnir2TDytIUYwd1wXC/jjxMRdSAMEAMVF5hGWITMxD0t/bo2sEeq3YmYN66OCwK+1ut7lelVgvYtj8Z0bGp8OvRGu//wxNmpro9bfdZOrdpge6dW+Lw77fRzMock4a4Y5hPB1iLDEYiqj96C5CbN29i7ty5KCgogK2tLSIiItC+fftqbXJzczFv3jwoFApUVlaiT58+WLBgAUxNG1/OxcTdgFotIKS/K2QtrdHM2gwrtp3Dv76Kw5JpfdG61fNPha1UqfHlrj9wLD4Dcp8OCBv5ikGMJ7wzpgcupd6HX4/WsLRofP+3RMZCbxcShoeHIzQ0FIcPH0ZoaCgWLlxYo82GDRvg6uqKffv24eeff8bly5dx5MgRfZVoMErLK3Hg9C30fcVZc2+nHl0csPx/fVBeUYk5X/323Ku6KyrVWL71HI7FZ2BC8EuY9pphhAcAONlbI7BPO4YHUQOnlwDJzc1FcnIy5HI5AEAulyM5ORl5eXnV2pmYmKCkpARqtRoVFRVQKpVwdHTUR4kG5Zdzt1FSqsRrA12rLe/cpgUi3vGDhZkU87+Ow9kkBXLyHtb4dyenCNuP3Ufi1WzMGN0d4wPdeI8oIqpzevkTUKFQwNHREVLpo2PvUqkUDg4OUCgUsLOz07SbPn06Zs6cCV9fX5SWlmLChAnw8vLSR4kGQ6VSY+/JG3i5gx3c2tnVeL11q6ZYOdMPizb9jmVbzz11PVIJMOf1XujXzVmX5RJRI2ZQxxAOHToENzc3fPvttygpKUFYWBgOHTqE4OBgrdeRlJSkwwp1L+n2o70If48mSEhIeGq78b5Nce2OFP+5i0cNzvbmsFAqkJCg0FGlDcuz+rKxYV9UYV+8GL0EiEwmQ3Z2NlQqFaRSKVQqFXJyciCTyaq1i4yMxIoVKyCRSNCsWTP4+/vj7NmztQoQDw8PWFjo7lRQQRCw+vsLaC9rjtcGdqrzde/47SScW1rjH8N9njtm4fOM1xISEhrd3tvTsC+qsC+qsC8eKS8vF/2Ht17GQOzt7eHu7o6YmBgAQExMDNzd3asdvgIAFxcXnDx5EgBQUVGBM2fOoHPnzvooUWsXrt/DsfgMbNl3GacuZtbpui/fyEVKRgFGDuxkMAPeRERPo7ezsBYtWoTIyEgEBQUhMjISixcvBgCEhYXh0qVLAID58+cjISEBw4cPx8iRI9G+fXuMHTtWXyVqJerX67C3sYRb2xZYvTMRt7MK62zd0bFpaG5tDn/vNnW2TiIiXdHbGIirqyuioqJqLN+0aZPmcdu2bbF161Z9lVRryTdzkZSWi7AQD/h0d8Z7X5zAiq3n8Pl7A0RfIf5YRnYRziVnIXSw2wvfsoSISB84oVQtRP2agubW5hjcpx3sbZpg7uu9kJ33EKv+LwFqtfBC6957Mg3mphIM9dHd7IBERHWJAaKlG3cfIP5KNkb076i5AK5rR3uEhXjgfHI2vv/lmuh15xeV4Vh8Bm9ZTkQNCgNES1G/XoeVpSmG+XSstnyoTwcM6tUGO49cw9kkcafM7o+7iUqVGiEDXJ/fmIjIQDBAtHAnpwinLmZimE+HGmMdJiYmmD6qOzq52GDVzkTcySmq1brPXFJg97EU9OvmrNX9rYiIDEWjDpCSUiWOnktHWUXlM9v9eCwVZlIJRvg9eQ/B3EyKeZN7w1QqwfKt51BYUqHV9s8lZ2Hld+fRqY0tZo3tUdvyiYjqVaMNkKzcEsz+8jes+eEC5q0/hdwHpU9sl5P/EMcTMjD4b+1g2+zp4xMOLaww53VvZOWW4N3Pj+NS2v1nbj/+SjY+2XYeHZxtsDisL6wseTtzImpYGmWAXL6Riw9Wn0R+YRkmDXHHnewi/HPNSaTeKajRNjo2FQDw94HPv6CxW6dW+Gxmf5ibSfHR16cQefAKVKqa9xpJvJaDFdvOoZ2sGZZM68u5MIioQWp0AXL0XDoWbDiF5tZm+Pe7/TE2oAtWzvSDRGKCOV/F4fRfri7PLyrDkd9vw9+7DVq10G4Sp05tbLH6g4EY5N0WPxy9jrnr4pCd91Dz+p8p97B8y1m4ODTF0v/ph6ZW5nX+GYmI9KHRBIhKLWDrvstY88MFdO1oj3/P6q8ZtO7gbIPP3+2PDs7N8cm357Hr6HUIgoCfT96AUqXGKP/a3U6liYUp3h3vidkTvZCeXYRZnx/Hbxfu4lLafSzZfBayltZY+j/90IzhQUQNmEHdjVdXSssr8fmOBJy9nIUh/dpj2shXYCqtnp0tmllixf/64Mtdf+C7g1dwW1GI81ey4fMCZ0f193RBl7Yt8O8dCVgZGQ9TqQlkLZti2ds+vN6DiBo8ow6QsopKnEi8i+jYVCjuF+N/XnsFw3w6PHVyJXMzKT4I7QkXx6aIPHgVADA2oMsL1eBkb41PZ/jih1+uI/lmLj6c4PXMwXgioobCKAMk834xDpy6haPn01FSqkR7WXMsCusLTzeH577XxMQE4wLc0MHZBjl5D9HB2eaF6zGVSjAh+KUXXg8RkSExugD54vtEnLqYA6nEBP26OWOYTwe83MGu1lO69n7ZSUcVEhEZB6MLkDvZRQgd7Iagvu1h19yyvsshIjJaRhcgK9/xg5WVdqfcEhGReEZ3Gq9UanQfiYjIIPG3LRERicIAISIiURggREQkCgOEiIhEYYAQEZEoDBAiIhKFAUJERKIwQIiISBQGCBERicIAISIiURggREQkCgOEiIhEYYAQEZEoDBAiIhKFAUJERKIwQIiISBQGCBERiaJVgFy9evWFN3Tz5k2MGzcOQUFBGDduHG7duvXEdgcOHMDw4cMhl8sxfPhw3L9//4W3TUREdU+rOdEnT54MBwcHhISEYPjw4XBwcKj1hsLDwxEaGoqQkBDs3bsXCxcuxPbt26u1uXTpEr766it8++23aNWqFYqKimBubl7rbRERke5ptQcSFxeHWbNm4c8//0RQUBCmTJmCvXv3orS0VKuN5ObmIjk5GXK5HAAgl8uRnJyMvLy8au22bduGKVOmoFWrVgCAZs2awcLCojafh4iI9ESrADE1NUVAQADWrl2LkydPYsiQIfjmm2/Qr18//Otf/0JCQsIz369QKODo6AipVAoAkEqlcHBwgEKhqNYuLS0NGRkZmDBhAl577TWsX78egiCI/GhERKRLWh3CeqykpARHjx7F/v37kZ2djWHDhkEmk2H27NkYMGAAwsPDX6gYlUqFa9euYevWraioqMDUqVPh7OyMkSNHar2OpKSkF6rBmDwv2BsT9kUV9kUV9sWL0SpAYmNjsXfvXpw8eRI9e/bEmDFjEBAQoDm8NGHCBLz66qtPDRCZTIbs7GyoVCpIpVKoVCrk5ORAJpNVa+fs7Izg4GCYm5vD3NwcgwYNwsWLF2sVIB4eHjzshUc/GF5eXvVdhkFgX1RhX1RhXzxSXl4u+g9vrQ5hff755+jatSsOHjyITZs2YdiwYdV+Sdva2mL+/PlPfb+9vT3c3d0RExMDAIiJiYG7uzvs7OyqtZPL5YiLi4MgCFAqlfj999/x0ksviflcRESkY1rtgezbt++5bcaMGfPM1xctWoS5c+di/fr1aN68OSIiIgAAYWFhmDVrFl555RUMGzYMSUlJGDp0KCQSCXx9fTF69GhtSiQiIj3TKkDeeecdTJ48Gd7e3ppl8fHx2L59O9auXavVhlxdXREVFVVj+aZNmzSPJRIJ5s2bh3nz5mm1TiIiqj9aHcI6f/48PD09qy3r0aMHzp49q5OiiIjI8GkVIObm5jWu+Xj48CFMTWt1EhcRERkRrQLE19cXCxcuRHFxMQCguLgYS5YsgZ+fn06LIyIiw6VVgMydOxfFxcXo3bs3+vbti969e6O4uPiZZ14REZFx0+oYlI2NDTZu3IicnBxkZWVBJpNpbjdCRESNU60GMRwcHNCqVSsIggC1Wg3g0ZlTRETU+GgVINnZ2ViyZAni4+NRWFhY7bUrV67opDAiIjJsWu0+hIeHw8zMDNu2bYOVlRWio6Ph7++PxYsX67o+IiIyUFrtgVy4cAHHjx+HlZUVTExM8NJLL2H58uUYP348xo4dq+saiYjIAGm1ByKRSDTXfDRv3hx5eXmwsrJCdna2TosjIiLDpdUeSPfu3XHixAkEBgbC19cX7733HiwtLeHh4aHr+oiIyEBpFSArV67UnHU1f/58bNmyBSUlJXjjjTd0WhwRERmu5waISqXC8uXLsXTpUgCApaUlpk+frvPCiIjIsD13DEQqleLUqVMwMTHRRz1ERNRAaDWI/sYbb+DLL7+EUqnUdT1ERNRAaDUGEhkZifv372Pr1q2ws7OrtjcSGxurq9qIiMiAaRUgn332ma7rICKiBkarAOndu7eu6yAiogZGqwBZs2bNU197991366wYIiJqOLQKkKysrGrP7927h/PnzyMgIEAnRRERkeHTKkA++eSTGstOnjyJ/fv313lBRETUMIiezMPX1xdHjx6ty1qIiKgB0WoPJCMjo9rz0tJSxMTEQCaT6aQoIiIyfFoFSGBgIExMTCAIAgCgSZMmcHd3x6effqrT4oiIyHBpFSBXr17VdR1ERNTAaDUGcuXKFSgUimrLFAoFg4WIqBHTKkBmz56NysrKasuUSiVmz56tk6KIiMjwaRUgmZmZaNOmTbVlbdu2xd27d3VSFBERGT6tAsTJyQmXL1+utuzy5ctwcHDQSVFERGT4tBpEnzx5MqZPn46pU6eibdu2SE9Px5YtW/D222/ruj4iIjJQWgXI2LFj0axZM+zevRtZWVlwcnLCnDlzEBwcrOv6iIjIQGkVIAAwZMgQDBkyRJe1EBFRA6LVGMiyZcuQmJhYbVliYiKWL1+uk6KIiMjwaRUgMTEx8PDwqLbMw8MDMTExOimKiIgMn1YB8tfbmDymUqmgVqu13tDNmzcxbtw4BAUFYdy4cbh169ZT2964cQPdu3dHRESE1usnIiL90ipAvL29sXr1ak1gqNVqrF27Ft7e3lpvKDw8HKGhoTh8+DBCQ0OxcOHCJ7ZTqVQIDw/nXCNERAZOqwD56KOPcPr0afj6+mL06NHw9fXFmTNn8PHHH2u1kdzcXCQnJ0MulwMA5HI5kpOTkZeXV6Ptxo0bMXDgQLRv3177T0FERHqn1VlYTk5OiI6OxsWLF6FQKNCyZUscPXoUo0ePRlxc3HPfr1Ao4OjoCKlUCgCQSqVwcHCAQqGAnZ2dpt3Vq1cRFxeH7du3Y/369SI/EhER6YPWp/EWFBTgzz//RHR0NK5duwZvb2989NFHdVaIUqnExx9/jE8++UQTNGIkJSXVWU0NXUJCQn2XYDDYF1XYF1XYFy/mmQGiVCpx7NgxREdHIy4uDm3btsWwYcOgUCiwevVq2Nvba7URmUyG7OxsqFQqSKVSqFQq5OTkVJuQ6t69e0hPT8e0adMAAIWFhRAEAcXFxVi6dKnWH8jDwwMWFhZatzdWCQkJ8PLyqu8yDAL7ogr7ogr74pHy8nLRf3g/M0B8fHxgYmKCv//975g5cya6du0KANi5c2etNmJvbw93d3fExMQgJCQEMTExcHd3r3b4ytnZGWfPntU8//LLL/Hw4UPMmTOnVtsiIiL9eOYgupubG4qKivDnn3/i0qVLePDggegNLVq0CJGRkQgKCkJkZCQWL14MAAgLC8OlS5dEr5eIiOrHM/dAvvvuO9y9exd79uzBli1bsGzZMvj6+uLhw4c15gd5HldXV0RFRdVYvmnTpie2nzlzZq3WT0RE+vXc03hbt26NGTNm4MiRI9i2bRtatWoFiUSCESNGYOXKlfqokYiIDJDWZ2EBjy4o9Pb2xoIFC/DLL79gz549OiqLiIgMXa0C5DELCwvI5XLNhYFERNT4aHUlOhER0X9jgBARkSgMECIiEoUBQkREojBAiIhIFAYIERGJwgAhIiJRGCBERCQKA4SIiERhgBARkSgMECIiEoUBQkREojBAiIhIFAYIERGJwgAhIiJRGCBERCQKA4SIiERhgBARkSgMECIiEoUBQkREojBAiIhIFAYIERGJwgAhIiJRGCBERCQKA4SIiERhgBARkSgMECIiEoUBQkREojBAiIhIFAYIERGJwgAhIiJRTPW1oZs3b2Lu3LkoKCiAra0tIiIi0L59+2pt1q1bhwMHDkAikcDMzAzvv/8+/Pz89FUiERHVgt4CJDw8HKGhoQgJCcHevXuxcOFCbN++vVqbbt26YcqUKWjSpAmuXr2KiRMnIi4uDpaWlvoqk4iItKSXQ1i5ublITk6GXC4HAMjlciQnJyMvL69aOz8/PzRp0gQA4ObmBkEQUFBQoI8SiYiolvQSIAqFAo6OjpBKpQAAqVQKBwcHKBSKp75nz549aNu2LZycnPRRIhER1ZLeDmHVxrlz57BmzRps2bKl1u9NSkrSQUUNU0JCQn2XYDDYF1XYF1XYFy9GLwEik8mQnZ0NlUoFqVQKlUqFnJwcyGSyGm0vXLiA2bNnY/369ejYsWOtt+Xh4QELC4u6KLtBS0hIgJeXV32XYRDYF1XYF1XYF4+Ul5eL/sNbL4ew7O3t4e7ujpiYGABATEwM3N3dYWdnV63dxYsX8f7772Pt2rXo2rWrPkojIiKR9HYdyKJFixAZGYmgoCBERkZi8eLFAICwsDBcunQJALB48WKUlZVh4cKFCAkJQUhICK5du6avEomIqBb0Ngbi6uqKqKioGss3bdqkefzjjz/qqxwiInpBvBKdiIhEYYAQEZEoDBAiIhKFAUJERKIwQIiISBQGCBERicIAISIiURggREQkCgOEiIhEYYAQEZEoDBAiIhKFAUJERKIwQIiISBQGCBERicIAISIiURggREQkCgOEiIhEYYAQEZEoDBAiIhKFAUJERKIwQIiISBQGCBERicIAISIiURggREQkCgOEiIhEYYAQEZEoDBAiIhKFAUJERKIwQIiISBQGCBERicIAISIiURggREQkCgOEiIhE0VuA3Lx5E+PGjUNQUBDGjRuHW7du1WijUqmwePFiBAQEIDAwEFFRUfoqj4iIaklvARIeHo7Q0FAcPnwYoaGhWLhwYY02+/btQ3p6Oo4cOYIffvgBX375Je7cuaOvEomIqBb0EiC5ublITk6GXC4HAMjlciQnJyMvL69auwMHDmDMmDGQSCSws7NDQEAADh06pI8SiYiolvQSIAqFAo6OjpBKpQAAqVQKBwcHKBSKGu2cnZ01z2UyGbKysvRRIhER1ZJpfRdQ15KSkuq7BIORkJBQ3yUYDPZFFfZFFfbFi9FLgMhkMmRnZ0OlUkEqlUKlUiEnJwcymaxGu8zMTHTr1g1AzT2SZxEEAQDQpUsXmJub1+0HaICSkpLg4eFR32UYBPZFFfZFFfbFIxUVFbh+/brmd2ht6CVA7O3t4e7ujpiYGISEhCAmJgbu7u6ws7Or1i44OBhRUVEYPHgwCgoKcPToUezYsUOrbSiVSgDA9evX67z+hop7Y1XYF1XYF1XYF1WUSiUsLS1r9R4TQUzsiJCWloa5c+eisLAQzZs3R0REBDp27IiwsDDMmjULr7zyClQqFZYsWYJTp04BAMLCwjBu3Dit1q9Wq1FSUgIzMzOYmJjo8qMQERkNQRCgVCphbW0NiaR2w+J6CxAiIjIuvBKdiIhEYYAQEZEoDBAiIhKFAUJERKIwQIiISBQGCBERicIAISIiUYwiQLSZa8RYRUREwN/fH25ubtWuwm+MfZKfn4+wsDAEBQVh+PDheOeddzR3fP7jjz8wYsQIBAUFYcqUKcjNza3nanVv+vTpGDFiBEaOHInQ0FBcuXIFQOP8bgDAV199Ve3npDF+JwDA398fwcHBCAkJQUhICH777TcAIvtDMAKTJk0S9uzZIwiCIOzZs0eYNGlSPVekP+fPnxcyMzOFV199Vbh27ZpmeWPsk/z8fOH333/XPP/000+FefPmCSqVSggICBDOnz8vCIIgrFu3Tpg7d259lak3hYWFmse//PKLMHLkSEEQGud3IykpSXjrrbc0PyeN9TshCEKN3xWCIIjujwa/B6LtXCPGytvbu8ZNKRtrn9ja2qJPnz6a5z169EBmZiaSkpJgYWEBb29vAMD48eMbxTwzzZo10zwuLi6GiYlJo/xuVFRUYMmSJVi0aJFmWWP9TjyN2P5o8Ldzf9ZcI/99s8bGgn3y6N5oO3fuhL+/f427OtvZ2UGtVqOgoAC2trb1V6QefPTRRzh16hQEQcA333zTKL8ba9aswYgRI+Di4qJZ1pi/EwDw4YcfQhAEeHl54YMPPhDdHw1+D4ToSZYuXQorKytMnDixvkupV8uXL0dsbCzef/99rFy5sr7L0bsLFy4gKSkJoaGh9V2KwdixYwd+/vln/PjjjxAEAUuWLBG9rgYfIH+dawTAU+caaUwae59ERETg9u3bWL16NSQSiWaemcfy8vIgkUgaxV+aj40cORJnz56Fk5NTo/punD9/HmlpaRg0aBD8/f2RlZWFt956C7dv326034nH/9fm5uYIDQ1FYmKi6J+RBh8gf51rBMBT5xppTBpzn6xatQpJSUlYt26dZmIxDw8PlJWVIT4+HgDw/fffIzg4uD7L1LmSkpJqU0YfO3YMNjY2je67MW3aNMTFxeHYsWM4duwYnJycsHnzZkydOrXRfScA4OHDhygqKgLw6DbuBw4cgLu7u+ifEaO4nfvT5hppDJYtW4YjR47g/v37aNGiBWxtbbF///5G2ScpKSmQy+Vo3769ZmIcFxcXrFu3DomJiQgPD0d5eTlat26Nzz77DC1btqzninXn/v37mD59OkpLSyGRSGBjY4M5c+aga9eujfK78Zi/vz82bNiALl26NLrvBABkZGRg5syZUKlUUKvVcHV1xYIFC+Dg4CCqP4wiQIiISP8a/CEsIiKqHwwQIiIShQFCRESiMECIiEgUBggREYnCACGjNHfuXHzxxRf1sm1BEDBv3jz06tULo0ePrpcaiPSBAUJ64e/vj759++Lhw4eaZVFRUZg0aVI9VqUbCQkJOHXqFE6cOIHdu3fXdzl1ZtKkSYiKiqrvMsiAMEBIb9RqNbZv317fZdTa49t+aOvu3bto3bo1rKysdFSRdiorK+t1+2T8GCCkN2+99Ra2bNmCwsLCGq/duXMHbm5u1X7p/fUv3p9++gnjx4/HihUr4O3tjUGDBiExMRE//fQTBgwYgL59+yI6OrraOvPz8/Hmm2/C09MTEydOxN27dzWvpaWl4c0330Tv3r0RFBSEAwcOaF6bO3cuwsPDERYWhh49euDs2bM16s3Ozsbbb7+N3r17IzAwELt27QLwaK9qwYIF+OOPP+Dp6Ym1a9fWeO/jz7JkyRJ4eXkhODgYZ86c0bxeVFSE+fPnw9fXF35+fvjiiy+qhdju3bsxZMgQ9OrVC2+99Va1z+Xm5oYdO3Zg8ODBGDx4cI1tT506FZGRkdWWjRgxAkeOHAEAJCYmYtSoUfDy8sKoUaOQmJgIAPjiiy8QHx+PJUuWwNPTU3MDvmf144kTJzB06FB4enrCz88PmzdvrlEPNXB1OVEJ0dO8+uqrwqlTp4QZM2YIq1atEgRBEHbt2iVMnDhREARByMjIELp06SIolUrNeyZOnCjs2rVLEARB+PHHHwV3d3dh9+7dQmVlpbBq1SphwIABwqJFi4Ty8nLht99+E3r06CEUFxcLgiAIc+bMEXr06CGcO3dOKC8vF5YuXSqMHz9eEARBKCkpEfr37y/s3r1bUCqVwuXLl4XevXsLKSkpmvf27NlTiI+PF1QqlVBWVlbj84SGhgrh4eFCWVmZkJycLPTp00c4ffq0ptbH23qSx59l69atQkVFhbB//36hZ8+eQn5+viAIgjB9+nTh448/FkpKSoT79+8Lo0aNEnbu3CkIwqOJoQICAoTU1FRBqVQK69atE8aNG6dZd5cuXYTJkycL+fn5QmlpaY1tR0dHV2ufkpIieHl5CeXl5UJ+fr7g7e0tREdHC0qlUti3b5/g7e0t5OXl1fj/0KYffXx8NBMUFRQUCElJSU/tE2qYuAdCejVr1ixERkaKmsDIxcUFo0aNglQqxdChQ6FQKDBjxgyYm5vD19cX5ubmSE9P17QfOHAgevXqBXNzc7z//vv4448/oFAoEBsbi9atW2PUqFEwNTXFyy+/jKCgoGoT6AwaNAheXl6QSCSwsLCoVodCoUBiYiI+/PBDWFhYwN3dHWPGjMHevXu1/ix2dnZ44403YGZmhqFDh6JDhw6IjY3F/fv3ceLECcyfPx9WVlawt7fH5MmTsX//fgCPbnI3bdo0uLq6wtTUFG+//TauXLlSbS9k2rRpsLW11dwP7K8CAgJw9epVTft9+/YhMDAQ5ubmiI2NRbt27TBy5EiYmppCLpejY8eOOH78+BM/w/P60dTUFKmpqSguLoaNjQ26du2qdf9Qw9DgJ5SihqVLly4YOHAgNm7cCFdX11q9197eXvP48S/Hv97szcLCAiUlJZrnTk5OmsfW1tawsbFBTk4O7t69i4sXL2pmXwMejXOMGDFC8/xZtzfPycmBjY0NmjZtqlnm7OyMpKQkrT+Lo6MjTExMqr0/JycHmZmZqKyshK+vr+Y1tVqtqSczMxMrVqxARESE5nVBEJCdnY3WrVs/t/amTZtiwIAB2L9/P6ZNm4aYmBgsW7ZM87n+OqnQ47qys7OfuK7n9ePatWvx9ddf4/PPP4ebmxv++c9/wtPTU6v+oYaBAUJ6N2vWLLz22muYMmWKZtnjAeeysjLNL+Z79+690HaysrI0j0tKSvDgwQM4ODhAJpOhV69e2Lp1q6j1Ojg44MGDByguLtbU+nimP21lZ2dDEARNiCgUCvj7+8PJyQnm5ub4/fffYWpa88dTJpPh7bffrhZ2/+2vwfQkcrkcX331FXr16oXy8nLNNMAODg7V5oR4XJefn98T1/O8fuzWrRu+/vprKJVK7NixA++99x5OnDjxzNqoYeEhLNK7du3aYejQofjuu+80y+zs7ODo6Ii9e/dCpVJh9+7dyMjIeKHtnDhxAvHx8aioqMCaNWvQvXt3yGQyDBw4ELdu3cKePXugVCqhVCpx8eJFpKWlabVemUwGT09PrFq1CuXl5bh69Sp27979zF/q/y0vLw/bt2+HUqnEwYMHkZaWhgEDBsDBwQE+Pj749NNPUVxcDLVajfT0dJw7dw7Ao7mqN27ciJSUFACPBtwPHjxYq34ZMGAAMjMzsXbtWgwdOhQSiUSz/NatW9i3bx8qKytx4MABpKamYuDAgQAe7e399f/kWf1YUVGBn3/+GUVFRTAzM4O1tbVmO2Q8+D9K9WLGjBnVrgkBHk1Du3nzZvTp0wepqakvfLhDLpdj3bp16NOnDy5fvozPPvsMwKPDOJs3b8aBAwfg5+cHX19f/Pvf/0ZFRYXW6161ahXu3r0LPz8/vPPOO5g5cyb69eun9fu7deuG27dv429/+xtWr16NtWvXokWLFgCAlStXQqlUYujQoejVqxdmzZql2RsLDAzE1KlT8cEHH6Bnz56Qy+U4efJkLXrl0Ux0gYGBOH36NORyuWZ5ixYtsGHDBmzduhV9+vTBN998gw0bNmgmm3r99ddx+PBh9OrVC8uWLXtuP+7duxf+/v7o2bMnvv/+e03/k/HgfCBEevbTTz8hKioKO3furO9SiF4I90CIiEgUBggREYnCQ1hERCQK90CIiEgUBggREYnCACEiIlEYIEREJAoDhIiIRGGAEBGRKP8PHjdxwb2CPNQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = sns.lineplot(data=peer_votes_num_acc_df_part, x=peer_votes_num_acc_df_part.index, y='Accuracy')\n",
    "# sns.lineplot(data=peer_votes_num_acc_df_part, x=peer_votes_num_acc_df_part.index, y='Total scaled')\n",
    "ax.set_ylim([-0.01, 1.01])\n",
    "ax.set_xlim([0, xlim])\n",
    "plt.xlabel(\"Number of peer votes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:06:31 \u001b[0m\u001b[1;33m------------------\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;32mDEBUG:\u001b[0m\u001b[1;33m at \u001b[0m\u001b[1;32m<ipython-input-15-f4e2ca066c58>:1 <module>\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m\u001b[0m\u001b[1;36mSame as majority rate\u001b[0m\u001b[1;33m\u001b[0m\n",
      "\u001b[1;33m------------------\u001b[0m\u001b[1;31m 2022-10-24 16:06:31 \u001b[0m\u001b[1;33m------------------\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Number of peer votes')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEOCAYAAACn00H/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA5A0lEQVR4nO3deVxU9f4/8NfMsCOLoOyaQam4oAhoXjXcUEQUr7n0o0xT9LqFbTeXTFxz6ZtdvWmWC2X6qBumqWhmWZpaiiJmhEvuCsPOoGzDMHN+fxCDEy6HgTOD8Ho+Hjwew+HMfN7zcTzv+XzOZ5EJgiCAiIioluTmDoCIiB5PTCBERGQUJhAiIjIKEwgRERmFCYSIiIzCBEJEREZhAiEiIqNYiDlJq9VCoVAYXcjKlSvx3XffIT09HXv37kXbtm3vW8bSpUtx9OhRyGQyTJkyBaNHjxZdhk6nQ3FxMSwtLSGTyYyOlYioKREEARqNBvb29pDLa9emEJVAevfujaFDhyIqKgqdO3eudYADBgzASy+9hBdeeOGB5+zduxc3b97EwYMHoVKpMGLECPTs2RM+Pj6iyiguLsalS5dqHRsREQFt27aFg4NDrZ4jKoFs2rQJe/bswbRp0+Dg4ICoqChERUXB09NTVCHBwcGPPGf//v0YPXo05HI5XFxcMHDgQBw4cAAxMTGiyrC0tARQWQlWVlaintOYpaamolOnTuYO475+TVXiaEo66nMJBFdHG7w8rCMU8pqtz4ZcF6bGuqjGuqhUXl6OS5cu6a+htSEqgXTs2BEdO3bE7NmzcezYMezZsweRkZHo0KEDoqKiEBERATs7u1oXfi+lUgkvLy/9756ensjMzBT9/KpuK7ZCqqWmppo7BAOCIOBY2l0c+u0OWjhaoJmt8d2i9ypR63DqfC78WmjQ0un+/wkaWl2YE+uiGuuimjFd/6ISSBW5XA4/Pz/4+vrit99+Q3Z2Nvbu3YtVq1Zh3rx5GDFiRK0DqG+dOnWCtbW1ucMwu+TkZAQFBZk7DD2dTsDmvak49Fs6QgN9MOv5QFha1M8YjmsZhYh9/zDsmvsgKLBml2dDqwtzYl1UY11UUqvVRidSUQmksLAQ3377LXbv3o2rV68iPDwcK1euRLdu3QAA586dw6RJk+qUQDw9PZGRkYGAgAAANVsk9PjSVOiw5ssUHEm5jeHP+mLSsE6Q36eryVg+bg6wUMhwNb0Qz94ngRCRNEQlkNDQUPTo0QPjxo3DwIEDa9xjCAgIwIABA+oUSHh4OBISEjBo0CCoVCr88MMP2L59e51ek8yvVF2BFZ+dwpmL2Xgpwh+j+j9d76PkLC3kaO3uiKvphfX6ukT0cKISyMGDB+Hm5lbjeE5ODlq2bAkAWLFixQOfv3TpUhw8eBC5ubl4+eWX4ezsjH379mHy5MmIjY1F586dERUVhd9++w2DBg0CAMyYMQOtWrUy5j1RA1FYpMbizSdw+ZYKsWO6IqzHE5KV5evthFPnMyEIAodxE5mIqAQSHh6OM2fO1Dg+dOhQJCUlPfL58+fPx/z582sc37hxo/6xQqHAokWLxIRDRipTV2DbgQvIVZWapLzLt1UouFOGeRO6o0cncSP2jPWktyN+OHUTBXfVcHG0kbQsIqokKoHcb8+poqIiftN7jNwpLsfizSfw580CeLs1AyD9v52jvRVe+3/d0NHXVfKyfL2cAABX0wuZQIhM5KEJJDQ0FDKZDGq1Gn379jX4m0qlwtChQ6WMjepJTkEp4jb+gsy8EswZH4KenRvf4IQn70kgwf7uZo6GqGl4aAJ57733IAgCpkyZglWrVumPy2QyuLq6wtfXV/IAqW5uZd3Fgo9/QYm6Aoum9ERnvxbmDkkS9raW8HC1w9UM3kgnMpWHJpDu3bsDAE6cOAFbW1uTBET158L1fCzefAIWCjlWzOit/5beWD3p5YRrHIlFZDIPTCAfffQRpk2bBgD45JNPHvgCs2bNqv+oqM4upZdiR8IvcHWyweIpPeHham/ukCTn6+2EE6lKlJRpYGdT+2UZiKh2HphA7l1GpDZLipBp3M6+i29/vQ6ttuYAh3KNFj+cyoOvtxMWxvSEs0PTmJnv6+0EQQCuK++gw5PS37gnauoemECqhtTqdDoMHz4cQUFBXKSwgbh4Ix+LNp1AWbkWNlb3/yds622DxdN6Nalv4lUjsa6lFzKBEJnAI4fxyuVyTJ8+HSkpKaaIhx7h9PksrNh6Ci4ONnh/Vk94trh/11RycnKTSh4A4OpkAwc7K1zNuGPuUIiaBFGr2YWEhODs2bMSh0KP8lPyLSzdchLeLZth5Su9H5g8miqZTAZfb0dcTVeZOxSiJkHUREIvLy9MnjwZAwYMgIeHh8EEQt5EN41vjlzG5j1/IOCpFnj75e5NrnUhlq+3MxKPXUWFVgcLBXdsJpKSqASiVqsxcOBAAEBWVpakAZEhQRDw2b40fP3TZfQK8MLr0d1gZVk/+2g0Rr5ejtBU6JCeXYQnPB3NHQ5RoyYqgSxfvlzqOOhvSso0+POWCgdP3sDPKekY8o82+Nc/A+674x5Ve9L7rxnpGYVMIEQSE72h1PXr15GYmIjs7Gy4ubkhMjISbdq0kTC0pkOnE3A7+y4u3ijAxZsFuHijADcz70AnADIZED2oHZ4f1I5rj4ng07IZrCzkuJpeiH5BXM2ZSEqiEsiPP/6IN998E/369YOXlxeuXbuG5557DqtWrarzPiBNnU4n4J2Pf8G5y7kAKpfkaNe6OXp29kS7J5qjbevmcLDj8GmxFAo5nvDk3iBEpiAqgXzwwQdYv349nnnmGf2xkydPYsmSJUwgdfTDqZs4dzkXY8PaIjTQB94tm9Xrbn1Nka+3E345l8G9QYgkJmqYSmZmJoKDgw2OBQUFcYZ6HRWVlOOzfWno8KQLXhjcHq3cHZg86sGTXk64W6JBrqrM3KEQNWqiEkj79u2xZcsWg2Px8fHw9/eXJKimYvuBCygqKce//hnAb8r1qHpvEJV5AyFq5ER1YS1cuBDTpk3D1q1b4enpCaVSCVtbW2zYsEHq+BqtaxmF2P/LNQz5x5Pw9W7cq+SaWhsvR8hkwNWMO5LvhEjUlIlKIH5+fti/fz/Onj2rH4XVpUsXWFpyMpsxBEHAhp3n0MzOCi+Etzd3OI2OrbUFvFrY4xr3BiGSlOhhvBYWFjXug5BxDp+5jbRr+Zg5uitHWEnkSS8n/HlLZe4wiBq1ByaQIUOG4NtvvwVQvbXt/VhaWsLHxwczZ85EUFCQNFE2IiVlGsTv/QNPt3JGWPfW5g6n0fL1dsKx3zJQVKoxdyhEjdYDE8iSJUv0j997770HvoBOp8OlS5cwe/Zs/PDDD/UbXSP0xcGLUBWpMX9iD464klDVfSV2YxFJ54EJ5N7uqqqtbR/kmWeewa1bt+ovqkbqZuYd7D16FWHdn0Db1s3NHU6jdu/eIN5ctJhIEqLvgZw/fx6nT59GQUEBBKF6F7yq1Xjffvvt+o+uEREEAZ988ztsrC3wUgSHP0utuaMNnB2scTWjEN5PmzsaosZJVAL53//+h+XLl6NXr174+eef8eyzz+L48eOchf43eYWlePujX6DWaGGhkMFCIa/8sZADgoDLtwsx9Z+d4dSsaWwxa26+Xk64ml6IPk9zUUUiKYhKIJs2bcKmTZsQHByMkJAQrFu3DkeOHMH+/fulju+x8ss5JdJzitC3mw8AQKPVQavVoUIroKJCh4h/tEF4zzbmDbIJ8fV2wjdHclChdTB3KESNkqgEkpeXp78nIpfLodPpEBoain//+9+SBve4OZWWCe+WzfDGCxyN1hD4ejmhQisg5w5HYhFJQdRSJh4eHrh9+zYAoE2bNjh06BBOnz7NiYT3KCnT4Pcreeje0cPcodBfnvSu7LrKLGACIZKCqBZITEwMrly5Ah8fH0yfPh2zZs2CRqPhjfN7nL2UgwqtDiEd3M0dCv3Fs0UzWFspcEVZhrRreTX+bqGQ4+lWzlyHjMhIohLIyJEj9Y9DQ0ORlJQEjUYDe3uOj6xyKi0L9raW8G/jYu5Q6C8KuQxPt3JG6pU8zP7w2H3PmTehO3p25npZRMYQPYwXAIqKilBcXGzwu7s7v3HrdAJOn89CUHs3WChE9QqSibz1YjC+/zkZTz9dcyyvpYUcHZ50NUNURI2DqATyyy+/4J133kFGRobBHBCZTIbz589LFtzj4s9bBVAVqRHSgfc/Gprmjjbw87RBYDs3c4dC1OiISiBvv/02pk+fjoiICNjY2Egd02PnVFoW5HIZgtrzIkVETYeoBKJWqzFy5EgoFAqjC7p27RrmzJkDlUoFZ2dnrFy5Em3atDE4Jy8vD3PnzoVSqURFRQV69OiB+fPnw8KiVj1tJpeUlgn/Ni5cWZeImhRRHfYTJkzApk2bDLqvaisuLg7R0dH47rvvEB0djQULFtQ4Z8OGDfDz88PevXuxZ88e/PHHHzh48KDRZZpCTkEprmXcQXeOviKiJkZUAhk0aBC++uorBAUFYcCAAQY/YuTl5SEtLQ2RkZEAgMjISKSlpSE/P9/gPJlMhuLiYuh0OpSXl0Oj0TT4m/SnzlfuC8/7H0TU1IjqG4qNjUVwcDDCw8ONugeiVCrh7u6u7wJTKBRwc3ODUqmEi0v1sNfp06fjlVdeQe/evVFaWooXXnihwe8xciotC56u9vBxa2buUIiITEpUArl9+za++eYbyOXSDlE9cOAA2rVrh88++wzFxcWYPHkyDhw4gPDwcNGvkZqaKmGEhsordEi5mIWQp5vhzJkzJitXrOTkZHOH0GCwLqqxLqqxLupGVAIZMGAATpw4gX/84x9GFeLp6YmsrCxotVooFApotVpkZ2fD09NwAte2bdvw7rvvQi6Xw8HBAf3798fJkydrlUA6deoEa2vTrHZ7MlUJrS4DkX0D0KVtS5OUKVZycnKDb72ZCuuiGuuiGuuiklqtNvqLt6gEUl5ejmnTpiE4OBiuroYTr1atWvXI57u6usLf3x+JiYmIiopCYmIi/P39DbqvAMDHxwc///wzAgICUF5ejl9//RVhYWG1eDumdep8FuxsLNDBl5PRiKjpEZVAnn766fvO5K2NhQsXYs6cOVi/fj0cHR2xcuVKAMDkyZMRGxuLzp07Y968eYiLi8OwYcOg1WrRo0cPjBkzpk7lSkUQBJxKy0RgOzdYWnD2ORE1PaISyMyZM+tckJ+fHxISEmoc37hxo/5x69atER8fX+eyTOHK7ULk31Fz+C4RNVn86mykU2mZkMmAoPZMIETUNDGBGCnpfBbaP+HC7WmJqMliAjFCXmEpLt9Sce8PImrSmECMcPp8NgCgO2efE1ETJiqBJCYm4sqVKwCAq1ev4oUXXsC4ceP0x5qaU2mZcGtui9YeDuYOhYjIbEQlkP/85z9wcnICUDnvIyAgAN27d8eiRYskDa4hSr6QhaS0TPTq4s2tUImoSRM1jDc/Px8tWrSAWq1GcnIy1q5dCwsLCzzzzDNSx9egKHOL8d62ZLTxdET04HbmDoeIyKxEJRAXFxfcuHEDly5dQufOnWFlZYXS0tI6Le/+uClVV2BZ/EnIZZX7aNtYNew9SoiIpCbqKjh9+nT9hlIffPABgMptbtu3by9pcA2FIAhY+78U3Mq6i4WTe8LD1d7cIRERmZ2oBDJy5EgMGTIEAGBrawsA6Nq1K1avXi1dZA3IrsNXcOy3DEwY2oF7axMR/UV0P0xV4hAEAYIgoHnz5pIF1ZCcvZSNz/b9gV5dvDCy31PmDoeIqMEQlUCysrKwePFinD59Gnfu3DH42/nz5yUJrCHIzCvGqs9Pw8fdAbPGBnLUFRHRPUQN442Li4OlpSU+/fRT2NnZYdeuXejfv3+jHsZbVl6B5Z+egk4A3n65O2ytedOciOheoq6KKSkp+Omnn2BnZweZTIb27dtj2bJleP755xvscut19fWPl3FNWYgFk56BVwtuV0tE9HeiWiByuRwWFpW5xtHREfn5+bCzs0NWVpakwZnTtYxCtHZ3QLA/17siIrofUS2QLl264MiRIwgLC0Pv3r3x6quvwsbGBp06dZI6PrPJLSxFC2dbc4dBRNRgiUogq1atgk6nAwDMmzcPW7ZsQXFxMcaPHy9pcOaUqyrFUz7O5g6DiKjBEpVAHB0d9Y9tbGwwffp0yQJqCMo1WhQWlaMlWyBERA/E5dzvI7ewFADYhUVE9BBMIPeRq2ICISJ6FCaQ+6hKIOzCIiJ6sFolEJ1Oh+zsbKliaTBy/kogrkwgREQPJCqB3LlzB2+88QYCAgIwaNAgAMChQ4f0K/M2NrmqMjjaW8HaUmHuUIiIGizRS5k0a9YMP/74IywtLQEAgYGB+PbbbyUNzlxyCkrQsjlbH0REDyNqGO+vv/6Ko0ePwtLSUr+goIuLC/Ly8iQNzlxyVaXc84OI6BFEtUAcHBxQUFBgcCwjIwMtW7aUJChzy1WV8gY6EdEjiEogo0ePRmxsLE6cOAGdToeUlBTMnj0bzz//vNTxmVxJmQbFZRUcwktE9AiiurAmT54Ma2trLF68GBUVFZg3bx7Gjh3bKJcy4RwQIiJxRCUQmUyG8ePHN8qE8Xe5qjIATCBERI8iepek9PR0XLhwASUlJQbHhw0bVu9BmVMOJxESEYkiKoF8/PHHWL9+Pfz8/GBjY6M/LpPJGl0CyVWVQiYDXJxsHn0yEVETJiqBbNmyBV9//TWeeuopqeMxu1xVKZo72MBCwVVeiIgeRtRV0tnZGd7e3lLH0iBwCC8RkTiiEsi8efPwzjvv4Pfff0dGRobBT2OToypBC85CJyJ6JFFdWBqNBsePH0diYqLBcZlMhvPnz4sq6Nq1a5gzZw5UKhWcnZ2xcuVKtGnTpsZ5+/fvx0cffQRBECCTyRAfH48WLVqIKqOuBEFAjqoMIR08TFIeEdHjTFQCWbRoEV5//XVEREQY3ESvjbi4OERHRyMqKgq7d+/GggULsHXrVoNzfv/9d3z44Yf47LPP0LJlS9y9exdWVlZGlWeMuyUalGu0HMJLRCSCqC4srVaLkSNHwt7eHgqFwuBHjLy8PKSlpSEyMhIAEBkZibS0NOTn5xuc9+mnn2LixIn6JVIcHBxgbW1dm/dTJ5xESEQknqgWyMSJE/HJJ59g6tSp+sUUa0OpVMLd3V2fcBQKBdzc3KBUKuHi4qI/78qVK/Dx8cELL7yAkpIShIWFYdq0abUqMzU1tdbxVbl4uzKB5CpvIFmjNPp1Gork5GRzh9BgsC6qsS6qsS7qRlQC+fzzz5Gbm4uPP/4Yzs7OBn87fPhwvQWj1Wpx8eJFxMfHo7y8HDExMfDy8sKIESNEv0anTp2MbrVkll0DkIc+PQPh4vh4zwNJTk5GUFCQucNoEFgX1VgX1VgXldRqtdFfvEUlkPfee8+oF6/i6emJrKwsaLVaKBQKaLVaZGdnw9PT0+A8Ly8vhIeHw8rKClZWVhgwYADOnTtXqwRSF7mqUlgoZHBuZrpuMyKix5WoBNK9e/c6FeLq6gp/f38kJiYiKioKiYmJ8Pf3N+i+AirvjRw5cgRRUVGoqKjAiRMnMHjw4DqVXRu5qlK4ONlCLq99Nx0RUVMjei2s8+fP4/Tp0ygoKIAgCPrjs2bNEvX8hQsXYs6cOVi/fj0cHR2xcuVKAJUr/cbGxqJz584YOnQoUlNTERERAblcjt69e2PUqFG1fEvGy+EkQiIi0UQlkP/9739Yvnw5evXqhZ9//hnPPvssjh8/jgEDBoguyM/PDwkJCTWOb9y4Uf9YLpdj7ty5mDt3rujXrU+5qlK0f8Ll0ScSEZG4YbybNm3Cpk2bsG7dOtjY2GDdunVYs2YNLCxEN2AaPJ1OQF5hKfdCJyISSVQCycvLQ3BwcOUT5HLodDqEhobip59+kjQ4U1IVqVGhFTgHhIhIJFFNCA8PD9y+fRs+Pj5o06YNDh06hObNm8PS0lLq+Ewml/uAEBHViqgEEhMTo5/kN336dMyaNQsajQZvv/221PGZTA5noRMR1YqoBDJy5Ej949DQUCQlJUGj0cDe3l6ywEyNy5gQEdWOqHsg+fn5KC4uBlA5W3zv3r34/vvvodPpJA3OlHJVpbCyVMDBrvF0yxERSUlUAvnXv/6FGzduAAA++OADbNmyBfHx8VixYoWkwdWX70/ewO3suw89p3IOiI1Ra30RETVFohLI9evX4e/vDwDYs2cPNm7ciM8++wz79++XNLj6oNUJ+G/CWXzx3cWHnperKmX3FRFRLYhKIHK5HBqNBhcvXoSDgwO8vLzg6Oio79ZqyErVFRAEIPliNiq0D+5yYwIhIqodUTfRn332WcyaNQsqlQoREREAgMuXL8Pd3V3S4OpDSZkGAFBcqsH5a/no/FTN3Q0rtDoU3CljAiEiqgVRCWTZsmXYtWsXLCws9CvjFhQU4JVXXpEytnpRUlahf5yUlnnfBJJfWAadALR0tjNlaEREjzVRCcTKygpjx441ONajRw9JAqpvxaWVLRAbKwVOpWVh0vBONc7J4SRCIqJaE3UP5HFW1YXVs7Mn0nOKkJFTVOOc6jkgj/cmUkREptToE0jxX11Yfbu1AgAkpWXVOIeTCImIaq/RJ5CqFkgbL0c84eGAU2mZNc7JVZXC3sYCdjacREhEJJZRCeTWrVu4fft2fcciiap7IHY2Fuje0QN/XM1D0V/HquRwCC8RUa2JSiCvv/46zpw5AwD4+uuvMXToUERGRt53g6iGpqSsAgq5DNaWCoT4e0CrE5ByMdvgnNxCJhAiotoSlUB+/fVXdOpUOXrp008/RXx8PBISEgx2E2yoiss0sLOxhEwmQ9snmsPR3gpJf+vG4iRCIqLaEzWMV6PRwMrKCllZWVCpVAgKCgIA5ObmShpcfSgprYC9beXbVMhlCPZ3x6m0TGh1AhRyGco1WhQWlXMILxFRLYlKIP7+/vj444+Rnp6Ovn37AgCysrLQrFkzKWOrF1UtkCohHdzx4+lbuHA9Hx19XZFbyBFYRETGENWFtWzZMly6dAlqtRqvvvoqACAlJQXDhg2TMrZ6UaqugP09CSSwrRsUcpl+NFZOwV+TCLkXOhFRrYhqgbRu3Rrvv/++wbHw8HCEh4dLElR9Ki7VwN2leokSe1tLdPJzxanzWZgQ2ZFzQIiIjCSqBSIIAr766iuMHz9e3+o4derUY7Gce0mZBva2hvM7Qjp44GbmXWTmFVcnECcmECKi2hCVQNasWYMdO3ZgzJgxUCqVAAAPDw9s2rRJ0uDqQ3FZBeysDRta3Tt4AABOpWUhR1UKp2ZWsLJUmCM8IqLHlqgEsmvXLmzYsAFDhw7V79jn4+ODW7duSRpcXQmCgNIyDez+1gLxbGEPH7dmSErL5BBeIiIjiUogWq0W9vb2AKBPIMXFxbCza9jLn5eqK6ATAHubmrd6Qjp4IPVKLm5lF7H7iojICKISSGhoKJYvX47y8nIAld/s16xZg379+kkaXF1V7QVyvzWuundwR4VWQHZ+CeeAEBEZQVQCmTt3LnJychAUFIS7d+8iMDAQGRkZePPNN6WOr06K/1pI0f4+CcS/jYv+5jq7sIiIak/UMN5mzZph3bp1yMvLQ3p6Ojw9PdGyZUupY6uzktK/WiC2Nd+mQiFHUHs3/JySzgRCRGQEUS2Qqm1sXV1dERAQoE8eI0eOlCyw+vCwFggAPNPRE0DlTXUiIqodUS2QGzdu1DgmCEKDX9K9VH8P5P5vs3dXLzR37IWnWzmbMCoiosbhoQnkrbfeAlC5mGLV4yrp6el46qmnpIusHuhbILb3b4HIZDJ08mthypCIiBqNhyaQ1q1b3/cxAHTr1q3BL2VStRshdxokIqp/D00gM2fOBAB06dIFffr0MUlA9am4rAJyGWBjxVnmRET1TdRN9D59+uDkyZOYO3cuJk2ahLlz5+LEiRO1KujatWsYO3YsBg8ejLFjx+L69esPPPfq1avo0qULVq5cWasy/q6kVAPbvzaTIiKi+iUqgSQkJODVV19Fy5YtERYWBjc3N7zxxhv46quvRBcUFxeH6OhofPfdd4iOjsaCBQvue55Wq0VcXBwGDhwo+rUfpLhMc99Z6EREVHeirq6bNm1CfHw82rdvrz82ZMgQxMbGYsyYMY98fl5eHtLS0hAfHw8AiIyMxJIlS5Cfnw8XFxeDcz/55BP07dsXJSUlKCkpqc17qaGkrIL3P4iIJCIqgahUKvj5+Rkc8/X1RWFhoahClEol3N3doVBU3otQKBRwc3ODUqk0SCAXLlzAsWPHsHXrVqxfv17sezCQmpqqf5yZkw9BAJKTk416rcddU33f98O6qMa6qMa6qBtRCaRbt25YsWIF3nzzTdja2qKkpASrV69GYGBgvQWi0WjwzjvvYPny5fpEY4xOnTrB2toaALD18GG4Otvo93BvSpKTk5vk+74f1kU11kU11kUltVpt8MW7NkQlkEWLFuG1115DcHAwnJycUFhYiMDAwBq7FD6Ip6cnsrKyoNVqoVAooNVqkZ2dDU9PT/05OTk5uHnzJqZMmQIAuHPnDgRBQFFREZYsWWLEW6u8B9LaxsGo5xIR0cOJSiBubm7Yvn07lEolcnJy4ObmBg8PD9GFuLq6wt/fH4mJiYiKikJiYiL8/f0Nuq+8vLxw8uRJ/e///e9/UVJSgtmzZ9fi7RgqKdM8cBY6ERHVjahRWMuWLcO5c+fg6emJgICAWiWPKgsXLsS2bdswePBgbNu2DYsWLQIATJ48Gb///nutX+9RBEFASVnFA2ehExFR3Yj6ei4IAqZPnw47OztERkYiMjISvr6+tSrIz88PCQkJNY5v3Ljxvue/8sortXr9v1NrtNDqBI7CIiKSiKgWyPz58/Hzzz8jLi4OSqUSY8eOxciRI/XDchuiqs2kOA+EiEgaohIIAMjlcvTq1QvLly9HYmIinJ2dsWrVKiljq5PiUq6DRUQkJdFfz0tKSvD9999j3759SEpKQkhICFasWCFlbHVSvZAiWyBERFIQdXWNjY3F0aNH0aFDBwwdOhQrVqyoMYO8oSl+yH7oRERUd6ISSOfOnTFnzhx4eXlJHU+9KXnEXiBERFQ3ohLI5MmTpY6j3hWXPnw3QiIiqhvRN9EfNyWP2A+diIjqptEmkOIyDWQywNaaLRAiIik8MIEcOnRI/1ij0ZgkmPpUUlYBW2sLyOXcTIqISAoPTCD//ve/9Y979OhhkmDqU+U6WOy+IiKSygP7d1q2bIlt27bBz88PWq0WJ06cgCAINc7r2bOnpAEaq6SsgrPQiYgk9MAr7PLly7F27Vps3boVGo0G8+bNq3GOTCYz6OpqSIpL2QIhIpLSAxNIt27d8OmnnwIAwsLC8P3335sqpnpRUqaBs4ONucMgImq0RI3CqkoeGRkZSElJgVKplDSo+lBcVgE7jsAiIpKMqCtsTk4OXnvtNZw9exbOzs5QqVTo0qULVq9eDXd3d6ljNEpJmQZ2nIVORCQZUS2QuLg4tG/fHklJSTh27BiSkpLg7++PuLg4qeMzWnEpb6ITEUlJ1BU2OTkZa9asgaVl5Td6Ozs7vPXWW+jTp4+kwRmrXKNFhVbHm+hERBIS1QJxcnLClStXDI5dvXoVjo6OkgRVV8X6ZUzYAiEikoqoK2xMTAwmTJiAUaNGwcvLCxkZGdi5cydmzZoldXxGqdqNkPdAiIikIyqBjBkzBq1atUJiYiIuXrwINzc3vP/++w12EmHVboRcSJGISDqi+3h69uzZYBPG35WWcSl3IiKpNcrVeIu5mRQRkeQaZQKp3g+dCYSISCqNMoFU7YfOUVhERNJplAmk5K+b6NxMiohIOqKusCqVClu2bMH58+dRUlJi8Lft27dLElhdFJdVwMZKAYWiUeZHIqIGQVQCeeONN1BeXo4hQ4bA1tZW6pjqjJtJERFJT1QCSUlJwYkTJ2BlZSV1PPWiuEwDe1t2XxERSUlUH0+7du2QmZkpdSz1pqS0gi0QIiKJifqa/swzzyAmJgYjR45EixYtDP42atQoSQKri+IyDRzsHo/WEhHR40pUAjl9+jTc3d1x/Phxg+MymaxBJpCSMg3cXezMHQYRUaMmKoF8/vnnUsdRr0rKKjgLnYhIYqLvNBcWFuKnn35CVlYW3N3d0a9fPzg5OUkZm9GKy3gPhIhIaqJuoqekpCAsLAxffvklLl68iC+//BJhYWFISUkRXdC1a9cwduxYDB48GGPHjsX169drnLNu3ToMHToUw4YNw8iRI3H06FHRr19Fq9WhXKPlLHQiIomJusq+++67iIuLw9ChQ/XH9u/fj6VLl+Lrr78WVVBcXByio6MRFRWF3bt3Y8GCBdi6davBOQEBAZg4cSJsbW1x4cIFvPjiizh27BhsbGxEv6ESdeUyJrZMIEREkhLVArl+/TqGDBlicGzw4MG4efOmqELy8vKQlpaGyMhIAEBkZCTS0tKQn59vcF6fPn30ExXbtWsHQRCgUqlElVGlVF21Dha7sIiIpCQqgTzxxBPYt2+fwbEDBw6gVatWogpRKpVwd3eHQqEAACgUCri5uUGpVD7wOd988w1at24NDw8PUWVUqUogvAdCRCQtUf088+bNw9SpU/H555/Dy8sL6enpuHHjBjZs2CBJUElJSVizZg22bNlS6+emnf8TAJB+6yqSyzPqO7THSnJysrlDaDBYF9VYF9VYF3UjKoF069YN33//PQ4fPozs7Gz069cPoaGhcHZ2FlWIp6cnsrKyoNVqoVAooNVqkZ2dDU9PzxrnpqSk4N///jfWr18PX1/fWr0ZAHD3agUgB10DOuIpH3HxNUbJyckICgoydxgNAuuiGuuiGuuiklqtRmpqqlHPFX2n2cnJCVFRUUYV4urqCn9/fyQmJiIqKgqJiYnw9/eHi4uLwXnnzp3Da6+9hrVr16Jjx45GlVXGeyBERCbxwAQyadIkbN68GQAQHR0NmUx23/PELue+cOFCzJkzB+vXr4ejoyNWrlwJAJg8eTJiY2PRuXNnLFq0CGVlZViwYIH+eatWrUK7du1Ev6ESddVuhByFRUQkpQdeZUeMGKF/PHr06DoX5Ofnh4SEhBrHN27cqH8sdkjww5SqtQC4HzoRkdQemECGDRumf+zr64suXbrUOOfcuXPSRFUHpeoKWFkqYMHNpIiIJCXqKvvyyy/f93hMTEy9BlMfytQVnIVORGQCD73S6nQ6CIJg8FPl5s2b+nkdDUkJ18EiIjKJhyaQDh06QCaTQRAEdOjQweBvcrkcU6dOlTQ4Y5SqK3gDnYjIBB56pT106BAEQcC4ceOwbds2/XGZTAYXF5darVFlKqXqCg7hJSIygYcmEG9vb2i1Wvj4+KBly5aPxZ7opeoKNHeyNncYRESN3iNvoisUCty+fRs6nc4U8dRZSRlbIEREpiBqFNaMGTOwcOFCpKenQ6vVQqfT6X8amjI1b6ITEZmCqLvN8+fPBwDs3r1bf0wQBMhkMpw/f16ayIyk5mZSREQmIepKe+jQIanjqFd2nIVORCQ5UQnE29sbQOW8kNzcXLRo0QJyecOd6c0WCBGR9ERlgaKiIrz11lsICAjAs88+i4CAAMyePRt3796VOj6j8B4IEZH0RCWQpUuXorS0FHv37sW5c+ewd+9elJaWYunSpVLHZxSOwiIikp6ovp6jR4/ihx9+0O9X/uSTT2L58uUICwuTNDhj2dmyC4uISGqiWiDW1tbIz883OFZQUNBgJxayC4uISHqivqqPGjUKEydOxIQJE+Dl5YWMjAx8+umnGDNmjNTxGYVrYRERSU/UlXbatGlwc3NDYmIisrOz4ebmhpiYGIwaNUrq+IzCeyBERNITlUBkMhlGjRrVYBPGvRRyOawsG94y80REjY3ovp4dO3Zg3759+hZIREQERo0a9cC90s3FzobJg4jIFEQlkFWrVuHQoUMYP348vL29kZ6eji1btuDatWt46623pI6xVmys2X1FRGQKohLIrl27sGvXLnh4eOiP9evXD//85z8bXAKxs2YLhIjIFEQN47W3t4e9vX2NY82aNZMkqLqwZQuEiMgkRLVAxo8fj5kzZ2LKlCnw8PCAUqnE5s2bMWHCBNy6dUt/XqtWrSQLVCxbDuElIjIJUVfbZcuWAQBOnjxpcPzXX3/VL2fSUJZ2t2UXFhGRSYhKIBcuXJA6jnpjxy4sIiKTqFV/T0ZGBrKysuDh4QFPT0+pYqoTG7ZAiIhMQlQCyc7Oxuuvv46zZ8/C2dkZKpUKXbp0werVq+Hu7i51jLXCm+hERKYhahTWwoUL0b59eyQlJeHYsWNISkqCv78/4uLipI6v1myteROdiMgURF1tk5OTsWbNGlhaVn67t7Ozw1tvvYU+ffpIGpwxOBOdiMg0RLVAnJyccOXKFYNjV69ehaOjoyRB1QVnohMRmYaoFkhMTAwmTJiAUaNG6Zdz37lzJ2bNmiV1fLXGmehERKYhKoGMGTMGrVq1QmJiIi5evAg3Nze8//776Nmzp9Tx1RpvohMRmcYjE4hWq8XgwYOxf//+Bpkw/o4z0YmITOOR90AUCgUUCgXUarUp4qkzjsIiIjINUTfRX3rpJbz66qtISkrCzZs3cevWLf2PWNeuXcPYsWMxePBgjB07FtevX69xjlarxaJFizBw4ECEhYUhISFB9OtXsbIQ9ZaIiKiORH1dX7JkCQDg+PHjBsdrs/5VXFwcoqOjERUVhd27d2PBggXYunWrwTl79+7FzZs3cfDgQahUKowYMQI9e/aEj4+PqDKqYiIiIumZZC2svLw8pKWlIT4+HgAQGRmJJUuWID8/Hy4uLvrz9u/fj9GjR0Mul8PFxQUDBw7EgQMHEBMTU6fyiYio/j00gZSWluKjjz7CpUuX0LFjR/zrX/+ClZVVrQtRKpVwd3eHQlE5xFahUMDNzQ1KpdIggSiVSnh5eel/9/T0RGZmZq3KSk1NrXV8jVVycrK5Q2gwWBfVWBfVWBd189AEsnjxYqSmpqJPnz747rvvoFKp8M4775gqtloRBAEA0LZtW6OSXGOTmpqKTp06mTuMBoF1UY11UY11Uam8vByXLl3SX0Nr46EJ5OjRo9i5cyfc3Nwwbtw4vPDCC0YlEE9PT2RlZUGr1UKhUECr1SI7O7vGir6enp7IyMhAQEAAgJotkofRaDQAgEuXLtU6vsaKrbFqrItqrItqrItqGo0GNjY2tXrOQxNISUkJ3NzcAFRe3IuKiowKzNXVFf7+/khMTERUVBQSExPh7+9v0H0FAOHh4UhISMCgQYOgUqnwww8/YPv27aLKsLe3R9u2bWFpackb6UREIgmCAI1GU2PbcjEemkC0Wi1OnDihb9pUVFQY/A5A9OTChQsXYs6cOVi/fj0cHR2xcuVKAMDkyZMRGxuLzp07IyoqCr/99hsGDRoEAJgxY4bobXLlcjkcHBxEnUtERNVq2/KoIhMe0vHVv3//hz9ZJsOhQ4eMKpiIiB5vD00gRERED8Jp20REZBQmECIiMgoTCBERGYUJhIiIjMIEQkRERmECISIiozSKBCJmr5HGauXKlejfvz/atWtnsIxLU6yTgoICTJ48GYMHD8awYcMwc+ZM5OfnAwDOnj2L4cOHY/DgwZg4cSLy8vLMHK30pk+fjuHDh2PEiBGIjo7Wb73QFD8bAPDhhx8a/D9pip8JoHJ+X3h4OKKiohAVFYWjR48CMLI+hEZg3LhxwjfffCMIgiB88803wrhx48wckemcOnVKyMjIEPr16ydcvHhRf7wp1klBQYFw4sQJ/e8rVqwQ5s6dK2i1WmHgwIHCqVOnBEEQhHXr1glz5swxV5gmc+fOHf3j77//XhgxYoQgCE3zs5GamipMmjRJ//+kqX4mBEGoca0QBMHo+njsWyBVe41ERkYCqNxrJC0tTf/Ns7ELDg6usShlU60TZ2dn9OjRQ/97165dkZGRgdTUVFhbWyM4OBgA8Pzzz+PAgQPmCtNk7l3ap6ioCDKZrEl+NsrLy7F48WIsXLhQf6ypfiYexNj6eOw3EBe710hTwjoBdDodvvjiC/Tv37/Gqs4uLi7Q6XRQqVRwdnY2X5Am8Pbbb+P48eMQBAGbNm1qkp+NNWvWYPjw4QY7mzblzwQAvPnmmxAEAUFBQXj99deNro/HvgVCdD9LliyBnZ0dXnzxRXOHYlbLli3D4cOH8dprr2HVqlXmDsfkUlJSkJqaiujoaHOH0mBs374de/bswddffw1BELB48WKjX+uxTyD37jUC4IF7jTQlTb1OVq5ciRs3buA///kP5HK5fp+ZKvn5+ZDL5U3im2aVESNG4OTJk/Dw8GhSn41Tp07hypUrGDBgAPr374/MzExMmjQJN27caLKfiap/aysrK0RHR+PMmTNG/x957BPIvXuNAHjgXiNNSVOuk9WrVyM1NRXr1q3T70zZqVMnlJWV4fTp0wCAL7/8EuHh4eYMU3LFxcVQKpX633/88Uc4OTk1uc/GlClTcOzYMfz444/48ccf4eHhgc2bNyMmJqbJfSaAyj2e7t69C6ByH5D9+/fD39/f6P8jjWI13itXrmDOnDm4c+eOfq8RX19fc4dlEkuXLsXBgweRm5uL5s2bw9nZGfv27WuSdfLnn38iMjISbdq00e9v4OPjg3Xr1uHMmTOIi4uDWq2Gt7c33nvvPbRo0cLMEUsnNzcX06dPR2lpKeRyOZycnDB79mx07NixSX42qvTv3x8bNmxA27Ztm9xnAgBu3bqFV155BVqtFjqdDn5+fpg/fz7c3NyMqo9GkUCIiMj0HvsuLCIiMg8mECIiMgoTCBERGYUJhIiIjMIEQkRERmECoUZpzpw5+OCDD8xStiAImDt3LkJCQjBq1CizxEBkCkwgZBL9+/dHz549UVJSoj+WkJCAcePGmTEqaSQnJ+P48eM4cuQIduzYYe5w6s24ceOQkJBg7jCoAWECIZPR6XTYunWrucOotaplP8RKT0+Ht7c37OzsJIpInIqKCrOWT40fEwiZzKRJk7BlyxbcuXOnxt9u376Ndu3aGVz07v3Gu3PnTjz//PN49913ERwcjAEDBuDMmTPYuXMnQkND0bNnT+zatcvgNQsKCvDyyy8jMDAQL774ItLT0/V/u3LlCl5++WV0794dgwcPxv79+/V/mzNnDuLi4jB58mR07doVJ0+erBFvVlYWpk6diu7duyMsLAxfffUVgMpW1fz583H27FkEBgZi7dq1NZ5b9V4WL16MoKAghIeH49dff9X//e7du5g3bx569+6NPn364IMPPjBIYjt27MCQIUMQEhKCSZMmGbyvdu3aYfv27Rg0aBAGDRpUo+yYmBhs27bN4Njw4cNx8OBBAMCZM2fw3HPPISgoCM899xzOnDkDAPjggw9w+vRpLF68GIGBgfoF+B5Wj0eOHEFERAQCAwPRp08fbN68uUY89Jirz41KiB6kX79+wvHjx4UZM2YIq1evFgRBEL766ivhxRdfFARBEG7duiW0bdtW0Gg0+ue8+OKLwldffSUIgiB8/fXXgr+/v7Bjxw6hoqJCWL16tRAaGiosXLhQUKvVwtGjR4WuXbsKRUVFgiAIwuzZs4WuXbsKSUlJglqtFpYsWSI8//zzgiAIQnFxsfDss88KO3bsEDQajfDHH38I3bt3F/7880/9c7t16yacPn1a0Gq1QllZWY33Ex0dLcTFxQllZWVCWlqa0KNHD+GXX37Rx1pV1v1UvZf4+HihvLxc2Ldvn9CtWzehoKBAEARBmD59uvDOO+8IxcXFQm5urvDcc88JX3zxhSAIlRtDDRw4ULh8+bKg0WiEdevWCWPHjtW/dtu2bYUJEyYIBQUFQmlpaY2yd+3aZXD+n3/+KQQFBQlqtVooKCgQgoODhV27dgkajUbYu3evEBwcLOTn59f49xBTj7169dJvUKRSqYTU1NQH1gk9ntgCIZOKjY3Ftm3bjNrAyMfHB8899xwUCgUiIiKgVCoxY8YMWFlZoXfv3rCyssLNmzf15/ft2xchISGwsrLCa6+9hrNnz0KpVOLw4cPw9vbGc889BwsLC3To0AGDBw822EBnwIABCAoKglwuh7W1tUEcSqUSZ86cwZtvvglra2v4+/tj9OjR2L17t+j34uLigvHjx8PS0hIRERF48skncfjwYeTm5uLIkSOYN28e7Ozs4OrqigkTJmDfvn0AKhe5mzJlCvz8/GBhYYGpU6fi/PnzBq2QKVOmwNnZWb8e2L0GDhyICxcu6M/fu3cvwsLCYGVlhcOHD+OJJ57AiBEjYGFhgcjISPj6+uKnn36673t4VD1aWFjg8uXLKCoqgpOTEzp27Ci6fujx8NhvKEWPl7Zt26Jv37745JNP4OfnV6vnurq66h9XXRzvXezN2toaxcXF+t89PDz0j+3t7eHk5ITs7Gykp6fj3Llz+t3XgMr7HMOHD9f//rDlzbOzs+Hk5IRmzZrpj3l5eSE1NVX0e3F3d4dMJjN4fnZ2NjIyMlBRUYHevXvr/6bT6fTxZGRk4N1338XKlSv1fxcEAVlZWfD29n5k7M2aNUNoaCj27duHKVOmIDExEUuXLtW/r3s3FaqKKysr676v9ah6XLt2LT766CO8//77aNeuHd544w0EBgaKqh96PDCBkMnFxsbin//8JyZOnKg/VnXDuaysTH9hzsnJqVM5mZmZ+sfFxcUoLCyEm5sbPD09ERISgvj4eKNe183NDYWFhSgqKtLHWrXTn1hZWVkQBEGfRJRKJfr37w8PDw9YWVnhxIkTsLCo+d/T09MTU6dONUh2f3dvYrqfyMhIfPjhhwgJCYFardZvA+zm5mawJ0RVXH369Lnv6zyqHgMCAvDRRx9Bo9Fg+/btePXVV3HkyJGHxkaPF3Zhkck98cQTiIiIwOeff64/5uLiAnd3d+zevRtarRY7duzArVu36lTOkSNHcPr0aZSXl2PNmjXo0qULPD090bdvX1y/fh3ffPMNNBoNNBoNzp07hytXroh6XU9PTwQGBmL16tVQq9W4cOECduzY8dCL+t/l5+dj69at0Gg0+Pbbb3HlyhWEhobCzc0NvXr1wooVK1BUVASdToebN28iKSkJQOVe1Z988gn+/PNPAJU33L/99tta1UtoaCgyMjKwdu1aREREQC6X649fv34de/fuRUVFBfbv34/Lly+jb9++ACpbe/f+mzysHsvLy7Fnzx7cvXsXlpaWsLe315dDjQf/RcksZsyYYTAnBKjchnbz5s3o0aMHLl++XOfujsjISKxbtw49evTAH3/8gffeew9AZTfO5s2bsX//fvTp0we9e/fG//3f/6G8vFz0a69evRrp6eno06cPZs6ciVdeeQX/+Mc/RD8/ICAAN27cwDPPPIP//Oc/WLt2LZo3bw4AWLVqFTQaDSIiIhASEoLY2Fh9aywsLAwxMTF4/fXX0a1bN0RGRuLnn3+uRa1U7kQXFhaGX375BZGRkfrjzZs3x4YNGxAfH48ePXpg06ZN2LBhg36zqZdeegnfffcdQkJCsHTp0kfW4+7du9G/f39069YNX375pb7+qfHgfiBEJrZz504kJCTgiy++MHcoRHXCFggRERmFCYSIiIzCLiwiIjIKWyBERGQUJhAiIjIKEwgRERmFCYSIiIzCBEJEREb5/3cKAuHaqfkXAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "debug(\"Same as majority rate\")\n",
    "ax = sns.lineplot(data=peer_votes_num_acc_df_part, x=peer_votes_num_acc_df_part.index, y='Same as majority rate')\n",
    "# sns.lineplot(data=peer_votes_num_acc_df_part, x=peer_votes_num_acc_df_part.index, y='Total scaled')\n",
    "ax.set_ylim([-0.01, 1.01])\n",
    "ax.set_xlim([0, xlim])\n",
    "ax.set_ylabel(\"Proportion of votes same as majority\")\n",
    "plt.xlabel(\"Number of peer votes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     0.704521\n",
       "1     0.705271\n",
       "2     0.714322\n",
       "3     0.732842\n",
       "4     0.745499\n",
       "5     0.752187\n",
       "6     0.730781\n",
       "7     0.734224\n",
       "8     0.811898\n",
       "9     0.824407\n",
       "10    0.831898\n",
       "11    0.840483\n",
       "12    0.848285\n",
       "13    0.853363\n",
       "14    0.859117\n",
       "15    0.863464\n",
       "16    0.834637\n",
       "17    0.845748\n",
       "18    0.876182\n",
       "19    0.876035\n",
       "20    0.912417\n",
       "21    0.917482\n",
       "Name: Confidence, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "peer_votes_num_acc_df_part[\"Confidence\"]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.12 ('cr4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "740471d2bc5e6e0b41d12bdc2e64373746aa6a34800f381ff958ff5f02fa0c53"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}