nbs/object-detection-metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "01ddcbcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "sys.path.insert(0, os.path.abspath('../'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1f22bb98",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from objdetecteval.metrics import image_metrics as im, coco_metrics as cm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58d50d4c",
   "metadata": {},
   "source": [
    "## Load Predictions and Labels in CSV format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cb409351",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_path = '../data/predictions.csv'\n",
    "labels_path = preds_path\n",
    "\n",
    "preds_df = pd.read_csv(preds_path)\n",
    "labels_df = pd.read_csv(labels_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4b3d49de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>xmin</th>\n",
       "      <th>ymin</th>\n",
       "      <th>xmax</th>\n",
       "      <th>ymax</th>\n",
       "      <th>label</th>\n",
       "      <th>score</th>\n",
       "      <th>image_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>686</td>\n",
       "      <td>295</td>\n",
       "      <td>854</td>\n",
       "      <td>451</td>\n",
       "      <td>Car</td>\n",
       "      <td>0.707925</td>\n",
       "      <td>000000001.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>289</td>\n",
       "      <td>1039</td>\n",
       "      <td>475</td>\n",
       "      <td>1212</td>\n",
       "      <td>Car</td>\n",
       "      <td>0.679780</td>\n",
       "      <td>000000001.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>971</td>\n",
       "      <td>677</td>\n",
       "      <td>1192</td>\n",
       "      <td>902</td>\n",
       "      <td>Truck</td>\n",
       "      <td>0.664441</td>\n",
       "      <td>000000001.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1294</td>\n",
       "      <td>1222</td>\n",
       "      <td>1497</td>\n",
       "      <td>1408</td>\n",
       "      <td>Car</td>\n",
       "      <td>0.645636</td>\n",
       "      <td>000000002.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>686</td>\n",
       "      <td>295</td>\n",
       "      <td>854</td>\n",
       "      <td>451</td>\n",
       "      <td>Truck</td>\n",
       "      <td>0.707925</td>\n",
       "      <td>000000002.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  xmin  ymin  xmax  ymax  label     score     image_name\n",
       "0   1   686   295   854   451    Car  0.707925  000000001.jpg\n",
       "1   2   289  1039   475  1212    Car  0.679780  000000001.jpg\n",
       "2   3   971   677  1192   902  Truck  0.664441  000000001.jpg\n",
       "3   4  1294  1222  1497  1408    Car  0.645636  000000002.jpg\n",
       "4   5   686   295   854   451  Truck  0.707925  000000002.jpg"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0a78755",
   "metadata": {},
   "source": [
    "## Calc inference dataframe and output metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "479404f7",
   "metadata": {},
   "source": [
    "### Calculate inference dataframe - one row for each bounding box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "745ee952",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>class</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>Confidence</th>\n",
       "      <th>IoU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000000001.jpg</td>\n",
       "      <td>Car</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.707925</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000000001.jpg</td>\n",
       "      <td>Car</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.679780</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000000001.jpg</td>\n",
       "      <td>Truck</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.664441</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000000002.jpg</td>\n",
       "      <td>Car</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.645636</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000000002.jpg</td>\n",
       "      <td>Truck</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.707925</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        image_id  class  TP  TN  FP  FN  Confidence  IoU\n",
       "0  000000001.jpg    Car   1   0   0   0    0.707925  1.0\n",
       "1  000000001.jpg    Car   1   0   0   0    0.679780  1.0\n",
       "2  000000001.jpg  Truck   1   0   0   0    0.664441  1.0\n",
       "3  000000002.jpg    Car   1   0   0   0    0.645636  1.0\n",
       "4  000000002.jpg  Truck   1   0   0   0    0.707925  1.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "infer_df = im.get_inference_metrics_from_df(preds_df, labels_df)\n",
    "infer_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "756977b0",
   "metadata": {},
   "source": [
    "### Summarise inference dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c18a4ab7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>TP</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>Total</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Car</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Truck</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Van</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   class  TP  FP  FN  Total  Precision  Recall\n",
       "0    Car   3   0   0      3        1.0     1.0\n",
       "1  Truck   4   0   0      4        1.0     1.0\n",
       "2    Van   1   0   0      1        1.0     1.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_summary_df = im.summarise_inference_metrics(infer_df)\n",
    "class_summary_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a215fc6",
   "metadata": {},
   "source": [
    "## Plot from the summary "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "febf9ec5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEvCAYAAADSG9NhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXp0lEQVR4nO3df7RdZX3n8feHGAiKIsRUZxFDwmAVWJQocZUZBSJWkAqhLlGXomMFjbWtM2qZFs3oAIWF1p9rOnU0jrW0tGr9wWBkrKIQxErUoIJExeIQJQo1JlHkp0S/88c51z5c7s3d595zf4S8X2vtde559vPs5znnj8999o+zd6oKSVLPXrM9AEmaSwxFSWoYipLUMBQlqWEoSlLDUJSkxsNmewATecxjHlNLly6d7WFIeoi57rrrflJVi0aXz/lQXLp0KRs3bpztYUh6iEny/bHK3X2WpIahKEkNQ1GSGnP+mKKk2XP//fezZcsW7r333tkeyqQtWLCAxYsXM3/+/E71DUVJ49qyZQuPfOQjWbp0KUlmezgDqyq2bdvGli1bWLZsWac2k9p9TvJPSSrJBR3qLkjytiS3JbknybVJjptMv5Jm1r333svChQt3y0AESMLChQsHmukOHIpJXgQcNUCTDwCvBN4MnALcBnwmyfJB+5Y083bXQBwx6PgH2n1OcgDwLuB1wD90qH8U8GLgzKr6YL/samATcD6waqDRStqjbNu2jWc+85kA3H777cybN49Fi3rXW19//fUcddRR7Ny5k8MOO4yLL76Yhz/84VPuc9Bjim8FbqyqDyWZMBTphd79wEdGCqpqZ5IPA+ck2aeq7htwDJJmydJzLh/q9ja/5Tm7XL9w4UK+8Y1vAHDuueey3377cfbZZwOw3377/XrdGWecwXvf+15e//rXT3lMnXefkzwd+E/AHw2w/SOAW6rq7lHlm4C9gUMH2JYkjenYY4/l5ptvHsq2OoVikr2B9wFvr6qbBtj+gcCOMcq3N+sladJ27tzJpz/9aY488sihbK/r7vOfAvsCFw6l1wkkWQ2sBliyZMlMdKkOhr3rtCsT7VZJ99xzD8uXLwd6M8WzzjprKNudMBSTLAHWAK8A9kmyT7N6nySPBn5eVb8co/kO4OAxykdmiNvHWEdVrQXWAqxYscIna0l6kH333ffXxxSHqcvu8yHAAuASeiE3sgCc3f97vHnrJmBZktGnhA4HfgEM5yCAJA1Jl1D8BvCMMRboBeUzGD/c1gHzgeePFCR5GPBC4LOeeZY010y4+1xVPwXWjy7vXxD5/apa339/MPA94PyqOr/f9utJPgK8O8l84Bbg1cAy4IyhfAJJM2Y2j/Wee+65D3h/5513Tks/w/ztc4B5PHj2+XJ6J2guAB4NXA88u6q+NsS+JWkoJh2KVZVR7zfTC8bR9e4BXt9fJGlO836KktQwFCWpYShKUsNQlKSGoShpTps3bx7Lly//9bJ582bWr1/P/vvvz/LlyznssMM477zzhtafjyOQ1N25+w95ez+bsMpYP+fbvHkzxx57LJ/61Ke46667WL58OaeeeipPecpTpjwkZ4qSdmuPeMQjOProo2f21mGSNFtG7oazfPlynvvc5z5o/bZt29iwYQNHHHHEUPpz91nSnDbe3XCuueYanvzkJ7PXXntxzjnnGIqS9mwjxxSHzd1nSWoYipLUcPdZUncdLqEZtrFuEbZy5UpWrlw5Lf05U5SkhqEoSQ1DUZIahqIkNQxFSWoYipLUMBQlzWnj3TosCevWrft1vVNOOYX169dPub9O1ykmOQn4M3oPsT8A2Ap8CTi3qr61i3ZL6T3WdCwH9B+fKmk3ceTFRw51e9982TcnrDPercMWL17MhRdeyKmnnjrUMXWdKR4IXAf8MXAi8AbgCGBD/3nPE7kI+A+jlp8PPFpJ6jvqqKPYf//9ueKKK4a63U4zxar6EPChtizJV4DvAKcD75hgE/+vqjZMaoSS9mgjtw4DWLZsGZdeeumv161Zs4Y3velNPOtZzxpaf1P5md+2/uvOYQxEksYy3q3DAI477jgAvvjFLw6tv4FOtCSZl2TvJE8A3gfczqgZ5DguSrIzyc+SfDLJcA9MSNpjrVmzhgsuuGBo2xv07POXgfuA7wK/BZxQVT/eRf376IXnq4BnAGcDRwJfSnLYeI2SrE6yMcnGrVu3DjhESXuSE088kR07dnDDDTcMZXuDhuJLgWOAFwN3AFf0zzCPqapuq6o/qKpPVNU1VfV+4DiggDW7aLe2qlZU1YpFixYNOERJe5o1a9Zw6623DmVbAx1TrKpv9//8cpJPA5uBc4A/GGAbtyb5IvDUQfqWNPu6XEIzbF1uHbZq1Sqqaij9Tfri7f41hjcDh052E5PtW5Kmy6RDMcljgScB3xuw3RLg6cBXJtu3JE2Xrr9ouRT4GnADvWOJvwm8jt7lOO/o1zke+DxwZlX9bb/sHfSC91p6v4J5Ir0Lv38FXDjMDyJJw9D1mOIG4AXAnwB7A7cC64GLqmpzv06AeTxw9rkJeDXw+8B+9K5tvBI4r6pumtrQJc2EqiLJbA9j0gY91tj1Fy1vBd46QZ319IKxLftr4K8HGpGkOWPBggVs27aNhQsX7pbBWFVs27aNBQsWdG7jg6skjWvx4sVs2bKF3fl64QULFrB48eLO9Q1FSeOaP38+y5Ytm+1hzCjvpyhJDUNRkhqGoiQ1DEVJahiKktQwFCWpYShKUsNQlKSGoShJDUNRkhqGoiQ1DEVJahiKktQwFCWpYShKUsNQlKSGoShJjU6hmOSkJFcmuT3JfUm2JPnHJId3aHtAkv+d5CdJ7kryuSRHTn3okjR8XWeKBwLXAX8MnEjvMaVHABuSHDxeo/SedLMOeDbwGuB5wHzgqiTdH5ogSTOk69P8PgR8qC1L8hXgO8Dp9J/9PIZVwNOAE6rqqn67a4FbgD8F/vPkhi1J02MqxxS39V937qLOKuBHI4EIUFU/ozd7PG0KfUvStBgoFJPMS7J3kicA7wNuZ9QMcpQjgBvHKN8ELEmy3yD9S9J0G/QRp18Gju7/fTO93eIf76L+gcDmMcq3918PAO4cvTLJamA1wJIlSwYc4q4tPefyoW5vPJvf8pwZ6UfScA26+/xS4BjgxcAdwBVJlg57UFW1tqpWVNWKRYsWDXvzkjSugUKxqr5dVV/un3h5JrAfcM4umuygNxsc7cBmvSTNGZM+0VJVP6W3C33oLqptondccbTDgR9U1YN2nSVpNk06FJM8FngS8L1dVPskcFCS45t2jwJO7a+TpDml04mWJJcCXwNuoHcs8TeB19G7HOcd/TrHA58Hzqyqv+03/SRwLXBJkv9Kb3f5DUCAvxjex5Ck4eh69nkD8ALgT4C9gVuB9cBFVbW5XyfAPJrZZ1X9KskpwNuB9wAL6IXkM6rq1iGMX5KGqusvWt4KvHWCOuvpBePo8u3Amf1FkuY075IjSQ1DUZIahqIkNQxFSWoYipLUMBQlqWEoSlLDUJSkhqEoSQ1DUZIahqIkNQxFSWoYipLUMBQlqWEoSlLDUJSkhqEoSQ1DUZIahqIkNSYMxSSnJ/l4ku8nuSfJTUkuSvLIDm1rnGX5UEYvSUPW5cFVZwM/AN4IbAGeDJwLPCPJf6yqX03Q/m+A940q++5gw5SkmdElFE+tqq3N+6uTbAcuBlYCV07Q/odVtWGS45OkGTXh7vOoQBzx1f7rQcMdjiTNrsmeaDm+//rtDnVfneS+JHcnuTLJsZPsU5Km3cChmOQg4Hzgc1W1cYLqlwB/CPwOsBpYCFyZZOWg/UrSTOhyTPHXkuwHXAbsBF4+Uf2qemnz9poklwE3AhcAT99FP6vphShLliwZZIiSNCWdZ4pJ9gXWAYcAJ1XVlkE7q6qfA5cDT52g3tqqWlFVKxYtWjRoN5I0aZ1miknmAx8DVgDPqqpvTrHfmmJ7SZoWE4Zikr2AvwdOAE6ZyuU1SR4FnAJ8ZbLbkKTp1GWm+FfA84ELgbuSHNOs21JVW5IcDHwPOL+qzgdIcjbwROAq4EfAwfQuBH8ccMbwPoIkDU+XUDy5/7qmv7TOo/frlgDzeOAxypuA5/aX/YE7gH8GzqoqZ4qS5qQJQ7Gqlnaos5leMLZl6+idmJGk3YZ3yZGkhqEoSQ1DUZIahqIkNQxFSWoYipLUMBQlqWEoSlLDUJSkhqEoSQ1DUZIahqIkNQxFSWoYipLUMBQlqWEoSlLDUJSkhqEoSQ1DUZIahqIkNSYMxSSnJ/l4ku8nuSfJTUkuSvLIDm0XJHlbktv6ba9Nctxwhi5Jw9dlpng28EvgjcCzgf8FvBq4IslE7T8AvBJ4M3AKcBvwmSTLJztgSZpOXZ77fGpVbW3eX51kO3AxsBK4cqxGSY4CXgycWVUf7JddDWwCzgdWTWHckjQtJpwpjgrEEV/tvx60i6argPuBjzTb2gl8GDgpyT4DjFOSZsRkT7Qc33/99i7qHAHcUlV3jyrfBOwNHDrJviVp2gwcikkOorf7+7mq2riLqgcCO8Yo396sH6+P1Uk2Jtm4detYE1VJmh4DhWKS/YDLgJ3Ay6dlREBVra2qFVW1YtGiRdPVjSQ9SOdQTLIvsA44BDipqrZM0GQHcMAY5SMzxO1jrJOkWdUpFJPMBz4GrAB+t6q+2aHZJmBZkoePKj8c+AVw8yADlaSZ0OXi7b2AvwdOAH6vqjZ03PY6YD7w/GZbDwNeCHy2qu4bfLiSNL26XKf4V/SC7ULgriTHNOu2VNWWJAcD3wPOr6rzAarq60k+Ary7P9O8hd5F38uAM4b5ISRpWLrsPp/cf10DXDtqeUV/XYB5Y2zv5cAHgQuAy4HHA8+uqq9NbdiSND0mnClW1dIOdTbTC8bR5fcAr+8vkjTneZccSWoYipLUMBQlqWEoSlLDUJSkhqEoSQ1DUZIahqIkNQxFSWoYipLUMBQlqWEoSlLDUJSkhqEoSQ1DUZIahqIkNQxFSWoYipLUMBQlqdH1uc+Lk/xlkmuT3J2kkizt2HZzv/7o5femMnBJmg5dHnEKcCjwAuA64BrgxAH7+Qxw7qiymwbchiRNu66h+IWqeixAklcweCj+pKo2DNhGkmZcp93nqvrVdA9EkuaCmTrRcmr/WOR9STZ4PFHSXDUTobgOeA1wEnAGcC9waZKXjNcgyeokG5Ns3Lp16wwMUZJ6uh5TnLSqek37PsmlwAbgIuCScdqsBdYCrFixoqZ7jJI0YsavU6yqXwIfBRYn+Xcz3b8k7cpsX7ztLFDSnDLjoZjkYcALgR9U1e0z3b8k7UrnY4pJTu//eXT/9eQkW4GtVXV1v85O4OKqOqv//kXAacD/BW4FHgv8EfAU4EVD+QSSNESDnGj56Kj37+m/Xg2s7P89r7+MuAX4DeBtwIHAXcBG4NlV9ZlBBytJ061zKFZVBq3T/xXLCZMYlyTNitk+0SJJc4qhKEkNQ1GSGoaiJDUMRUlqGIqS1DAUJalhKEpSw1CUpIahKEkNQ1GSGoaiJDUMRUlqGIqS1DAUJalhKEpSw1CUpIahKEkNQ1GSGp1CMcniJH+Z5NokdyepJEs7tt0ryRuSbE5yb5LrkzxvSqOWpGnSdaZ4KPACYAdwzYB9/DlwLvA/gZOBDcBHk/zugNuRpGnX9Wl+X6iqxwIkeQVwYpdGSX4DOBt4S1W9vV98VZJDgbfQex60JM0ZnWaKVfWrSW7/JGBv4JJR5ZcARyZZNsntStK0mO4TLUcA9wE3jyrf1H89fJr7l6SBTHcoHgj8tKpqVPn2Zr0kzRldjynOqCSrgdUAS5YsmeXRSMOx9JzLZ6yvzW95zoz19VAz3TPFHcCjk2RU+cgMcTtjqKq1VbWiqlYsWrRoWgcoSa3pDsVNwD7Avx9VPnIs8VvT3L8kDWS6Q/GfgPuBM0aVvwS4sapumeb+JWkgnY8pJjm9/+fR/deTk2wFtlbV1f06O4GLq+osgKr6cZJ3Am9I8nPga8ALgROAVUP6DJI0NIOcaPnoqPfv6b9eDazs/z2vv7TWAHcC/wV4HHAT8IKq+tRAI5WkGdA5FKtq9MmSTnWq6pfABf1FkuY075IjSQ1DUZIahqIkNQxFSWoYipLUMBQlqWEoSlLDUJSkhqEoSQ1DUZIahqIkNQxFSWoYipLUMBQlqWEoSlLDUJSkhqEoSQ1DUZIahqIkNQxFSWp0CsUkj0/ysSQ/S3JHkk8kWdKxbY2zLJ/SyCVpGkz4NL8kDweuBO4DXgYUvSfzXZXkt6rqrg79/A3wvlFl3x1sqJI0/bo84vSVwCHAE6vqZoAkNwD/ArwKeGeHbfywqjZMepSSNEO67D6vAjaMBCJAVd0C/DNw2nQNTJJmQ5dQPAK4cYzyTcDhHft5dZL7ktyd5Mokx3YeoSTNoC6heCCwY4zy7cABHdpfAvwh8DvAamAhcGWSleM1SLI6ycYkG7du3dqhC0kaji7HFKekql7avL0myWX0Zp4XAE8fp81aYC3AihUrarrHKEkjuswUdzD2jHC8GeQuVdXPgcuBpw7aVpKmW5dQ3ETvuOJohwPfmkLfzgAlzTldQvGTwDFJDhkpSLIUeFp/3UCSPAo4BfjKoG0labp1CcX3A5uBy5KclmQVcBlwK80F2UkOTrIzyZubsrOTvD/Ji5OsTPIyepfyPA5YM8wPIknDMOGJlqq6K8kJwLuAvwMCfB54bVXd2VQNMI8HBu1NwHP7y/7AHfRC8ayqcqYoac7pdPa5qn4APG+COpvpBWNbtg5YN9nBSdJM8y45ktQwFCWpYShKUsNQlKSGoShJDUNRkhqGoiQ1DEVJahiKktQwFCWpYShKUsNQlKSGoShJDUNRkhqGoiQ1DEVJahiKktQwFCWpYShKUqNTKCZ5fJKPJflZkjuSfCLJko5tFyR5W5LbktyT5Nokx01t2JI0PSYMxSQPB64EngS8DHgp8ATgqiSP6NDHB4BXAm+m97zn24DPJFk+yTFL0rTp8jS/VwKHAE+sqpsBktwA/AvwKuCd4zVMchTwYuDMqvpgv+xqYBNwPrBqSqOXpCHrsvu8CtgwEogAVXULvec3n9ah7f3AR5q2O4EPAycl2WfgEUvSNOoSikcAN45Rvgk4vEPbW6rq7jHa7g0c2qF/SZoxXULxQGDHGOXbgQOm0HZkvSTNGV2OKc64JKuB1f23dya5aTbHMxl566SaPQb4yXBHsnvy+5sav79ODh6rsEso7mDsGeF4s8DRbcfqeGSGuH2MdVTVWmBth7E9pCTZWFUrZnscuyu/v6nx++vpsvu8id6xwdEOB77Voe2y/mU9o9v+Arj5wU0kafZ0CcVPAsckOWSkIMlS4Gn9dbuyDpgPPL9p+zDghcBnq+q+QQcsSdOpSyi+H9gMXJbktCSrgMuAW4H3jVRKcnCSnUnePFJWVV+ndznOu5O8Iskz6V2Oswz478P7GA8Ze9whgyHz+5savz8gVTVxpd5P+t4FPAsI8HngtVW1uamzFLgFOK+qzm3K9wUupHcR96OB64E/q6r1w/kIkjQ8nUJRkvYU3iVHkhqGoiQ1DEVJahiKsyDJvklem+SqJP+a5Bf95V/7Za8d49pODSDJcUmunO1xzFVJViY5I8lTxll/UHslyZ7EEy0zLMnj6d2fcim9Ow1t4oG/BT+c3jWg3weeWVU/mIVh7vaSPA/4x6qaN9tjmUuS7Ad8FvhteleSFHAFvdv7/aip99vAl/bE729O/vb5Ie7dwD3AE9pLmlr9y5v+D73LoJ43Q+PaLXS94zuwaFoHsvt6I3AY8PvAV4GVwHnAl5OcVFUT/UrtIc+Z4gxL8jPgJVW1boJ6q4C/q6r9Z2Zku4ckv6I3u5mwKlB74kxnV5J8B3hPVf2Ppuwgej/IWAqcXFVfdaaomTTIfyH/Yz3YPcAXgI9NUG8F/3anJf2bJcDX24Kq+mGS44FPAZ9Lchq973mPZCjOvM8BFya5sX8H8wfp7z7/Ob1jPXqg64FfVtUHdlUpyU8xFMfyY2Dx6MKquivJycDHgcuBd8z0wOYKQ3HmvRa4Cvhukg307mo+cgu2A+jdkegYer83f90sjG+uuw44vWPdTOdAdlMb6T1G5EOjV1TVvf1Z4j8A/409dE/FY4qzoP978NXAqfRCcOT+kjvonY3+JPD+MR7jsMfrH/86tKqunu2x7I76Z+XPBk6pqm3j1AnwHuDZVbVsJsc3FxiKktTw4m1JahiKktQwFCWpYShKUsNQlKTG/wfVSwatuf/UUQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEvCAYAAADSG9NhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXbUlEQVR4nO3de7hWZZ3/8fc3MAEzBKN+v1LY8JOMjaLpVpkIwVOiGYx5ajTD8YBezTQ6jgcaGxK1RidPqRXimAe0MhkNLW0MAS0VFU0dUBIaUMkOCBuUo4D374/n2XSz2exn7c0+Ie/Xde3r2dzrvtf67uePD/e613rWEyklJEklH2jvAiSpIzEUJSljKEpSxlCUpIyhKEkZQ1GSMp3bu4BKPvKRj6Sqqqr2LkPS+8xzzz33VkqpV/32Dh+KVVVVzJo1q73LkPQ+ExGvNdTu6bMkZQxFScoYipKU6fBritL2bt26dSxatIg1a9a0dynbpC5durDbbruxww47FOpvKEod3KJFi9h5552pqqoiItq7nG1KSoklS5awaNEi+vbtW2hModPniNgtIm6MiKciYlVEpIioKjj2AxHx9YhYGBFrIuLFiDiuUHWSWLNmDbvuuquB2AwRwa677tqkWXbRNcU9gBOBWuDXTazrcuBS4CbgKGAmcG9EHN3E/UjbLQOx+Zr63hUNxcdTSh9LKR0N3NuEYj4KXABcmVK6OqU0PaV0NjAduLJJlUpqN506dWLfffdlr7324oQTTmDVqlVbvc9x48YxderULW6fMGECd95551Yfp6kKrSmmlN5r5v6PBD4I3FWv/S7ghxHRN6W0oJn7lrZLVWN/0aL7W3jl5yv26dq1Ky+88AIAp5xyChMmTOD888/fuH39+vV07ty0SxSXXXZZo9vPOeecJu2vpbT2LTkDgbXA/Hrtc8qv1a18fEktbOjQocyfP58ZM2YwdOhQRo4cSXV1NRs2bODCCy/kgAMOYNCgQdx8880bx1x11VXsvffe7LPPPowdOxaA0047jcmTJwMwduxYqqurGTRoEBdccAEAl156KVdffTUAL7zwAoMHD2bQoEEce+yx1NbWAjB8+HAuvvhiDjzwQD75yU/y6183dXVvc6199bknsCxt/p0HS7PtkrYR69ev5+GHH2bEiBEAPP/888yePZu+ffsyceJEunfvzrPPPsvatWsZMmQIn/vc55g7dy5Tpkzh6aefplu3bixdunSTfS5ZsoT777+fuXPnEhEsW7Zss+N+5Stf4cYbb2TYsGGMGzeO8ePHc/3112+s6ZlnnuGhhx5i/PjxjZ6SF9Ehb8mJiDHAGIDevXu36L5b+tSjJS3scnJ7l9CoqjU/au8SGvV+ff9uGfl/WbdoWcsWk3mpvO9BH9jyStbq1avZd+CeAAw96NOccdT+PDnrRQ7cp5q+Oy6DN3/LI1N+ykuvzGPyjycBsPydFcyb+TBTZzzF3x97GN2W/Q6WlWdCb74Gq5bC0gV0X9mHLp0SZ/zd33LM4UM55vCDYdUO8M4f4b1uLJ/7OMuWLGbYsGEAjB49mhNOOGFjbV/84hcB2H///Vm4cOFWvx+tHYq1wC4REfVmi3UzxKUNjCGlNBGYCFBTU+M3a0ntrGuXHXnhVz/ZrH2nbl02/p5I3HjFRRw5/DOb9PnvGU81uu/OnTvzzC8m8ehvnmHyL6Zy0233MO3eiYVr23HHHYHSxaD169cXHrclrb2mOAfYEfh/9drr1hJfbuXjS2ojRw77G35w52TWrVsHwKu/f42Vq1ZzxMEHcds9D7Bq9WoAltYu32TcipWrWP7OCo4+7LNcd+m/8OLL8zbZ3v3DO9Oj+84b1wsnTZq0cdbYGlp7pvhLYB1wCjA+a/8yMNsrz9L7x5knH8vCN95kvxGnkFKiV88e/OyH1zDikCG8MOdVao76Mh/cYQeOPnQI3/761zaOe2fFSkadfj5r1q4lJbj2m+dvtu87rr+Mcy68kFWrVtGvXz9uu+22Vvs7ouj3PkfE8eVfDwPOAb4KLAYWp5QeK/dZD9yRUjojG3clcB7wr8DzwEnA2cDIlNLPKx23pqYmteTzFF1TbD7XFLfO1qwpfqx3vxauZnONrSl2CB//dLOHvvLKKwwYMGCTtoh4LqVUU79vU2aK9W/a/n759TFgePn3TuWf3CXACuBc4P8AvwNOLBKIktTWCodiSqniZ2Ua6pNS2gBcUf6RpA7N5ylKUsZQlKSMoShJGUNRkjKGoqSKOu1ew75HfIm9Dj2BL4w+l2XL32nR/Vcd9HneWlp6yMOH+g9p0X03VYf87LOkLRv0n31adH8vndng1x9vIv+Y3+hzx/G92+/hknPPbNE6OgpnipKa5G/2H8Qf/rQYgN8vfIMRp/wD+484maHHns7c+aUbwP+8eAnHnvEv7HP4Sexz+Ek8+eyLAPzt6eez/4iTGXjI8Uy867/a7W9ojDNFSYVt2LCBR3/zDGf83SgAxlx0BROuvIT+/Xrz9PP/w1e//u9Mu3ci//Rv/8Gwwftx/63XsGHDBlasLD2p+4fXfJOePbqzevUaDvj8qRx39GHs2nOXdvyLNmcoSqpo9Zq17HvEl/jDn/7CgP59OeLgwaxYuYonn3uJE86+aGO/te++C8C0J57lzu9eDpSeXtP9wzsDcMMPf8z9D08H4I03/8y8Ba8bipK2PXVriqtWr+bIk/+B793+U0478Qvs8uGdG3ykWENmPDmLqb9+hqcevJ1uXbsy/PizWLP23VauvOlcU5RUWLeuXbnh8ou45uZJdOvahb67f5x7H/wVUPqO5RfnvArAYZ89kB/cWXpcwoYNG1j+9jssf2cFPbrvTLeuXZk7fwEzn/+fdvs7GmMoSmqST+/1KQYN6M+Pf/ZL7r7pW9z6k5+xz+EnMfCQ45nyyAwAvnvZhUx/chZ7H3Yi+484hZdf/V9GDP8M6zdsYMCwLzL22zcyeL+92/cP2QJPn6VtTJFbaFrainlPbPLvB+/47sbff3n39zbr/7FeuzLltus2a3/4rpsa3P/Cp//6SL/6x2przhQlKWMoSlLGUJSkjKEodXCJRNGvDdHmmvreGYpSB/fasnWsX/W2wdgMKSWWLFlCly5dKncu8+qz1MHd+HQtXwP67PIWQcVvBWm2V2Jxq+27RSx/pVnDunTpwm677Va4v6EodXBvr32Pbz2+pNWP09G/DZFLl1fu0wI8fZakjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhUIxInaPiMkRsTwi3o6I+yKid8GxvSPijoh4PSJWR8SrEXFFROy0daVLUsur+MVVEdENmAasBUYDCbgCmB4Rg1JKKxsZuxMwFdgB+DfgdeAAYDzQHzhpa/8ASWpJRb7N7yygH7BnSmk+QES8BMwDzgaubWTsEErhd2RK6ZFy2/SI6AlcEBHdUkqrml29JLWwIqfPI4GZdYEIkFJaADwBjKow9oPl17frtS8rH7v1vsRWkpqhSCgOBGY30D4HqK4wdiqlGeVVEVEdER+KiEOBc4EJjZ16S1J7KBKKPYHaBtqXAj0aG5hSWgN8tnycOcA7wKPAz4F/3NK4iBgTEbMiYtbixYsLlChJLaNVb8mJiC7APcBHgVOBYcCFlC6wfG9L41JKE1NKNSmlml69erVmiZK0iSIXWmppeEa4pRlk7gxgOLBHSun35bbHI2I5MDEiJqSUXixarCS1tiIzxTmU1hXrqwZerjB2b6A2C8Q6z5RfBxQ4viS1mSKh+AAwOCL61TVERBWl220eqDD2T0CPiNijXvtB5dc/FKxTktpEkVC8BVgITImIURExEpgCvAHcXNcpIvpExPqIGJeNvZ3SxZWHImJ0RBwSERcCVwPPUbqtR5I6jIqhWL5t5lDgVWAScDewADg0pbQi6xpAp3yfKaWFwGDgBUqfgnmI0s3gE4EjUkrvtcQfIUktpciFFlJKrwPHVeizkAZuxk4pvQyc2JziJKmt+ZQcScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhUIxInaPiMkRsTwi3o6I+yKid9GDRMSAiLg3It6KiNUR8buIOLf5ZUtS6+hcqUNEdAOmAWuB0UACrgCmR8SglNLKCuNryuNnAGcCy4H+wIe2qnJJagUVQxE4C+gH7JlSmg8QES8B84CzgWu3NDAiPgDcCTyaUjo22zS92RVLUisqcvo8EphZF4gAKaUFwBPAqApjhwMDaCQ4JakjKRKKA4HZDbTPAaorjP1s+bVLRMyMiHUR8ZeIuCEiujalUElqC0VCsSdQ20D7UqBHhbEfL7/eAzwCHAH8B6W1xR9taVBEjImIWRExa/HixQVKlKSWUWRNcWvUhe5dKaVx5d9nREQn4MqIGJBSeqX+oJTSRGAiQE1NTWrlGiVpoyIzxVoanhFuaQaZW1J+/VW99kfKr58ucHxJajNFQnEOpXXF+qqBlwuMbcx7BY4vSW2mSCg+AAyOiH51DRFRBQwpb2vMw5TubzyyXvuI8uusYmVKUtsoEoq3AAuBKRExKiJGAlOAN4Cb6zpFRJ+IWB8RdWuHpJSWAP8OnBMR346IwyNiLDAOuCO/zUeSOoKKF1pSSisj4lDgOmASEMCjwHkppRVZ1wA6sXnQXga8A3wVuAD4I/Ad4PKtrl6SWlihq88ppdeB4yr0WUgpGOu3J0o3b3sDt6QOz6fkSFLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoYipKUMRQlKWMoSlLGUJSkjKEoSRlDUZIyhqIkZQxFScoUCsWI2D0iJkfE8oh4OyLui4jeTT1YRIyNiBQRv2l6qZLU+iqGYkR0A6YBnwJGA6cC/YHpEbFT0QNFRD/gG8BfmleqJLW+zgX6nAX0A/ZMKc0HiIiXgHnA2cC1BY/1A+BuYM+Cx5WkNlfk9HkkMLMuEAFSSguAJ4BRRQ4SEScD+wFfb06RktRWioTiQGB2A+1zgOpKgyOiB3AdcFFKaWnTypOktlUkFHsCtQ20LwV6FBj/HeBV4PbiZUlS+2jVtb2IGAp8BdgvpZSaMG4MMAagd+8mX+SWpGYrMlOspeEZ4ZZmkLmbgVuBRRGxS0TsQimIO5X/vWNDg1JKE1NKNSmlml69ehUoUZJaRpGZ4hxK64r1VQMvVxg7oPxzTgPbaoF/Bq4vUIMktYkiofgAcHVE9Esp/S9ARFQBQ4CxFcYe0kDb9UAn4GvA/Aa2S1K7KRKKtwD/CEyJiG8ACbgceIPS6TEAEdEH+D1wWUrpMoCU0oz6O4uIZUDnhrZJUnuruKaYUloJHErpCvIkSjdgLwAOTSmtyLoGpRmgn6eWtM0qdPU5pfQ6cFyFPgspBWOlfQ0vckxJag/O6iQpYyhKUsZQlKSMoShJGUNRkjKGoiRlDEVJyhiKkpQxFCUpYyhKUsZQlKSMoShJGUNRkjKGoiRlDEVJyhiKkpQxFCUpYyhKUsZQlKSMoShJGUNRkjKGoiRlDEVJyhiKkpQxFCUpYyhKUsZQlKSMoShJGUNRkjKGoiRlDEVJyhiKkpQxFCUpYyhKUsZQlKSMoShJGUNRkjKGoiRlDEVJyhQKxYjYPSImR8TyiHg7Iu6LiN4FxtVExMSImBsRqyLi9Yi4OyL6bn3pktTyKoZiRHQDpgGfAkYDpwL9gekRsVOF4V8CBgI3AEcBY4H9gFkRsftW1C1JraJzgT5nAf2APVNK8wEi4iVgHnA2cG0jY69KKS3OGyLiCWBBeb/jmlO0JLWWIqfPI4GZdYEIkFJaADwBjGpsYP1ALLe9BiwGPtG0UiWp9RUJxYHA7Aba5wDVTT1gRAwAPgq80tSxktTaioRiT6C2gfalQI+mHCwiOgMTKM0Ub23KWElqC219S85NwGeAL6eUGgpaACJiTETMiohZixdvdgYuSa2mSCjW0vCMcEszyAZFxJXAGOD0lNIjjfVNKU1MKdWklGp69epV9BCStNWKXH2eQ2ldsb5q4OUiB4mIS4CLga+llCYVL0+S2laRmeIDwOCI6FfXEBFVwJDytkZFxD8BVwCXpJRuamadktQmioTiLcBCYEpEjIqIkcAU4A3g5rpOEdEnItZHxLis7UvA9cAvgWkRMTj7afKVa0lqbRVPn1NKKyPiUOA6YBIQwKPAeSmlFVnXADqxadCOKLePKP/kHgOGN7tySWoFRdYUSSm9DhxXoc9CSgGYt50GnNa80iSp7fmUHEnKGIqSlDEUJSljKEpSxlCUpIyhKEkZQ1GSMoaiJGUMRUnKGIqSlDEUJSljKEpSxlCUpIyhKEkZQ1GSMoaiJGUMRUnKGIqSlDEUJSljKEpSxlCUpIyhKEkZQ1GSMoaiJGUMRUnKGIqSlDEUJSljKEpSxlCUpIyhKEkZQ1GSMoaiJGUMRUnKGIqSlDEUJSljKEpSxlCUpIyhKEkZQ1GSMoVCMSJ2j4jJEbE8It6OiPsionfBsV0i4jsR8ceIWB0RT0XEwVtXtiS1joqhGBHdgGnAp4DRwKlAf2B6ROxU4Bi3AmcB44BjgD8C/x0R+zazZklqNZ0L9DkL6AfsmVKaDxARLwHzgLOBa7c0MCL2AU4GTk8p3VZuewyYA1wGjNyq6iWphRU5fR4JzKwLRICU0gLgCWBUgbHrgHuyseuBnwBHRsSOTa5YklpRkVAcCMxuoH0OUF1g7IKU0qoGxn4Q2KPA8SWpzRQJxZ5AbQPtS4EeWzG2brskdRhF1hTbXESMAcaU/7kiIn7XnvW0lYCPAG+1dx1bdkx7F9Ao37+t0+Hfv/HR0nvs01BjkVCspeEZ4ZZmgfXHNnTguhni0ga2kVKaCEwsUNv7SkTMSinVtHcd2yrfv63j+1dS5PR5DqW1wfqqgZcLjO1bvq2n/th3gfmbD5Gk9lMkFB8ABkdEv7qGiKgChpS3NeZBYAfghGxsZ+Ak4JGU0tqmFixJralIKN4CLASmRMSoiBgJTAHeAG6u6xQRfSJifUSMq2tLKf2W0u0410fEmRFxGKXbcfoC32y5P+N9Y7tbMmhhvn9bx/cPiJRS5U6lj/RdBxwBBPAocF5KaWHWpwpYAIxPKV2atXcFvkXpJu5dgBeBi1NKM1rmT5CkllMoFCVpe+FTciQpYyhKUsZQlKSModgOIqJrRJwXEdMj4s8R8W7558/ltvMauLdTTRARB0fEtPauo6OKiOERcUpE7LeF7Z/I7yTZnnihpY1FxO6Unk9ZRelJQ3PY9LPg1ZTuAX0NOCyl9Ho7lLnNi4jjgJ+mlDq1dy0dSUR8CHgEOIjSnSQJ+BWlx/u9mfU7CHhye3z/OuRnn9/nrgdWA/3zW5py5dubfkbpNqjj2qiubULRJ74DvVq1kG3XvwIDgNOAZ4HhwHjg6Yg4MqVU6VNq73vOFNtYRCwHvpxSerBCv5HApJRS97apbNsQEe9Rmt1U7Aqk7XGm05iImAt8P6V0Q9b2CUofyKgCjkopPetMUW2pKf8L+T/W5lYDjwOTK/Sr4a9PWtJf9QZ+mzeklP4QEcOAnwNTI2IUpfd5u2Qotr2pwLciYnb5CeabKZ8+X05prUebehHYkFK6tbFOEbEMQ7EhfwF2q9+YUloZEUcB/wX8ArimrQvrKAzFtnceMB14NSJmUnqqed0j2HpQeiLRYEqfN//ndqivo3sOOL5g3xZ/AN/7wCxKXyPy4/obUkpryrPEHwHfYDs9U3FNsR2UPw8+BvgCpRCse75kLaWr0Q8AtzTwNQ7bvfL61x4ppcfau5ZtUfmq/AXAMSmlJVvoE8D3gREppb5tWV9HYChKUsabtyUpYyhKUsZQlKSMoShJGUNRkjL/H2hNBW+cdPsUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize = (5, 5)\n",
    "fontsize = 16\n",
    "\n",
    "fig_confusion = (\n",
    "    class_summary_df[[\"TP\", \"FP\", \"FN\"]]\n",
    "    .plot(kind=\"bar\", figsize=figsize, width=1, align=\"center\", fontsize=fontsize)\n",
    "    .get_figure()\n",
    ")\n",
    "\n",
    "fig_pr = (\n",
    "    class_summary_df[[\"Precision\", \"Recall\"]]\n",
    "    .plot(kind=\"bar\", figsize=figsize, width=1, align=\"center\", fontsize=fontsize)\n",
    "    .get_figure()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6537775",
   "metadata": {},
   "source": [
    "## Coco Metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac07df8f",
   "metadata": {},
   "source": [
    "Calculate the coco metrics (from pycocotools) from the predictions and labels dataframes. Optionally choose to output per class values.\n",
    "The function returns the results as a dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7f4c0682",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=0.00s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.01s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.806\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 1.000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'All': {'AP_all': 1.0,\n",
       "  'AP_all_IOU_0_50': 1.0,\n",
       "  'AP_all_IOU_0_75': 1.0,\n",
       "  'AP_small': -1.0,\n",
       "  'AP_medium': -1.0,\n",
       "  'AP_large': 1.0,\n",
       "  'AR_all_dets_1': 0.8055555555555556,\n",
       "  'AR_all_dets_10': 1.0,\n",
       "  'AR_all': 1.0,\n",
       "  'AR_small': -1.0,\n",
       "  'AR_medium': -1.0,\n",
       "  'AR_large': 1.0}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = cm.get_coco_from_dfs(preds_df, labels_df, False)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "477a30c2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0146649d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py38",
   "language": "python",
   "name": "py38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}