notebooks/evaluations2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "from utils.helpers import get_database_connection\n",
    "from utils.constants import SQL_READ_PRED_LABELS, SQL_READ_FEATURES_TEST_PRED\n",
    "\n",
    "from pipeline.evaluation import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "db_conn = get_database_connection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Params\n",
    "model_id = 8\n",
    "joco_k = 75\n",
    "doco_k = 40\n",
    "threshold = 0.01\n",
    "as_of_date = '2021-07-01'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Coufusion Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confusion matrix: Both counties\n",
    "model_results = get_test_pred_labels(db_conn, model_id)\n",
    "\n",
    "if len(model_results) > 0:\n",
    "    cf = get_confusion_matrix(model_results, doco_k=doco_k, joco_k=joco_k)\n",
    "    plt_cf = plot_confusion_matrix(cf)\n",
    "    plt_cf.plot()\n",
    "else: \n",
    "    print(\"No results available from selected model_id: \", model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confusion matrix: Johnson County\n",
    "if len(model_results) > 0:\n",
    "    cf = get_confusion_matrix(model_results, doco_k=None, joco_k=joco_k)\n",
    "    plt_cf = plot_confusion_matrix(cf)\n",
    "    plt_cf.plot()\n",
    "else:\n",
    "    print(\"No results available from selected model_id: \", model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confusion matrix: Douglas County\n",
    "if len(model_results) > 0:\n",
    "    cf = get_confusion_matrix(model_results, doco_k=doco_k, joco_k=None)\n",
    "    plt_cf = plot_confusion_matrix(cf)\n",
    "    plt_cf.plot()\n",
    "else:\n",
    "    print(\"No results available from selected model_id: \", model_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature-Labels Crosstabs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Crosstabs: categorical demographics\n",
    "features_table = 'demographics_cat'\n",
    "features_test_pred = get_features_test_pred(db_conn, as_of_date, model_id, features_table, doco_k=doco_k, joco_k=joco_k)\n",
    "features_test_pred = features_test_pred.fillna('Missing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sex: Both counties \n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred, \"dem_sex\")\n",
    "print(\"SEX:\")\n",
    "print('----------------------------------')\n",
    "print(label_crosstab)\n",
    "print('----------------------------------')\n",
    "print(pred_crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sex: Johnson County\n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred.loc[features_test_pred['county'] == 'joco'], \"dem_sex\")\n",
    "print(\"Johnson County - SEX:\")\n",
    "print('----------------------------------')\n",
    "print(label_crosstab)\n",
    "print('----------------------------------')\n",
    "print(pred_crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sex: Douglas County\n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred.loc[features_test_pred['county'] == 'doco'], \"dem_sex\")\n",
    "print(\"Douglas County - SEX:\")\n",
    "print('----------------------------------')\n",
    "print(label_crosstab)\n",
    "print('----------------------------------')\n",
    "print(pred_crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Race: Both Counties\n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred, \"dem_race\")\n",
    "print(\"RACE:\")\n",
    "print('-----------------------------------------------------')\n",
    "print(label_crosstab)\n",
    "print('-----------------------------------------------------')\n",
    "print(pred_crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Race: Johnson County\n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred.loc[features_test_pred['county'] == 'joco'], \"dem_race\")\n",
    "print(\"Johnson County - RACE:\")\n",
    "print('-----------------------------------------------------')\n",
    "print(label_crosstab)\n",
    "print('-----------------------------------------------------')\n",
    "print(pred_crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Race: Douglas County\n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred.loc[features_test_pred['county'] == 'doco'], \"dem_race\")\n",
    "print(\"Douglas County - RACE:\")\n",
    "print('-----------------------------------------------------')\n",
    "print(label_crosstab)\n",
    "print('-----------------------------------------------------')\n",
    "print(pred_crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Crosstabs: demographics numeric \n",
    "features_table = 'demographics_num'\n",
    "features_test_pred = get_features_test_pred(db_conn, as_of_date, model_id, features_table, doco_k=doco_k, joco_k=joco_k)\n",
    "features_test_pred[\"dem_age_yrs\"] = features_test_pred[\"dem_age\"] / 365.25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Age spliting buckets\n",
    "bins = [0.0, 20.0, 40.0, 65.0, np.inf]\n",
    "names = ['<20', '21-40', '41-65', '66+']\n",
    "split_tuple = (bins, names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Age: Both Counties\n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred, \"dem_age_yrs\", split_tuple)\n",
    "print(\"AGE:\")\n",
    "print('-----------------------------------------------------')\n",
    "print(label_crosstab)\n",
    "print('-----------------------------------------------------')\n",
    "print(pred_crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Age: Johnson County\n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred.loc[features_test_pred['county'] == 'joco'], \"dem_age_yrs\", split_tuple)\n",
    "print(\"Johnson County - AGE:\")\n",
    "print('-----------------------------------------------------')\n",
    "print(label_crosstab)\n",
    "print('-----------------------------------------------------')\n",
    "print(pred_crosstab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Age: Douglas County\n",
    "label_crosstab, pred_crosstab = create_crosstabs(features_test_pred.loc[features_test_pred['county'] == 'doco'], \"dem_age_yrs\", split_tuple)\n",
    "print(\"Douglas County - AGE:\")\n",
    "print('-----------------------------------------------------')\n",
    "print(label_crosstab)\n",
    "print('-----------------------------------------------------')\n",
    "print(pred_crosstab)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dojo-mh",
   "language": "python",
   "name": "dojo-mh"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}