notebooks/aggregate_referral_lists.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import warnings  \n",
    "import csv\n",
    "# ignore pandas FutureWarning below\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from utils.constants import PREDICTIONS_DIR\n",
    "from dateutil.relativedelta import relativedelta\n",
    "from datetime import date\n",
    "from postmodeling.aggregate_lists import df_val_date, get_referral_lists, get_all_referred_joids, get_aggregated_referral_info\n",
    "from postmodeling.analyze_labels import get_all_flagged_events\n",
    "from utils.helpers import get_database_connection\n",
    "from pipeline.matrix import make_str_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_val_date, max_val_date = date(2020, 6, 1), date(2021, 5, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joco_exp_ids = [372, 231, 233, 363, 8]\n",
    "joco_model_set_ids = [4739, 1924, 1926, 4733, 38]\n",
    "doco_exp_ids = [149, 231, 233, 201, 371]\n",
    "doco_model_set_ids = [792, 1924, 1926, 1487, 4737]\n",
    "# best_models = {\n",
    "#     'joco': {'exp_ids': joco_exp_ids, 'model_set_ids': joco_model_set_ids},\n",
    "#     'doco': {'exp_ids': doco_exp_ids, 'model_set_ids': doco_model_set_ids}\n",
    "# }\n",
    "best_models = {\n",
    "    'joco': {'model_set_ids': joco_model_set_ids},\n",
    "    'doco': {'model_set_ids': doco_model_set_ids}\n",
    "}\n",
    "label_groups = 'death only', 'potentially fatal', 'suicide-related only', 'drug-related only', 'all behavioral crises'\n",
    "db_conn = get_database_connection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df, deaths_df = get_aggregated_referral_info(db_conn, best_models, label_groups, min_val_date, max_val_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df[(concat_df['county'] == 'joco') & (concat_df['label_group'] == 'potentially fatal')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df_no_death_flag = concat_df[concat_df['event_type'] != 'death_flag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df_no_death_flag.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.catplot(y='event_type', x='validation_period', row='county', col='label_group', kind='bar',\n",
    "        data=concat_df_no_death_flag, orient='h', color='#33485E')\n",
    "# for ax in g.fig.axes:\n",
    "#     ax.tick_params(axis='x', rotation=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for county in ['joco', 'doco']:\n",
    "    display(concat_df[(concat_df['event_type'] == 'suicide_attempt_flag') & (concat_df['county'] == county) & (concat_df['label_group'] == 'potentially fatal')])\n",
    "\n",
    "concat_df[(concat_df['event_type'] == 'suicide_attempt_flag') & (concat_df['county'] == 'doco') & (concat_df['label_group'] == 'potentially fatal')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.catplot(row='county', col='label_group', kind='bar', data=deaths_df, orient='h', color='#33485E')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prettier plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set(font_scale = 2)\n",
    "\n",
    "concat_df_no_death_flag = concat_df[concat_df['event_type'] != 'death_flag']\n",
    "concat_df_no_death_flag['event_type'] = concat_df_no_death_flag['event_type'].str.replace('_flag', '')\n",
    "concat_df_no_death_flag['event_type'] = concat_df_no_death_flag['event_type'].str.replace('_', ' ')\n",
    "\n",
    "concat_df_no_death_flag_joco = concat_df_no_death_flag[concat_df_no_death_flag['county']=='joco']\n",
    "concat_df_no_death_flag_doco = concat_df_no_death_flag[concat_df_no_death_flag['county']=='doco']\n",
    "\n",
    "g = sns.catplot(y='event_type', x='validation_period', col='label_group', kind='bar',\n",
    "    data=concat_df_no_death_flag_joco, orient='h', color='#33485E')\n",
    "\n",
    "g.set_titles(\"Model for {col_name}\", size = 20)\n",
    "g.set(xticks = [0, 100, 200, 300, 400])\n",
    "g.set(xlabel = 'Counts')\n",
    "g.set(ylabel = 'Event Type')\n",
    "g.fig.suptitle('Johnson County', y = 1.05)\n",
    "\n",
    "\n",
    "g = sns.catplot(y='event_type', x='validation_period', col='label_group', kind='bar',\n",
    "    data=concat_df_no_death_flag_doco, orient='h', color='#33485E')\n",
    "\n",
    "g.set_titles(\"Model for {col_name}\", size = 20)\n",
    "g.set(xticks = [0, 100, 200, 300, 400])\n",
    "g.set(xlabel = 'Counts')\n",
    "g.set(ylabel = 'Event Type')\n",
    "g.fig.suptitle('Douglas County', y = 1.05)\n",
    "\n",
    "\n",
    "# for ax in g.fig.axes:\n",
    "#     ax.tick_params(axis='x', rotation=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_df = deaths_df.rename(columns = {'suic_or_od': 'both'})\n",
    "deaths_df_joco = deaths_df[deaths_df['county'] == 'joco']\n",
    "deaths_df_doco = deaths_df[deaths_df['county'] == 'doco']\n",
    "\n",
    "g = sns.catplot(col='label_group', kind='bar', data=deaths_df_joco, orient='h', color='#33485E')\n",
    "g.set_titles(\"Model for {col_name}\", size = 20)\n",
    "g.set(xticks = [0,2,4,6,8])\n",
    "g.fig.suptitle('Johnson County', y = 1.05)\n",
    "\n",
    "g = sns.catplot(col='label_group', kind='bar', data=deaths_df_doco, orient='h', color='#33485E')\n",
    "g.set_titles(\"Model for {col_name}\", size = 20)\n",
    "g.set(xticks = [0,2,4,6,8])\n",
    "g.fig.suptitle('Douglas County', y = 1.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get a sample lists for each county"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_set_id = 1924\n",
    "min_val_date = max_val_date = date(2021, 9, 1)\n",
    "ref_lists = {}\n",
    "for county in ['joco', 'doco']:\n",
    "    low_k = (40 if county == 'doco' else 75)\n",
    "    high_k = low_k\n",
    "    all_lists = get_referral_lists(model_set_id, low_k, high_k, county, min_val_date, max_val_date, 1)\n",
    "    ref_lists[county] = all_lists[high_k][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write joids to a csv\n",
    "for county in ['joco', 'doco']:\n",
    "    print(len(ref_lists[county]))\n",
    "    with open(f'sample_list_{county}.csv', 'w') as f:\n",
    "        writer = csv.writer(f)\n",
    "        for joid in ref_lists[county]:\n",
    "            writer.writerow([joid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = []\n",
    "for county in ['joco', 'doco']:\n",
    "    print(len(ref_lists[county]))\n",
    "    with open(f'sample_list_{county}.csv', 'r') as f:\n",
    "        dfs.append(pd.read_csv(f, header=None, names=['joid']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table_names = ['joco', 'doco']\n",
    "db_conn = get_database_connection()\n",
    "for df, tn in zip(dfs, table_names):\n",
    "    df.to_sql(tn, db_conn, schema='sample_lists')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "3b13c6f641af22525ec1bb5ca2867395e8a50d86a8864b59bafbc48fdc461750"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}