benchmark_last_visit_for_PSdummy.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": 3
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python_defaultSpec_1595418701889",
   "display_name": "Python 3.7.0 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark last visit (for dummy Parelsnoer)\n",
    "This python notebook trains, tests and evaluates the benchmark last visit. \n",
    "Tesing is done on a cross-sectional data set, Parelsnoer Dummy, which is a resricted version of ADNI D3 data set and serves as an example dataset for the Parelsnoer dataset. Data set used for training and evalution are D1_D2 and D4 ADNI data sets respectively."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train model on data set D1_D2 and test model on cross sectional data set Parelsnoer dummy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n[]\n"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import datetime\n",
    "from pathlib import Path\n",
    "from tadpole_algorithms.models import BenchmarkLastVisit\n",
    "from tadpole_algorithms.preprocessing.split import split_test_train_parelsnoer\n",
    "from tadpole_algorithms.preprocessing.rewrite_df import rewrite_d3\n",
    "from tadpole_algorithms.preprocessing.create_dummy_ps import create_parelsnoer_dummy\n",
    "\n",
    "\"\"\"\n",
    "Train model on ADNI data set D1_D2\n",
    "Test model on parelsnoer dummy\n",
    "\"\"\"\n",
    "\n",
    "# Load D1_D2 train and possible test data set\n",
    "data_path_train = Path(\"data/TADPOLE_D1_D2.csv\")\n",
    "data_df_train = pd.read_csv(data_path_train)\n",
    "\n",
    "# Load D3 dataset and convert D3 to Parelsnoer dummy dataset\n",
    "data_path_test = Path(\"data/TADPOLE_D3.csv\")\n",
    "data_df_test = pd.read_csv(data_path_test) \n",
    "data_df_test = create_parelsnoer_dummy(data_df_test)\n",
    "\n",
    "# Load D4 evaluation data set\n",
    "data_path_eval = Path(\"data/TADPOLE_D4_corr.csv\")\n",
    "data_df_eval = pd.read_csv(data_path_eval)\n",
    "\n",
    "\n",
    "# Split data in test, train and evulation data\n",
    "train_df, test_df, eval_df = split_test_train_parelsnoer(data_df_train, data_df_test, data_df_eval)\n",
    "test_df = test_df.fillna(0)\n",
    "test_df = rewrite_d3(test_df)\n",
    "\n",
    "# Define and train model\n",
    "model = BenchmarkLastVisit()\n",
    "model.train(train_df)\n",
    "\n",
    "# Predict forecast on the test set\n",
    "forecast_df_d3 = model.predict(test_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate model tested on Parelsnoer dummy on ADNI data set D4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "[[69  8  9]\n [16 64 12]\n [ 2 12 18]]\nmAUC (multiclass Area Under Curve): 0.753\nbca (balanced classification accuracy): 0.771\nventsMAE (Ventricles Mean Aboslute Error): 0.012\nventsWES (Ventricles Weighted Error Score ): 0.006\nventsCPA (Ventricles Coverage Probability Accuracy for 50% Confidence Interval: 0.020\n"
    }
   ],
   "source": [
    "from tadpole_algorithms.evaluation import evaluate_forecast\n",
    "from tadpole_algorithms.evaluation import print_metrics\n",
    "\n",
    "# Evaluate the model \n",
    "dictionary = evaluate_forecast(eval_df, forecast_df_d3)\n",
    "\n",
    "# Print metrics\n",
    "print_metrics(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}