trying_hdf5_files.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shahad.hardan/.conda/envs/fusion/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import h5py\n",
    "import torch\n",
    "import sparse\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trying hdf5 Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = h5py.File(\n",
    "    \"/home/shahad.hardan/Documents/research_tf/preprocessed_fiddle_3/features/outcome=ARF,T=12.0,dt=1.0/splits.hdf5\",\n",
    "    \"r\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['with_notes']>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['test', 'train', 'val']>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits[\"with_notes\"].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['S', 'X', 'attention_mask', 'input_ids', 'label', 'token_type_ids']>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits[\"with_notes\"][\"train\"].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6626, 12, 5132)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits[\"with_notes\"][\"train\"][\"X\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6626, 96)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits[\"with_notes\"][\"train\"][\"S\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1389, 12, 5132)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits[\"with_notes\"][\"val\"][\"X\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1389, 96)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits[\"with_notes\"][\"val\"][\"S\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# splits['with_notes'][\"train\"][\"X\"][()]\n",
    "splits.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = h5py.File(\n",
    "    \"/home/shahad.hardan/Documents/research_tf/preprocessed_fiddle_3/features/outcome=ARF,T=12.0,dt=1.0/Xs.hdf5\",\n",
    "    \"r\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['S', 'X']>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14174, 96)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs[\"S\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14174, 12, 5132)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs[\"X\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "notes = h5py.File(\n",
    "    \"/home/shahad.hardan/Documents/research_tf/preprocessed_fiddle_3/features/outcome=ARF,T=12.0,dt=1.0/notes.hdf5\",\n",
    "    \"r\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['notes']>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['axis0', 'axis1', 'block0_items', 'block0_values', 'block1_items', 'block1_values']>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes[\"notes\"].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of axis0 (4,)\n",
      "shape of axis1 (9393,)\n",
      "shape of block0_items (1,)\n",
      "shape of block0_values (9393, 1)\n",
      "shape of block1_items (3,)\n",
      "shape of block1_values (1,)\n"
     ]
    }
   ],
   "source": [
    "print(\"shape of axis0\", notes[\"notes\"][\"axis0\"].shape)\n",
    "print(\"shape of axis1\", notes[\"notes\"][\"axis1\"].shape)\n",
    "print(\"shape of block0_items\", notes[\"notes\"][\"block0_items\"].shape)\n",
    "print(\"shape of block0_values\", notes[\"notes\"][\"block0_values\"].shape)\n",
    "print(\"shape of block1_items\", notes[\"notes\"][\"block1_items\"].shape)\n",
    "print(\"shape of block1_values\", notes[\"notes\"][\"block1_values\"].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([b'ICUSTAY_ID', b'input_ids', b'token_type_ids', b'attention_mask'],\n",
       "      dtype='|S14')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes[\"notes\"][\"axis0\"][()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   0,    1,    2, ..., 9390, 9391, 9392])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes[\"notes\"][\"axis1\"][()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([b'ICUSTAY_ID'], dtype='|S10')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "notes[\"notes\"][\"block0_items\"][()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "notes.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trying Torch Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shahad.hardan/.conda/envs/fusion/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.9538,  1.3804,  0.8969],\n",
       "        [-1.0373,  1.1481, -0.2625]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "x = torch.randn(2, 3)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.9538,  1.3804,  0.8969],\n",
       "        [-1.0373,  1.1481, -0.2625],\n",
       "        [ 1.9538,  1.3804,  0.8969],\n",
       "        [-1.0373,  1.1481, -0.2625],\n",
       "        [ 1.9538,  1.3804,  0.8969],\n",
       "        [-1.0373,  1.1481, -0.2625]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat((x, x, x), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.9538,  1.3804,  0.8969,  1.9538,  1.3804,  0.8969,  1.9538,  1.3804,\n",
       "          0.8969],\n",
       "        [-1.0373,  1.1481, -0.2625, -1.0373,  1.1481, -0.2625, -1.0373,  1.1481,\n",
       "         -0.2625]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat((x, x, x), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 2., 3., 4., 5.])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.arange(1.0, 6.0)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.return_types.topk(\n",
       "values=tensor([5., 4., 3.]),\n",
       "indices=tensor([4, 3, 2]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.topk(x, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 1],\n",
       "        [4, 3]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.tensor([[1, 2], [3, 4]])\n",
    "torch.gather(t, 1, torch.tensor([[0, 0], [1, 0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 1],\n",
       "        [4, 3]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.gather(t, -1, torch.tensor([[0, 0], [1, 0]]))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking npz files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tbody><tr><th style=\"text-align: left\">Format</th><td style=\"text-align: left\">coo</td></tr><tr><th style=\"text-align: left\">Data Type</th><td style=\"text-align: left\">int64</td></tr><tr><th style=\"text-align: left\">Shape</th><td style=\"text-align: left\">(14174, 12, 5132)</td></tr><tr><th style=\"text-align: left\">nnz</th><td style=\"text-align: left\">13508911</td></tr><tr><th style=\"text-align: left\">Density</th><td style=\"text-align: left\">0.015476046226568408</td></tr><tr><th style=\"text-align: left\">Read-only</th><td style=\"text-align: left\">True</td></tr><tr><th style=\"text-align: left\">Size</th><td style=\"text-align: left\">412.3M</td></tr><tr><th style=\"text-align: left\">Storage ratio</th><td style=\"text-align: left\">0.1</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<COO: shape=(14174, 12, 5132), dtype=int64, nnz=13508911, fill_value=0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_sparse = sparse.load_npz(\n",
    "    \"/home/shahad.hardan/Documents/research_tf/preprocessed_fiddle_3/features/outcome=ARF,T=12.0,dt=1.0/X.npz\"\n",
    ")\n",
    "x_sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tbody><tr><th style=\"text-align: left\">Format</th><td style=\"text-align: left\">coo</td></tr><tr><th style=\"text-align: left\">Data Type</th><td style=\"text-align: left\">int64</td></tr><tr><th style=\"text-align: left\">Shape</th><td style=\"text-align: left\">(14174, 96)</td></tr><tr><th style=\"text-align: left\">nnz</th><td style=\"text-align: left\">158269</td></tr><tr><th style=\"text-align: left\">Density</th><td style=\"text-align: left\">0.11631405507737172</td></tr><tr><th style=\"text-align: left\">Read-only</th><td style=\"text-align: left\">True</td></tr><tr><th style=\"text-align: left\">Size</th><td style=\"text-align: left\">2.4M</td></tr><tr><th style=\"text-align: left\">Storage ratio</th><td style=\"text-align: left\">0.2</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<COO: shape=(14174, 96), dtype=int64, nnz=158269, fill_value=0>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s_sparse = sparse.load_npz(\n",
    "    \"/home/shahad.hardan/Documents/research_tf/preprocessed_fiddle_3/features/outcome=ARF,T=12.0,dt=1.0/S.npz\"\n",
    ")\n",
    "s_sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "population = pd.read_csv(\n",
    "    \"/home/shahad.hardan/Documents/research_tf/preprocessed_fiddle_3/population/ARF_12.0h.csv\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14174, 3)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "population.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ID', 'ARF_ONSET_HOUR', 'ARF_LABEL'], dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "population.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({0: 12806, 1: 1368})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(population[\"ARF_LABEL\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "icustays_mv = pd.read_csv(\n",
    "    \"/home/shahad.hardan/Documents/research_tf/preprocessed_fiddle_3/prep/icustays_MV.csv\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23620, 13)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icustays_mv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17710"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(icustays_mv[\"SUBJECT_ID\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "ed21b98d8506aa6344334002e67f5d0be1c6f9d15a09fd4e94f600c09cb17cfe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}