fireman_data_pec_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports\n",
    "This notebook uses data available to FIREMAN members on [LUT sharepoint](https://lut.sharepoint.com/:f:/r/sites/o365fireman/Shared%20Documents/Colab_PowerElectronicConverter/PEC_datasets/New%20Data?csf=1&web=1&e=JINTw3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import pandas as pd\n",
    "# to save results to data directory\n",
    "module_path = '..'\n",
    "if module_path not in sys.path:\n",
    "    sys.path.insert(0, module_path)\n",
    "# increase displayed columns in jupyter notebook\n",
    "pd.set_option('display.max_columns', 200)\n",
    "pd.set_option('display.max_rows', 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import scipy.io\n",
    "import src.common as common\n",
    "\n",
    "# increase displayed columns in jupyter notebook\n",
    "pd.set_option('display.max_columns', 200)\n",
    "pd.set_option('display.max_rows', 300)\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "matplotlib.use(\"pgf\")\n",
    "matplotlib.rcParams.update({\n",
    "    \"pgf.texsystem\": \"pdflatex\",\n",
    "    'font.family': 'serif',\n",
    "    'text.usetex': True,\n",
    "    'pgf.rcfonts': False,\n",
    "})\n",
    "\n",
    "# temporarily remove deprecation warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "\n",
    "output_dir = 'pec_output'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**identifiers**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_types = common.json_load(\"#datasets/PEC/column_types.json\")\n",
    "\n",
    "identifier = column_types[\"identifier\"]\n",
    "target = column_types[\"target\"]\n",
    "datasets = column_types[\"datasets\"]\n",
    "feature_ids = column_types[\"feature_ids\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**agregate data from matlab files**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/abeattie/miniconda/envs/masters-thesis/lib/python3.10/site-packages/scipy/io/matlab/mio.py:226: MatReadWarning: Duplicate variable name \"None\" in stream - replacing previous with new\n",
      "Consider mio5.varmats_from_mat to split file into single variable files\n",
      "  matfile_dict = MR.get_variables(variable_names)\n",
      "/Users/abeattie/miniconda/envs/masters-thesis/lib/python3.10/site-packages/scipy/io/matlab/mio.py:226: MatReadWarning: Duplicate variable name \"None\" in stream - replacing previous with new\n",
      "Consider mio5.varmats_from_mat to split file into single variable files\n",
      "  matfile_dict = MR.get_variables(variable_names)\n",
      "/Users/abeattie/miniconda/envs/masters-thesis/lib/python3.10/site-packages/scipy/io/matlab/mio.py:226: MatReadWarning: Duplicate variable name \"None\" in stream - replacing previous with new\n",
      "Consider mio5.varmats_from_mat to split file into single variable files\n",
      "  matfile_dict = MR.get_variables(variable_names)\n",
      "/Users/abeattie/miniconda/envs/masters-thesis/lib/python3.10/site-packages/scipy/io/matlab/mio.py:226: MatReadWarning: Duplicate variable name \"None\" in stream - replacing previous with new\n",
      "Consider mio5.varmats_from_mat to split file into single variable files\n",
      "  matfile_dict = MR.get_variables(variable_names)\n"
     ]
    }
   ],
   "source": [
    "data_df_final = pd.DataFrame(columns = list(feature_ids.keys()).extend([\"fault_id\", \"fault\"]))\n",
    "\n",
    "for dataset in datasets.keys():\n",
    "    data_raw_dict = {}\n",
    "    data_raw = scipy.io.loadmat(\"#datasets/PEC/\" + dataset + \"/in.mat\")\n",
    "    in_len = len(data_raw[\"in\"])\n",
    "    for feature in feature_ids.keys():\n",
    "        data_raw_dict.update({feature: data_raw[feature_ids[feature][0]][:in_len,feature_ids[feature][1]]})\n",
    "    data_raw_dict.update({\"label\": [dataset]*in_len})\n",
    "    data_df = pd.DataFrame(data_raw_dict)\n",
    "    data_df[\"fault\"] = 0\n",
    "    if len(datasets[dataset][\"failure\"][0])==1:\n",
    "        data_df.loc[datasets[dataset][\"failure\"][0][0]:, \"fault\"] = 1\n",
    "    if len(datasets[dataset][\"failure\"][0])==2:\n",
    "        data_df.loc[datasets[dataset][\"failure\"][0][0]:datasets[dataset][\"failure\"][0][1], \"fault\"] = 1\n",
    "    data_df.drop(data_df.index[datasets[dataset][\"drop\"][0][0]:datasets[dataset][\"drop\"][0][1]], inplace=True)\n",
    "    data_df_final = pd.concat([data_df_final, data_df], ignore_index=True)\n",
    "data_df_final[identifier] = data_df_final.index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**save to pickle for further use**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df_final.to_pickle(\"#datasets/PEC/dataset.pkl\")\n",
    "data_df_final.to_pickle(\"../masters-thesis-graphing/_data/PEC/dataset.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple display of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "    f_c            P    m_d  m_q       theta   P_ref   V_DC   V_phaseA  \\\n0  50.0  2499.997221  311.0  0.0  312.486512  2500.0  800.0   2.487576   \n1  50.0  2499.997250  311.0  0.0  312.502220  2500.0  800.0   7.372088   \n2  50.0  2499.997280  311.0  0.0  312.517928  2500.0  800.0  12.254782   \n3  50.0  2499.997310  311.0  0.0  312.533636  2500.0  800.0  17.134452   \n4  50.0  2499.997341  311.0  0.0  312.549344  2500.0  800.0  22.009894   \n\n     V_phaseB    V_phaseC  I_phaseA  I_phaseB  I_phaseC     label  fault     \n0 -270.569073  268.081497 -0.392875 -4.447507  4.840382  LL_Fault      0  0  \n1 -272.944264  265.572176 -0.308598 -4.494417  4.803015  LL_Fault      0  1  \n2 -275.252112  262.997330 -0.224245 -4.540218  4.764463  LL_Fault      0  2  \n3 -277.492044  260.357593 -0.139837 -4.584899  4.724735  LL_Fault      0  3  \n4 -279.663510  257.653616 -0.055394 -4.628448  4.683842  LL_Fault      0  4  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>f_c</th>\n      <th>P</th>\n      <th>m_d</th>\n      <th>m_q</th>\n      <th>theta</th>\n      <th>P_ref</th>\n      <th>V_DC</th>\n      <th>V_phaseA</th>\n      <th>V_phaseB</th>\n      <th>V_phaseC</th>\n      <th>I_phaseA</th>\n      <th>I_phaseB</th>\n      <th>I_phaseC</th>\n      <th>label</th>\n      <th>fault</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>50.0</td>\n      <td>2499.997221</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.486512</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>2.487576</td>\n      <td>-270.569073</td>\n      <td>268.081497</td>\n      <td>-0.392875</td>\n      <td>-4.447507</td>\n      <td>4.840382</td>\n      <td>LL_Fault</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>50.0</td>\n      <td>2499.997250</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.502220</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>7.372088</td>\n      <td>-272.944264</td>\n      <td>265.572176</td>\n      <td>-0.308598</td>\n      <td>-4.494417</td>\n      <td>4.803015</td>\n      <td>LL_Fault</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>50.0</td>\n      <td>2499.997280</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.517928</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>12.254782</td>\n      <td>-275.252112</td>\n      <td>262.997330</td>\n      <td>-0.224245</td>\n      <td>-4.540218</td>\n      <td>4.764463</td>\n      <td>LL_Fault</td>\n      <td>0</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>50.0</td>\n      <td>2499.997310</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.533636</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>17.134452</td>\n      <td>-277.492044</td>\n      <td>260.357593</td>\n      <td>-0.139837</td>\n      <td>-4.584899</td>\n      <td>4.724735</td>\n      <td>LL_Fault</td>\n      <td>0</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>50.0</td>\n      <td>2499.997341</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.549344</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>22.009894</td>\n      <td>-279.663510</td>\n      <td>257.653616</td>\n      <td>-0.055394</td>\n      <td>-4.628448</td>\n      <td>4.683842</td>\n      <td>LL_Fault</td>\n      <td>0</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    }
   ],
   "source": [
    "data_df_final.plot(subplots=True, figsize=(16,15));"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "3b99005fd577fa40f3cce433b2b92303885900e634b2b5344c07c59d06c8792d"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 5
}