Preprocessing.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Preprocessing.ipynb","provenance":[],"collapsed_sections":["-OGDbCjFuQtX","gDYGgd6PudY7","moY0IwT-LQyW","qA3MZ1nawX0G","_J_FBZYFwsE7","zHOJAISrxOCy","348g8NqD41Jo","I7FEEB_e8b_3","hrKI12qRYwyo","82YTtg9njQgP","WslJfzKsMXKT"]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"jKOQK-IXWo-A"},"source":["### ***Utilities***"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pCnzYM1FWaJq","executionInfo":{"status":"ok","timestamp":1623163009599,"user_tz":-270,"elapsed":94257,"user":{"displayName":"Mohammad Mirsafaei","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgMMfkaVHqEsQp4cQUbu20VYNCJm5MedDnb8-sOcA=s64","userId":"14901075838947117855"}},"outputId":"d6e66d52-b57a-4341-dd21-eea79ef6a663"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-8K1dqlGWujN","executionInfo":{"status":"ok","timestamp":1623170776043,"user_tz":-270,"elapsed":605,"user":{"displayName":"Mohammad Mirsafaei","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgMMfkaVHqEsQp4cQUbu20VYNCJm5MedDnb8-sOcA=s64","userId":"14901075838947117855"}},"outputId":"68063b1d-52de-4ff3-fa0a-df2047f1f305"},"source":["import gc\n","gc.collect()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["141"]},"metadata":{"tags":[]},"execution_count":58}]},{"cell_type":"markdown","metadata":{"id":"-OGDbCjFuQtX"},"source":["## ***Import and Options:***"]},{"cell_type":"code","metadata":{"id":"L_mWuNRQuH4S"},"source":["import pandas as pd\n","import numpy as np\n","from sklearn.impute import SimpleImputer\n","import math"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2SHx4nccweKB"},"source":["pd.set_option('display.max_columns',500)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"gDYGgd6PudY7"},"source":["## ***Reading Dataset:***"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"emg1SSb7u05W","executionInfo":{"status":"ok","timestamp":1623170770021,"user_tz":-270,"elapsed":30650,"user":{"displayName":"Mohammad Mirsafaei","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgMMfkaVHqEsQp4cQUbu20VYNCJm5MedDnb8-sOcA=s64","userId":"14901075838947117855"}},"outputId":"eff5b292-903f-4651-b6a9-b1fd8ffbd982"},"source":["df = pd.read_csv('/content/drive/MyDrive/FDM/dataset/US_births(2018).csv',low_memory=False)\n","df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>ATTEND</th>\n","      <th>BFACIL</th>\n","      <th>BMI</th>\n","      <th>CIG_0</th>\n","      <th>DBWT</th>\n","      <th>DLMP_MM</th>\n","      <th>DLMP_YY</th>\n","      <th>DMAR</th>\n","      <th>DOB_MM</th>\n","      <th>DOB_TT</th>\n","      <th>DOB_WK</th>\n","      <th>DOB_YY</th>\n","      <th>DWgt_R</th>\n","      <th>FAGECOMB</th>\n","      <th>FEDUC</th>\n","      <th>FHISPX</th>\n","      <th>FRACE15</th>\n","      <th>FRACE31</th>\n","      <th>FRACE6</th>\n","      <th>ILLB_R</th>\n","      <th>ILOP_R</th>\n","      <th>ILP_R</th>\n","      <th>IMP_SEX</th>\n","      <th>IP_GON</th>\n","      <th>LD_INDL</th>\n","      <th>MAGER</th>\n","      <th>MAGE_IMPFLG</th>\n","      <th>MAR_IMP</th>\n","      <th>MBSTATE_REC</th>\n","      <th>MEDUC</th>\n","      <th>MHISPX</th>\n","      <th>MM_AICU</th>\n","      <th>MRACE15</th>\n","      <th>MRACE31</th>\n","      <th>MRACEIMP</th>\n","      <th>MRAVE6</th>\n","      <th>MTRAN</th>\n","      <th>M_Ht_In</th>\n","      <th>NO_INFEC</th>\n","      <th>NO_MMORB</th>\n","      <th>NO_RISKS</th>\n","      <th>PAY</th>\n","      <th>PAY_REC</th>\n","      <th>PRECARE</th>\n","      <th>PREVIS</th>\n","      <th>PRIORDEAD</th>\n","      <th>PRIORLIVE</th>\n","      <th>PRIORTERM</th>\n","      <th>PWgt_R</th>\n","      <th>RDMETH_REC</th>\n","      <th>RESTATUS</th>\n","      <th>RF_CESAR</th>\n","      <th>RF_CESARN</th>\n","      <th>SEX</th>\n","      <th>WTGAIN</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>30.7</td>\n","      <td>0</td>\n","      <td>3657</td>\n","      <td>4</td>\n","      <td>2017</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1227</td>\n","      <td>2</td>\n","      <td>2018</td>\n","      <td>231</td>\n","      <td>31</td>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>16</td>\n","      <td>33</td>\n","      <td>16</td>\n","      <td></td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>30</td>\n","      <td></td>\n","      <td></td>\n","      <td>1</td>\n","      <td>6</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td></td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>66</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>3</td>\n","      <td>8</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>190</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>M</td>\n","      <td>41</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>33.3</td>\n","      <td>2</td>\n","      <td>3242</td>\n","      <td>99</td>\n","      <td>9999</td>\n","      <td>2</td>\n","      <td>1</td>\n","      <td>1704</td>\n","      <td>2</td>\n","      <td>2018</td>\n","      <td>185</td>\n","      <td>35</td>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>180</td>\n","      <td>888</td>\n","      <td>180</td>\n","      <td></td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>35</td>\n","      <td></td>\n","      <td></td>\n","      <td>1</td>\n","      <td>9</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td></td>\n","      <td>3</td>\n","      <td>N</td>\n","      <td>63</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>9</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>188</td>\n","      <td>4</td>\n","      <td>2</td>\n","      <td>Y</td>\n","      <td>2</td>\n","      <td>F</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>30.0</td>\n","      <td>0</td>\n","      <td>3470</td>\n","      <td>4</td>\n","      <td>2017</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>336</td>\n","      <td>2</td>\n","      <td>2018</td>\n","      <td>273</td>\n","      <td>31</td>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>999</td>\n","      <td>888</td>\n","      <td>999</td>\n","      <td></td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>28</td>\n","      <td></td>\n","      <td></td>\n","      <td>1</td>\n","      <td>6</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td></td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>71</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>5</td>\n","      <td>4</td>\n","      <td>5</td>\n","      <td>17</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>215</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>M</td>\n","      <td>58</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>23.7</td>\n","      <td>0</td>\n","      <td>3140</td>\n","      <td>5</td>\n","      <td>2017</td>\n","      <td>2</td>\n","      <td>1</td>\n","      <td>938</td>\n","      <td>2</td>\n","      <td>2018</td>\n","      <td>138</td>\n","      <td>26</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>43</td>\n","      <td>888</td>\n","      <td>43</td>\n","      <td></td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>23</td>\n","      <td></td>\n","      <td></td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td></td>\n","      <td>3</td>\n","      <td>N</td>\n","      <td>64</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>5</td>\n","      <td>6</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>138</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>F</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>35.5</td>\n","      <td>0</td>\n","      <td>2125</td>\n","      <td>99</td>\n","      <td>9999</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>830</td>\n","      <td>3</td>\n","      <td>2018</td>\n","      <td>219</td>\n","      <td>35</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>999</td>\n","      <td>999</td>\n","      <td>999</td>\n","      <td></td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>37</td>\n","      <td></td>\n","      <td></td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td></td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>66</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>5</td>\n","      <td>15</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>220</td>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>M</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   ATTEND  BFACIL   BMI  CIG_0  DBWT  DLMP_MM  DLMP_YY DMAR  DOB_MM  DOB_TT  \\\n","0       1       1  30.7      0  3657        4     2017    1       1    1227   \n","1       1       1  33.3      2  3242       99     9999    2       1    1704   \n","2       1       1  30.0      0  3470        4     2017    1       1     336   \n","3       3       1  23.7      0  3140        5     2017    2       1     938   \n","4       1       1  35.5      0  2125       99     9999    1       1     830   \n","\n","   DOB_WK  DOB_YY  DWgt_R  FAGECOMB  FEDUC  FHISPX  FRACE15  FRACE31  FRACE6  \\\n","0       2    2018     231        31      3       1        1        1       1   \n","1       2    2018     185        35      4       0        3        3       3   \n","2       2    2018     273        31      4       0        1        1       1   \n","3       2    2018     138        26      2       0        3        3       3   \n","4       3    2018     219        35      3       0        2        2       2   \n","\n","   ILLB_R  ILOP_R  ILP_R IMP_SEX IP_GON LD_INDL  MAGER MAGE_IMPFLG MAR_IMP  \\\n","0      16      33     16              N       N     30                       \n","1     180     888    180              N       N     35                       \n","2     999     888    999              N       N     28                       \n","3      43     888     43              N       N     23                       \n","4     999     999    999              N       N     37                       \n","\n","   MBSTATE_REC  MEDUC  MHISPX MM_AICU  MRACE15  MRACE31 MRACEIMP  MRAVE6  \\\n","0            1      6       0       N        1        1                1   \n","1            1      9       0       N        3        3                3   \n","2            1      6       0       N        1        1                1   \n","3            1      2       0       N        3        3                3   \n","4            1      4       0       N        1        1                1   \n","\n","  MTRAN  M_Ht_In  NO_INFEC  NO_MMORB  NO_RISKS  PAY  PAY_REC  PRECARE  PREVIS  \\\n","0     N       66         1         1         1    2        2        3       8   \n","1     N       63         1         1         0    1        1        3       9   \n","2     N       71         1         1         0    5        4        5      17   \n","3     N       64         1         1         1    1        1        5       6   \n","4     N       66         1         1         1    1        1        5      15   \n","\n","   PRIORDEAD  PRIORLIVE  PRIORTERM  PWgt_R  RDMETH_REC  RESTATUS RF_CESAR  \\\n","0          0          1          2     190           1         2        N   \n","1          0          2          0     188           4         2        Y   \n","2          0          1          0     215           1         1        N   \n","3          0          2          0     138           1         2        N   \n","4          0          1          4     220           3         1        N   \n","\n","   RF_CESARN SEX  WTGAIN  \n","0          0   M      41  \n","1          2   F       0  \n","2          0   M      58  \n","3          0   F       0  \n","4          0   M       0  "]},"metadata":{"tags":[]},"execution_count":56}]},{"cell_type":"markdown","metadata":{"id":"TQaJRt9sLB9o"},"source":["# ***Preprocessing***"]},{"cell_type":"code","metadata":{"id":"FBXsR9JQ3voj"},"source":["df['MRACE6'] = df['MRAVE6']\n","df = df.drop(columns=['MRAVE6'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"moY0IwT-LQyW"},"source":["## ***Droping null columns***"]},{"cell_type":"code","metadata":{"id":"VT-sjApWLMl_"},"source":["df = df.drop(columns=['IMP_SEX', 'MAGE_IMPFLG', 'MAR_IMP', 'MRACEIMP'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"qA3MZ1nawX0G"},"source":["## ***Droping unnecessary columns based on meaning***"]},{"cell_type":"markdown","metadata":{"id":"_J_FBZYFwsE7"},"source":["### ***ATTEND: Attendant at Birth like Doctor of Medicine or Midwife***"]},{"cell_type":"code","metadata":{"id":"aF1cxcizw1Dt"},"source":["df = df.drop(columns=['ATTEND'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"zHOJAISrxOCy"},"source":["### ***BFACIL: Birth Place like Hospital***"]},{"cell_type":"code","metadata":{"id":"c5x2q3-7xeww"},"source":["df = df.drop(columns=['BFACIL'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"5-Qpqq5Qxi8P"},"source":["### ***DOB_TT: Time Of Birth***"]},{"cell_type":"code","metadata":{"id":"x-i9Dn_iyZY5"},"source":["df = df.drop(columns=['DOB_TT'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"YopQizKPyeF2"},"source":["### ***DOB_WK: Birth Day of Week***"]},{"cell_type":"code","metadata":{"id":"Lsh2-JlOynnt"},"source":["df = df.drop(columns=['DOB_WK'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Jgp4p-fYyp4W"},"source":["### ***DOB_YY: Birth Year, Always 2018***"]},{"cell_type":"code","metadata":{"id":"fSnVYm-Hy2f1"},"source":["df = df.drop(columns=['DOB_YY'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"rKsoqJr4y5qi"},"source":["### ***DWgt_R: Mother Delivery Weight***"]},{"cell_type":"code","metadata":{"id":"CtYrB0cr2-ei"},"source":["df = df.drop(columns=['DWgt_R'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"9DSMU6ZY4q3D"},"source":["### ***RDMETH_REC: Delivery Method***"]},{"cell_type":"code","metadata":{"id":"H7JsHUD74yvL"},"source":["df = df.drop(columns=['RDMETH_REC'])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"348g8NqD41Jo"},"source":["## ***Working on null values***"]},{"cell_type":"markdown","metadata":{"id":"I7FEEB_e8b_3"},"source":["### ***BMI***"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pSxwNZTf8bAN","executionInfo":{"status":"ok","timestamp":1623170803099,"user_tz":-270,"elapsed":615,"user":{"displayName":"Mohammad Mirsafaei","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgMMfkaVHqEsQp4cQUbu20VYNCJm5MedDnb8-sOcA=s64","userId":"14901075838947117855"}},"outputId":"93d7edfc-4025-4977-b691-181ea057f5dc"},"source":["df.loc[(df['BMI'] == 99.9) & (df['PWgt_R'] != 999) & (df['M_Ht_In'] != 99)]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>BMI</th>\n","      <th>CIG_0</th>\n","      <th>DBWT</th>\n","      <th>DLMP_MM</th>\n","      <th>DLMP_YY</th>\n","      <th>DMAR</th>\n","      <th>DOB_MM</th>\n","      <th>FAGECOMB</th>\n","      <th>FEDUC</th>\n","      <th>FHISPX</th>\n","      <th>FRACE15</th>\n","      <th>FRACE31</th>\n","      <th>FRACE6</th>\n","      <th>ILLB_R</th>\n","      <th>ILOP_R</th>\n","      <th>ILP_R</th>\n","      <th>IP_GON</th>\n","      <th>LD_INDL</th>\n","      <th>MAGER</th>\n","      <th>MBSTATE_REC</th>\n","      <th>MEDUC</th>\n","      <th>MHISPX</th>\n","      <th>MM_AICU</th>\n","      <th>MRACE15</th>\n","      <th>MRACE31</th>\n","      <th>MTRAN</th>\n","      <th>M_Ht_In</th>\n","      <th>NO_INFEC</th>\n","      <th>NO_MMORB</th>\n","      <th>NO_RISKS</th>\n","      <th>PAY</th>\n","      <th>PAY_REC</th>\n","      <th>PRECARE</th>\n","      <th>PREVIS</th>\n","      <th>PRIORDEAD</th>\n","      <th>PRIORLIVE</th>\n","      <th>PRIORTERM</th>\n","      <th>PWgt_R</th>\n","      <th>RESTATUS</th>\n","      <th>RF_CESAR</th>\n","      <th>RF_CESARN</th>\n","      <th>SEX</th>\n","      <th>WTGAIN</th>\n","      <th>MRACE6</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>380</th>\n","      <td>99.9</td>\n","      <td>0</td>\n","      <td>3629</td>\n","      <td>99</td>\n","      <td>9999</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>39</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>37</td>\n","      <td>888</td>\n","      <td>37</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>28</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>61</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>8</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>75</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>M</td>\n","      <td>38</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>12582</th>\n","      <td>99.9</td>\n","      <td>0</td>\n","      <td>3033</td>\n","      <td>4</td>\n","      <td>2017</td>\n","      <td>2</td>\n","      <td>1</td>\n","      <td>28</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>84</td>\n","      <td>888</td>\n","      <td>84</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>28</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>55</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>15</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>340</td>\n","      <td>1</td>\n","      <td>Y</td>\n","      <td>3</td>\n","      <td>M</td>\n","      <td>3</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>16287</th>\n","      <td>99.9</td>\n","      <td>10</td>\n","      <td>2523</td>\n","      <td>5</td>\n","      <td>2017</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>24</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>189</td>\n","      <td>49</td>\n","      <td>49</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>33</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>64</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>11</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>75</td>\n","      <td>2</td>\n","      <td>Y</td>\n","      <td>1</td>\n","      <td>M</td>\n","      <td>41</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>25599</th>\n","      <td>99.9</td>\n","      <td>30</td>\n","      <td>2409</td>\n","      <td>4</td>\n","      <td>2017</td>\n","      <td>2</td>\n","      <td>4</td>\n","      <td>18</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>888</td>\n","      <td>46</td>\n","      <td>46</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>21</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>70</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>6</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>90</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>F</td>\n","      <td>38</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>34769</th>\n","      <td>99.9</td>\n","      <td>0</td>\n","      <td>2636</td>\n","      <td>10</td>\n","      <td>2017</td>\n","      <td>2</td>\n","      <td>6</td>\n","      <td>30</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>75</td>\n","      <td>888</td>\n","      <td>75</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>28</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>60</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>4</td>\n","      <td>13</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>375</td>\n","      <td>1</td>\n","      <td>Y</td>\n","      <td>1</td>\n","      <td>F</td>\n","      <td>19</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>3765541</th>\n","      <td>99.9</td>\n","      <td>0</td>\n","      <td>4167</td>\n","      <td>12</td>\n","      <td>2017</td>\n","      <td>1</td>\n","      <td>9</td>\n","      <td>39</td>\n","      <td>6</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>63</td>\n","      <td>888</td>\n","      <td>63</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>38</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>63</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>11</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>375</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>M</td>\n","      <td>50</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>3768792</th>\n","      <td>99.9</td>\n","      <td>0</td>\n","      <td>3121</td>\n","      <td>12</td>\n","      <td>2017</td>\n","      <td>2</td>\n","      <td>9</td>\n","      <td>99</td>\n","      <td>9</td>\n","      <td>9</td>\n","      <td>99</td>\n","      <td>99</td>\n","      <td>9</td>\n","      <td>888</td>\n","      <td>888</td>\n","      <td>888</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>23</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>38</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>7</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>210</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>F</td>\n","      <td>49</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>3776407</th>\n","      <td>99.9</td>\n","      <td>0</td>\n","      <td>3420</td>\n","      <td>1</td>\n","      <td>2018</td>\n","      <td>1</td>\n","      <td>10</td>\n","      <td>38</td>\n","      <td>6</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>38</td>\n","      <td>888</td>\n","      <td>38</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>28</td>\n","      <td>2</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>67</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>9</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>76</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>M</td>\n","      <td>92</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>3779188</th>\n","      <td>99.9</td>\n","      <td>0</td>\n","      <td>3202</td>\n","      <td>1</td>\n","      <td>2018</td>\n","      <td>1</td>\n","      <td>10</td>\n","      <td>39</td>\n","      <td>7</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>20</td>\n","      <td>888</td>\n","      <td>20</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>35</td>\n","      <td>2</td>\n","      <td>8</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>78</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>9</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>114</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>M</td>\n","      <td>38</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3781234</th>\n","      <td>99.9</td>\n","      <td>0</td>\n","      <td>2950</td>\n","      <td>1</td>\n","      <td>2018</td>\n","      <td>2</td>\n","      <td>10</td>\n","      <td>25</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>82</td>\n","      <td>999</td>\n","      <td>999</td>\n","      <td>N</td>\n","      <td>N</td>\n","      <td>26</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>N</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>N</td>\n","      <td>78</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>9</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>112</td>\n","      <td>1</td>\n","      <td>N</td>\n","      <td>0</td>\n","      <td>F</td>\n","      <td>30</td>\n","      <td>2</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>428 rows × 44 columns</p>\n","</div>"],"text/plain":["          BMI  CIG_0  DBWT  DLMP_MM  DLMP_YY DMAR  DOB_MM  FAGECOMB  FEDUC  \\\n","380      99.9      0  3629       99     9999    1       1        39      3   \n","12582    99.9      0  3033        4     2017    2       1        28      2   \n","16287    99.9     10  2523        5     2017    2       2        24      3   \n","25599    99.9     30  2409        4     2017    2       4        18      1   \n","34769    99.9      0  2636       10     2017    2       6        30      3   \n","...       ...    ...   ...      ...      ...  ...     ...       ...    ...   \n","3765541  99.9      0  4167       12     2017    1       9        39      6   \n","3768792  99.9      0  3121       12     2017    2       9        99      9   \n","3776407  99.9      0  3420        1     2018    1      10        38      6   \n","3779188  99.9      0  3202        1     2018    1      10        39      7   \n","3781234  99.9      0  2950        1     2018    2      10        25      2   \n","\n","         FHISPX  FRACE15  FRACE31  FRACE6  ILLB_R  ILOP_R  ILP_R IP_GON  \\\n","380           0        1        1       1      37     888     37      N   \n","12582         0        2        2       2      84     888     84      N   \n","16287         0        2        2       2     189      49     49      N   \n","25599         0        1        1       1     888      46     46      N   \n","34769         0        2        2       2      75     888     75      N   \n","...         ...      ...      ...     ...     ...     ...    ...    ...   \n","3765541       0        2        2       2      63     888     63      N   \n","3768792       9       99       99       9     888     888    888      N   \n","3776407       0        2        2       2      38     888     38      N   \n","3779188       0        1        1       1      20     888     20      N   \n","3781234       0        2        2       2      82     999    999      N   \n","\n","        LD_INDL  MAGER  MBSTATE_REC  MEDUC  MHISPX MM_AICU  MRACE15  MRACE31  \\\n","380           N     28            1      2       0       N        1        1   \n","12582         N     28            1      2       0       N        2        2   \n","16287         N     33            1      4       0       N        2        2   \n","25599         N     21            1      3       0       N        1        1   \n","34769         N     28            1      4       0       N        2        2   \n","...         ...    ...          ...    ...     ...     ...      ...      ...   \n","3765541       N     38            1      4       0       N        2        2   \n","3768792       N     23            1      3       0       N        2        2   \n","3776407       N     28            2      3       0       N        2        2   \n","3779188       N     35            2      8       0       N        1        1   \n","3781234       N     26            1      3       0       N        2        2   \n","\n","        MTRAN  M_Ht_In  NO_INFEC  NO_MMORB  NO_RISKS  PAY  PAY_REC  PRECARE  \\\n","380         N       61         1         1         1    1        1        2   \n","12582       N       55         1         1         0    1        1        2   \n","16287       N       64         0         1         0    1        1        2   \n","25599       N       70         0         1         1    1        1        2   \n","34769       N       60         1         1         0    2        2        4   \n","...       ...      ...       ...       ...       ...  ...      ...      ...   \n","3765541     N       63         1         1         1    2        2        2   \n","3768792     N       38         0         1         1    1        1        7   \n","3776407     N       67         1         1         1    1        1        3   \n","3779188     N       78         1         1         1    1        1        2   \n","3781234     N       78         1         1         1    1        1        3   \n","\n","         PREVIS  PRIORDEAD  PRIORLIVE  PRIORTERM  PWgt_R  RESTATUS RF_CESAR  \\\n","380           8          0          2          0      75         1        N   \n","12582        15          0          3          0     340         1        Y   \n","16287        11          0          1          2      75         2        Y   \n","25599         6          0          0          2      90         2        N   \n","34769        13          0          1          0     375         1        Y   \n","...         ...        ...        ...        ...     ...       ...      ...   \n","3765541      11          0          1          0     375         2        N   \n","3768792       3          0          0          0     210         1        N   \n","3776407       9          0          3          0      76         1        N   \n","3779188       9          0          2          0     114         1        N   \n","3781234       9          0          1          3     112         1        N   \n","\n","         RF_CESARN SEX  WTGAIN  MRACE6  \n","380              0   M      38       1  \n","12582            3   M       3       2  \n","16287            1   M      41       2  \n","25599            0   F      38       1  \n","34769            1   F      19       2  \n","...            ...  ..     ...     ...  \n","3765541          0   M      50       2  \n","3768792          0   F      49       2  \n","3776407          0   M      92       2  \n","3779188          0   M      38       1  \n","3781234          0   F      30       2  \n","\n","[428 rows x 44 columns]"]},"metadata":{"tags":[]},"execution_count":68}]},{"cell_type":"code","metadata":{"id":"KQejprg-8jFG"},"source":["def bmi_func(x):\n","  if (x['BMI'] == 99.9) and (x['PWgt_R'] != 999) and (x['M_Ht_In'] != 99):\n","    return round( (x['PWgt_R'] / math.pow(x['M_Ht_In'], 2) ) * 703, 1)\n","  else:\n","    return x['BMI']\n","\n","df['BMI'] = df.apply(bmi_func, axis=1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0k2l0XKvCPLH","executionInfo":{"status":"ok","timestamp":1623170850153,"user_tz":-270,"elapsed":39,"user":{"displayName":"Mohammad Mirsafaei","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgMMfkaVHqEsQp4cQUbu20VYNCJm5MedDnb8-sOcA=s64","userId":"14901075838947117855"}},"outputId":"309f6625-95b9-4579-cea9-7a71734bb988"},"source":["df.loc[(df['BMI'] == 99.9) & (df['PWgt_R'] != 999) & (df['M_Ht_In'] != 99)]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>BMI</th>\n","      <th>CIG_0</th>\n","      <th>DBWT</th>\n","      <th>DLMP_MM</th>\n","      <th>DLMP_YY</th>\n","      <th>DMAR</th>\n","      <th>DOB_MM</th>\n","      <th>FAGECOMB</th>\n","      <th>FEDUC</th>\n","      <th>FHISPX</th>\n","      <th>FRACE15</th>\n","      <th>FRACE31</th>\n","      <th>FRACE6</th>\n","      <th>ILLB_R</th>\n","      <th>ILOP_R</th>\n","      <th>ILP_R</th>\n","      <th>IP_GON</th>\n","      <th>LD_INDL</th>\n","      <th>MAGER</th>\n","      <th>MBSTATE_REC</th>\n","      <th>MEDUC</th>\n","      <th>MHISPX</th>\n","      <th>MM_AICU</th>\n","      <th>MRACE15</th>\n","      <th>MRACE31</th>\n","      <th>MTRAN</th>\n","      <th>M_Ht_In</th>\n","      <th>NO_INFEC</th>\n","      <th>NO_MMORB</th>\n","      <th>NO_RISKS</th>\n","      <th>PAY</th>\n","      <th>PAY_REC</th>\n","      <th>PRECARE</th>\n","      <th>PREVIS</th>\n","      <th>PRIORDEAD</th>\n","      <th>PRIORLIVE</th>\n","      <th>PRIORTERM</th>\n","      <th>PWgt_R</th>\n","      <th>RESTATUS</th>\n","      <th>RF_CESAR</th>\n","      <th>RF_CESARN</th>\n","      <th>SEX</th>\n","      <th>WTGAIN</th>\n","      <th>MRACE6</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["Empty DataFrame\n","Columns: [BMI, CIG_0, DBWT, DLMP_MM, DLMP_YY, DMAR, DOB_MM, FAGECOMB, FEDUC, FHISPX, FRACE15, FRACE31, FRACE6, ILLB_R, ILOP_R, ILP_R, IP_GON, LD_INDL, MAGER, MBSTATE_REC, MEDUC, MHISPX, MM_AICU, MRACE15, MRACE31, MTRAN, M_Ht_In, NO_INFEC, NO_MMORB, NO_RISKS, PAY, PAY_REC, PRECARE, PREVIS, PRIORDEAD, PRIORLIVE, PRIORTERM, PWgt_R, RESTATUS, RF_CESAR, RF_CESARN, SEX, WTGAIN, MRACE6]\n","Index: []"]},"metadata":{"tags":[]},"execution_count":70}]},{"cell_type":"markdown","metadata":{"id":"hrKI12qRYwyo"},"source":["### ***Dropping rows with null values for DLMP_MM & DLMP_YY***"]},{"cell_type":"code","metadata":{"id":"X2QtbMFwY6ix"},"source":["df.drop(df[df['DLMP_MM'].eq(99)].index, inplace=True)\n","df.drop(df[df['DLMP_YY'].eq(9999)].index, inplace=True)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"82YTtg9njQgP"},"source":["## ***Replacing null values in categorical columns with median***"]},{"cell_type":"code","metadata":{"id":"tBxzXNm-lVDo"},"source":["categorical_columns_09 = ['FEDUC', 'FHISPX',  'FRACE6', 'MEDUC', 'MHISPX', 'NO_INFEC', 'NO_MMORB', 'NO_RISKS', 'PAY', 'PAY_REC']\n","categorical_columns_99 = ['FRACE15', 'FRACE31']\n","categorical_columns_empty = ['DMAR']\n","categorical_columns_u = ['IP_GON', 'LD_INDL', 'MM_AICU', 'MTRAN', 'RF_CESAR']\n","categorical_columns_3 = ['MBSTATE_REC']"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"NN4OY98GjLwd"},"source":["imputer = SimpleImputer(missing_values=9, strategy='median')\n","df[categorical_columns_09] = imputer.fit_transform(df[categorical_columns_09])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"WMkCWUu6XAbp"},"source":["imputer = SimpleImputer(missing_values=99, strategy='median')\n","df[categorical_columns_99] = imputer.fit_transform(df[categorical_columns_99])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-vjgf4AkXa0j"},"source":["imputer = SimpleImputer(missing_values=\" \", strategy='constant', fill_value=\"1\")\n","df[categorical_columns_empty] = imputer.fit_transform(df[categorical_columns_empty])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ykfjadeNxRqL"},"source":["imputer = SimpleImputer(missing_values=\"U\", strategy='constant', fill_value=\"N\")\n","df[categorical_columns_u] = imputer.fit_transform(df[categorical_columns_u])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"EofcF0EUxV85"},"source":["imputer = SimpleImputer(missing_values=3, strategy='median')\n","df[categorical_columns_3] = imputer.fit_transform(df[categorical_columns_3])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ac8V7lRf8Rch"},"source":["## ***Replacing null values in numerical columns with mean***"]},{"cell_type":"code","metadata":{"id":"LxnejSfR-XwC"},"source":["numerical_columns_99_9 = ['BMI']\n","numerical_columns_99 = ['CIG_0', 'FAGECOMB', 'M_Ht_In', 'PRECARE', 'PREVIS', 'PRIORDEAD', 'PRIORLIVE', 'PRIORTERM', 'RF_CESARN', 'WTGAIN']\n","numerical_columns_999 = ['ILLB_R', 'ILOP_R', 'ILP_R', 'PWgt_R']"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"BAcMdSak8d1d"},"source":["imputer = SimpleImputer(missing_values=99.9, strategy='mean')\n","df[numerical_columns_99_9] = imputer.fit_transform(df[numerical_columns_99_9])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"N28QMZKIepKS"},"source":["imputer = SimpleImputer(missing_values=99, strategy='mean')\n","df[numerical_columns_99] = imputer.fit_transform(df[numerical_columns_99])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0gqFcEjn5-D7"},"source":["imputer = SimpleImputer(missing_values=99, strategy='mean')\n","df[numerical_columns_999] = imputer.fit_transform(df[numerical_columns_999])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"WslJfzKsMXKT"},"source":["## ***New Features***"]},{"cell_type":"markdown","metadata":{"id":"xBjELsf1MiXK"},"source":["### ***Smoked***"]},{"cell_type":"code","metadata":{"id":"dYOwu9z1Mh_0"},"source":["df['smoked'] = np.where(df['CIG_0'].eq(0), 'N', 'Y')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ZSyoKHEKTZNH"},"source":["### ***Pregnancy Length***"]},{"cell_type":"code","metadata":{"id":"D_kLI8-ETpPm"},"source":["conditions = [(df['DOB_MM'] > df['DLMP_MM']) & (2018 == df['DLMP_YY']),\n","                  (df['DOB_MM'] > df['DLMP_MM']) & (2018 > df['DLMP_YY']),\n","                  (df['DOB_MM'] < df['DLMP_MM']) & (2018 > df['DLMP_YY'])]\n","choices = [df['DOB_MM'] - df['DLMP_MM'],\n","          ((2018 - df['DLMP_YY'])* 12) + df['DOB_MM'] - df['DLMP_MM'],\n","          ((2018 - df['DLMP_YY'])* 12) - df['DLMP_MM'] + df['DOB_MM']]\n","df['pregnancy_length'] = np.select(conditions,choices, 12)\n","\n","df.drop(df[df['pregnancy_length'].gt(12)].index,inplace=True)\n","df.drop(df[df['pregnancy_length'].lt(5)].index,inplace=True)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"OAq29jZMTxL0"},"source":["### ***First Live Birth***"]},{"cell_type":"code","metadata":{"id":"wNPPnoLCUy33"},"source":["df['first_live_birth'] = np.where(df['ILP_R'].eq(888), \"Y\", \"N\")"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"OKDw2dvPU54E"},"source":["### ***First Birth***"]},{"cell_type":"code","metadata":{"id":"YJKT6OtcU_oA"},"source":["df['first_birth'] = np.where(df['ILLB_R'].eq(888), \"Y\", \"N\")"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"jazg2sXcZfgi"},"source":["### ***Had PriorDead***"]},{"cell_type":"code","metadata":{"id":"gGQTIxg5ZmYL"},"source":["df['had_priordead'] = np.where(df['PRIORDEAD'].eq(0), 'N', 'Y')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"EzDyAVlxZ1A2"},"source":["## ***Target variable***"]},{"cell_type":"markdown","metadata":{"id":"-ni6HKAmZ7C1"},"source":["### ***Option 1: Dropping rows with null value***"]},{"cell_type":"code","metadata":{"id":"c4JJFF0EaW6V"},"source":["df.drop(df[df['DBWT'].eq(9999)].index, inplace=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"khyyCIQKfcte"},"source":["df.to_csv('/content/drive/MyDrive/FDM/Preprocessed/df_droped.csv', index=False)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"hc8JScjSan_t"},"source":["### ***Option 2: Replacing with mean***"]},{"cell_type":"code","metadata":{"id":"kpkTBiJnatYN"},"source":["imputer = SimpleImputer(missing_values=9999, strategy='mean')\n","df['DBWT'] = imputer.fit_transform(df[['DBWT']])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uF5BduiTgGpM"},"source":["df.to_csv('/content/drive/MyDrive/FDM/Preprocessed/df.csv', index=False)"],"execution_count":null,"outputs":[]}]}