test2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2142accb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d3c8589f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_discharge = pd.read_csv(\"C:/Users/zhi-n/Documents/FIT3163/mimic-iv-note/discharge.csv\")\n",
    "df_discharge = pd.read_csv(\"discharge.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6c919653",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cab3e7de",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    # Remove '\\n' characters and extra spaces\n",
    "    cleaned_text = re.sub(r'\\n+', '\\n', text)\n",
    "    cleaned_text = re.sub(r' +', ' ', cleaned_text)\n",
    "    return cleaned_text.strip()\n",
    "\n",
    "# Apply the clean_text function to the 'text' column\n",
    "df_discharge['text'] = df_discharge['text'].apply(clean_text)\n",
    "\n",
    "# Set the display option to show the full content of the 'text' column\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "28ed24c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          note_id  subject_id   hadm_id note_type  note_seq  \\\n",
      "0  10000032-DS-21    10000032  22595853        DS        21   \n",
      "\n",
      "             charttime            storetime  \\\n",
      "0  2180-05-07 00:00:00  2180-05-09 15:26:00   \n",
      "\n",
      "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         text  \n",
      "0  Name: ___ Unit No: ___\\n \\nAdmission Date: ___ Discharge Date: ___\\n \\nDate of Birth: ___ Sex: F\\n \\nService: MEDICINE\\n \\nAllergies: \\nNo Known Allergies / Adverse Drug Reactions\\n \\nAttending: ___\\n \\nChief Complaint:\\nWorsening ABD distension and pain \\n \\nMajor Surgical or Invasive Procedure:\\nParacentesis\\n \\nHistory of Present Illness:\\n___ HCV cirrhosis c/b ascites, hiv on ART, h/o IVDU, COPD, \\nbioplar, PTSD, presented from OSH ED with worsening abd \\ndistension over past week. \\nPt reports self-discontinuing lasix and spirnolactone ___ weeks \\nago, because she feels like \"they don't do anything\" and that \\nshe \"doesn't want to put more chemicals in her.\" She does not \\nfollow Na-restricted diets. In the past week, she notes that she \\nhas been having worsening abd distension and discomfort. She \\ndenies ___ edema, or SOB, or orthopnea. She denies f/c/n/v, d/c, \\ndysuria. She had food poisoning a week ago from eating stale \\ncake (n/v 20 min after food ingestion), which resolved the same \\nday. She denies other recent illness or sick contacts. She notes \\nthat she has been noticing gum bleeding while brushing her teeth \\nin recent weeks. she denies easy bruising, melena, BRBPR, \\nhemetesis, hemoptysis, or hematuria. \\nBecause of her abd pain, she went to OSH ED and was transferred \\nto ___ for further care. Per ED report, pt has brief period of \\nconfusion - she did not recall the ultrasound or bloodwork at \\nosh. She denies recent drug use or alcohol use. She denies \\nfeeling confused, but reports that she is forgetful at times. \\nIn the ED, initial vitals were 98.4 70 106/63 16 97%RA \\nLabs notable for ALT/AST/AP ___ ___: ___, \\nTbili1.6, WBC 5K, platelet 77, INR 1.6 \\n \\nPast Medical History:\\n1. HCV Cirrhosis \\n2. No history of abnormal Pap smears. \\n3. She had calcification in her breast, which was removed \\npreviously and per patient not, it was benign. \\n4. For HIV disease, she is being followed by Dr. ___ Dr. \\n___. \\n5. COPD \\n6. Past history of smoking. \\n7. She also had a skin lesion, which was biopsied and showed \\nskin cancer per patient report and is scheduled for a complete \\nremoval of the skin lesion in ___ of this year. \\n8. She also had another lesion in her forehead with purple \\ndiscoloration. It was biopsied to exclude the possibility of \\n___'s sarcoma, the results is pending. \\n9. A 15 mm hypoechoic lesion on her ultrasound on ___ \\nand is being monitored by an MRI. \\n10. History of dysplasia of anus in ___. \\n11. Bipolar affective disorder, currently manic, mild, and PTSD. \\n \\n12. History of cocaine and heroin use. \\n \\nSocial History:\\n___\\nFamily History:\\nShe a total of five siblings, but she is not \\ntalking to most of them. She only has one brother that she is in \\n \\ntouch with and lives in ___. She is not aware of any \\nknown GI or liver disease in her family. \\nHer last alcohol consumption was one drink two months ago. No \\nregular alcohol consumption. Last drug use ___ years ago. She \\nquit smoking a couple of years ago. \\n \\nPhysical Exam:\\nVS: 98.1 107/61 78 18 97RA \\nGeneral: in NAD \\nHEENT: CTAB, anicteric sclera, OP clear \\nNeck: supple, no LAD \\nCV: RRR,S1S2, no m/r/g \\nLungs: CTAb, prolonged expiratory phase, no w/r/r \\nAbdomen: distended, mild diffuse tenderness, +flank dullness, \\ncannot percuss liver/spleen edge ___ distension \\nGU: no foley \\nExt: wwp, no c/e/e, + clubbing \\nNeuro: AAO3, converse normally, able to recall 3 times after 5 \\nminutes, CN II-XII intact \\nDischarge:\\nPHYSICAL EXAMINATION: \\nVS: 98 105/70 95\\nGeneral: in NAD \\nHEENT: anicteric sclera, OP clear \\nNeck: supple, no LAD \\nCV: RRR,S1S2, no m/r/g \\nLungs: CTAb, prolonged expiratory phase, no w/r/r \\nAbdomen: distended but improved, TTP in RUQ, \\nGU: no foley \\nExt: wwp, no c/e/e, + clubbing \\nNeuro: AAO3, CN II-XII intact \\n \\nPertinent Results:\\n___ 10:25PM GLUCOSE-109* UREA N-25* CREAT-0.3* SODIUM-138 \\nPOTASSIUM-3.4 CHLORIDE-105 TOTAL CO2-27 ANION GAP-9\\n___ 10:25PM estGFR-Using this\\n___ 10:25PM ALT(SGPT)-100* AST(SGOT)-114* ALK PHOS-114* \\nTOT BILI-1.6*\\n___ 10:25PM LIPASE-77*\\n___ 10:25PM ALBUMIN-3.3*\\n___ 10:25PM WBC-5.0# RBC-4.29 HGB-14.3 HCT-42.6 MCV-99* \\nMCH-33.3* MCHC-33.5 RDW-15.7*\\n___ 10:25PM NEUTS-70.3* LYMPHS-16.5* MONOS-8.1 EOS-4.2* \\nBASOS-0.8\\n___ 10:25PM PLT COUNT-71*\\n___ 10:25PM ___ PTT-30.9 ___\\n___ 10:25PM ___\\n.\\nCXR: No acute cardiopulmonary process. \\nU/S: \\n1. Nodular appearance of the liver compatible with cirrhosis. \\nSigns of portal \\nhypertension including small amount of ascites and splenomegaly. \\n \\n2. Cholelithiasis. \\n3. Patent portal veins with normal hepatopetal flow. \\nDiagnostic para attempted in the ED, unsuccessful. \\nOn the floor, pt c/o abd distension and discomfort.\\n \\nBrief Hospital Course:\\n___ HCV cirrhosis c/b ascites, hiv on ART, h/o IVDU, COPD, \\nbioplar, PTSD, presented from OSH ED with worsening abd \\ndistension over past week and confusion. \\n# Ascites - p/w worsening abd distension and discomfort for last \\nweek. likely ___ portal HTN given underlying liver disease, \\nthough no ascitic fluid available on night of admission. No \\nsigns of heart failure noted on exam. This was ___ to med \\nnon-compliance and lack of diet restriction. SBP negative\\ndiuretics: \\n> Furosemide 40 mg PO DAILY \\n> Spironolactone 50 mg PO DAILY, chosen over the usual 100mg \\ndose d/t K+ of 4.5. \\n CXR was wnl, UA negative, Urine culture blood culture negative. \\n \\nPt was losing excess fluid appropriately with stable lytes on \\nthe above regimen. Pt was scheduled with current PCP for \\n___ check upon discharge. \\nPt was scheduled for new PCP with Dr. ___ at ___ and \\nfollow up in Liver clinic to schedule outpatient screening EGD \\nand ___. \\n \\n \\nMedications on Admission:\\nThe Preadmission Medication list is accurate and complete.\\n1. Furosemide 20 mg PO DAILY \\n2. Spironolactone 50 mg PO DAILY \\n3. Albuterol Inhaler 2 PUFF IH Q4H:PRN wheezing, SOB \\n4. Raltegravir 400 mg PO BID \\n5. Emtricitabine-Tenofovir (Truvada) 1 TAB PO DAILY \\n6. Nicotine Patch 14 mg TD DAILY \\n7. Ipratropium Bromide Neb 1 NEB IH Q6H SOB \\n \\nDischarge Medications:\\n1. Albuterol Inhaler 2 PUFF IH Q4H:PRN wheezing, SOB \\n2. Emtricitabine-Tenofovir (Truvada) 1 TAB PO DAILY \\n3. Furosemide 40 mg PO DAILY \\nRX *furosemide 40 mg 1 tablet(s) by mouth Daily Disp #*30 Tablet \\nRefills:*3\\n4. Ipratropium Bromide Neb 1 NEB IH Q6H SOB \\n5. Nicotine Patch 14 mg TD DAILY \\n6. Raltegravir 400 mg PO BID \\n7. Spironolactone 50 mg PO DAILY \\n8. Acetaminophen 500 mg PO Q6H:PRN pain \\n \\nDischarge Disposition:\\nHome\\n \\nDischarge Diagnosis:\\nAscites from Portal HTN\\n \\nDischarge Condition:\\nMental Status: Clear and coherent.\\nLevel of Consciousness: Alert and interactive.\\nActivity Status: Ambulatory - Independent.\\n \\nDischarge Instructions:\\nDear Ms. ___,\\nIt was a pleasure taking care of you! You came to us with \\nstomach pain and worsening distension. While you were here we \\ndid a paracentesis to remove 1.5L of fluid from your belly. We \\nalso placed you on you 40 mg of Lasix and 50 mg of Aldactone to \\nhelp you urinate the excess fluid still in your belly. As we \\ndiscussed, everyone has a different dose of lasix required to \\nmake them urinate and it's likely that you weren't taking a high \\nenough dose. Please take these medications daily to keep excess \\nfluid off and eat a low salt diet. You will follow up with Dr. \\n___ in liver clinic and from there have your colonoscopy \\nand EGD scheduled. Of course, we are always here if you need us. \\nWe wish you all the best!\\nYour ___ Team. \\n \\nFollowup Instructions:\\n___  \n"
     ]
    }
   ],
   "source": [
    "# Print the top 10 rows of the DataFrame\n",
    "top_10_rows = df_discharge.head(1)\n",
    "print(top_10_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4dcf6e14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_discharge_detail = pd.read_csv(\"C:/Users/zhi-n/Documents/FIT3163/mimic-iv-note/discharge_detail.csv\")\n",
    "df_discharge_detail = pd.read_csv(\"discharge_detail.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "de5ac3b6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          note_id  subject_id field_name field_value  field_ordinal\n",
      "0  10000032-DS-21    10000032     author         ___              1\n",
      "1  10000032-DS-22    10000032     author         ___              1\n",
      "2  10000032-DS-23    10000032     author         ___              1\n",
      "3  10000032-DS-24    10000032     author         ___              1\n",
      "4  10000084-DS-17    10000084     author         ___              1\n",
      "5  10000117-DS-21    10000117     author         ___              1\n",
      "6  10000117-DS-22    10000117     author         ___              1\n",
      "7  10000248-DS-10    10000248     author         ___              1\n",
      "8  10000560-DS-15    10000560     author         ___              1\n",
      "9  10000764-DS-11    10000764     author         ___              1\n"
     ]
    }
   ],
   "source": [
    "top_10_rows_discharge_detail = df_discharge_detail.head(10)\n",
    "print(top_10_rows_discharge_detail)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "823a835e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All values in 'field_value' column are empty.\n",
      "['___']\n",
      "0         ___\n",
      "1         ___\n",
      "2         ___\n",
      "3         ___\n",
      "4         ___\n",
      "         ... \n",
      "331789    ___\n",
      "331790    ___\n",
      "331791    ___\n",
      "331792    ___\n",
      "331793    ___\n",
      "Name: field_value, Length: 331794, dtype: object\n"
     ]
    }
   ],
   "source": [
    "no_values_condition = (df_discharge_detail['field_value'] == '___').all()\n",
    "\n",
    "if no_values_condition:\n",
    "    print(\"All values in 'field_value' column are empty.\")\n",
    "else:\n",
    "    print(\"There are non-empty values in 'field_value' column.\")\n",
    "    \n",
    "unique_values = df_discharge_detail['field_value'].unique()\n",
    "\n",
    "# Print the unique values\n",
    "print(unique_values)\n",
    "\n",
    "field_value_column = df_discharge_detail['field_value']\n",
    "\n",
    "# Print the 'field_value' column\n",
    "print(field_value_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "05bbd7c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_radiology = pd.read_csv(\"C:/Users/zhi-n/Documents/FIT3163/mimic-iv-note/radiology.csv\")\n",
    "df_radiology = pd.read_csv(\"radiology.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "509449ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          note_id  subject_id     hadm_id note_type  note_seq  \\\n",
      "0  10000032-RR-14    10000032  22595853.0        RR        14   \n",
      "1  10000032-RR-15    10000032  22595853.0        RR        15   \n",
      "2  10000032-RR-16    10000032  22595853.0        RR        16   \n",
      "3  10000032-RR-18    10000032         NaN        RR        18   \n",
      "4  10000032-RR-20    10000032         NaN        RR        20   \n",
      "5  10000032-RR-21    10000032         NaN        RR        21   \n",
      "6  10000032-RR-22    10000032  22841357.0        RR        22   \n",
      "7  10000032-RR-23    10000032  22841357.0        RR        23   \n",
      "8  10000032-RR-24    10000032         NaN        RR        24   \n",
      "9  10000032-RR-27    10000032         NaN        RR        27   \n",
      "\n",
      "             charttime            storetime  \\\n",
      "0  2180-05-06 21:19:00  2180-05-06 23:32:00   \n",
      "1  2180-05-06 23:00:00  2180-05-06 23:26:00   \n",
      "2  2180-05-07 09:55:00  2180-05-07 11:15:00   \n",
      "3  2180-06-03 12:46:00  2180-06-03 14:01:00   \n",
      "4  2180-07-08 13:18:00  2180-07-08 14:15:00   \n",
      "5  2180-06-30 08:16:00  2180-06-30 12:58:00   \n",
      "6  2180-06-26 17:15:00  2180-06-26 19:28:00   \n",
      "7  2180-06-26 17:17:00  2180-06-26 17:28:00   \n",
      "8  2180-07-22 12:40:00  2180-07-22 18:33:00   \n",
      "9  2180-07-12 13:35:00  2180-07-12 17:29:00   \n",
      "\n",
      "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 text  \n",
      "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               EXAMINATION:  CHEST (PA AND LAT)\\n\\nINDICATION:  ___ with new onset ascites  // eval for infection\\n\\nTECHNIQUE:  Chest PA and lateral\\n\\nCOMPARISON:  None.\\n\\nFINDINGS: \\n\\nThere is no focal consolidation, pleural effusion or pneumothorax.  Bilateral\\nnodular opacities that most likely represent nipple shadows. The\\ncardiomediastinal silhouette is normal.  Clips project over the left lung,\\npotentially within the breast. The imaged upper abdomen is unremarkable.\\nChronic deformity of the posterior left sixth and seventh ribs are noted.\\n\\nIMPRESSION: \\n\\nNo acute cardiopulmonary process.\\n  \n",
      "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         EXAMINATION:  LIVER OR GALLBLADDER US (SINGLE ORGAN)\\n\\nINDICATION:  ___ year-old female with cirrhosis, jaundice.\\n\\nTECHNIQUE:  Grey scale and color Doppler ultrasound images of the abdomen were\\nobtained.\\n\\nCOMPARISON:  None.\\n\\nFINDINGS: \\n\\nLIVER: The liver is coarsened and nodular in echotexture. There is no focal\\nliver mass. Main portal vein and its major branches are patent with normal\\nhepatopetal flow.  The main hepatic artery shows normal arterial waveform. \\nThere is a small amount of ascites.\\n\\nBILE DUCTS: There is no intrahepatic biliary dilation. The CBD measures 4 mm.\\n\\nGALLBLADDER: The gallbladder is contracted with a shadowing gallstone.\\n\\nPANCREAS: Imaged portion of the pancreas appears within normal limits, without\\nmasses or pancreatic ductal dilation, with portions of the pancreatic tail\\nobscured by overlying bowel gas.\\n\\nSPLEEN: Normal echogenicity, measuring 13.5 cm.\\n\\nKIDNEYS: The right kidney measures 12.1 cm. The left kidney measures 13.4 cm.\\nNormal cortical echogenicity and corticomedullary differentiation is seen\\nbilaterally.\\nRETROPERITONEUM: Visualized portions of aorta and IVC are within normal\\nlimits.\\n\\nIMPRESSION: \\n\\n1. Nodular appearance of the liver compatible with cirrhosis. Signs of portal\\nhypertension including small amount of ascites and splenomegaly.\\n2.  Cholelithiasis.\\n3.  Patent portal veins with normal hepatopetal flow.\\n  \n",
      "2                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          INDICATION:  ___ HCV cirrhosis c/b ascites, hiv on ART, h/o IVDU, COPD,\\nbioplar, PTSD, presented from OSH ED with worsening abd distension over past\\nweek.  // SBP\\n\\nTECHNIQUE:  Ultrasound guided diagnostic and therapeutic paracentesis\\n\\nCOMPARISON:  Abdominal ultrasound ___\\n\\nFINDINGS: \\n\\nLimited grayscale ultrasound imaging of the abdomen demonstrated\\nmoderateascites. A suitable target in the deepest pocket in the right lower\\nquadrant was selected for paracentesis.\\n\\nPROCEDURE:  The procedure, risks, benefits and alternatives were discussed\\nwith the patient and written informed consent was obtained.\\n\\nA preprocedure time-out was performed discussing the planned procedure,\\nconfirming the patient's identity with 3 identifiers, and reviewing a\\nchecklist per ___ protocol.\\n\\nUnder ultrasound guidance, an entrance site was selected and the skin was\\nprepped and draped in the usual sterile fashion. 1% lidocaine was instilled\\nfor local anesthesia.\\n\\nA 5 ___ catheter was advanced into the largest fluid pocket in the right\\nlower quadrant and 1.5 L of serosanguinous fluid was removed.\\n\\nThe patient tolerated the procedure well without immediate complication.\\nEstimated blood loss was minimal.  A sample of the fluid was sent to the lab\\nas requested.\\n\\nDr. ___ attending radiologist, was present throughout the critical\\nportions of the procedure.\\n\\nIMPRESSION: \\n\\nSuccessful uncomplicated ultrasound guided diagnostic and therapeutic\\nparacentesis yielding 1.5 L of serosanguineous fluid from the right lower\\nquadrant.  Sample was sent to the lab as requested.\\n  \n",
      "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        EXAMINATION:  Ultrasound-guided paracentesis.\\n\\nINDICATION:  ___ year old woman with cirrhosis, ascites despite diuretics  //\\nPlease do LVP with albumin post per protocol\\n\\nTECHNIQUE:  Ultrasound guided therapeutic paracentesis\\n\\nCOMPARISON:  ___.\\n\\nFINDINGS: \\n\\nLimited grayscale ultrasound imaging of the abdomen demonstrated\\nmoderateascites. A suitable target in the deepest pocket in the right lower\\nquadrant was selected for paracentesis.\\n\\nPROCEDURE:  The procedure, risks, benefits and alternatives were discussed\\nwith the patient and written informed consent was obtained.\\n\\nA preprocedure time-out was performed discussing the planned procedure,\\nconfirming the patient's identity with 3 identifiers, and reviewing a\\nchecklist per ___ protocol.\\n\\nUnder ultrasound guidance, an entrance site was selected and the skin was\\nprepped and draped in the usual sterile fashion. 1% lidocaine was instilled\\nfor local anesthesia.\\n\\nA 5 ___ catheter was advanced into the largest fluid pocket in the right\\nlower quadrant and 1.75 L of clear, straw-coloredfluid was removed.\\n\\nThe patient tolerated the procedure well without immediate complication.\\nEstimated blood loss was minimal.\\n\\nDr. ___ attending radiologist, personally supervised the procedure,\\nsubsequently reviewing and has agreed with the preliminary findings.\\n\\nIMPRESSION: \\n\\nUneventful therapeutic paracentesis yielding 1.75 L of clear, straw-colored\\nascitic fluid.\\n  \n",
      "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             EXAMINATION:  Paracentesis\\n\\nINDICATION:  ___ year old woman with cirrhosis, ascites despite diuretics  //\\nPlease do LVP with albumin post per protocol\\n\\nTECHNIQUE:  Ultrasound guided therapeutic paracentesis\\n\\nCOMPARISON:  Paracentesis on ___\\n\\nFINDINGS: \\n\\nLimited grayscale ultrasound imaging of the abdomen demonstrated a large\\namount of ascites. A suitable target in the deepest pocket in the right lower\\nquadrant was selected for paracentesis.\\n\\nPROCEDURE:  The procedure, risks, benefits and alternatives were discussed\\nwith the patient and written informed consent was obtained.\\n\\nA preprocedure time-out was performed discussing the planned procedure,\\nconfirming the patient's identity with 3 identifiers, and reviewing a\\nchecklist per ___ protocol.\\n\\nUnder ultrasound guidance, an entrance site was selected and the skin was\\nprepped and draped in the usual sterile fashion. 1% lidocaine was instilled\\nfor local anesthesia.\\n\\nA 5 ___ catheter was advanced into the largest fluid pocket in the right\\nlower quadrant and 4.75 L of slightly cloudy, blood tinged, yellow fluid was\\nremoved, similar to the prior exam in appearance.\\n\\nThe patient tolerated the procedure well without immediate complication.\\nEstimated blood loss was minimal.\\n\\nDr. ___ supervised the trainee during the key components of the\\nprocedure and reviewed and agrees with the trainee's findings.\\n\\nIMPRESSION: \\n\\n4.75 L of slightly cloudy, blood tinged fluid was removed from the right lower\\nquadrant without complication.\\n  \n",
      "5                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              EXAMINATION:  ULTRASOUND INTERVENTIONAL PROCEDURE\\n\\nINDICATION:  ___ year old woman with cirrhosis, ascites despite diuretics  //\\nPlease do LVP with albumin post per protocol. Please schedule on ___ or\\n___\\n\\nTECHNIQUE:  Ultrasound guided therapeutic paracentesis\\n\\nCOMPARISON:  ___.\\n\\nFINDINGS: \\n\\nLimited grayscale ultrasound imaging of the abdomen demonstrated\\nmoderateascites. A suitable target in the deepest pocket in the right lower\\nquadrant was selected for paracentesis.\\n\\nPROCEDURE:  The procedure, risks, benefits and alternatives were discussed\\nwith the patient and existing annual signed consent was reviewed.\\n\\nA preprocedure time-out was performed discussing the planned procedure,\\nconfirming the patient's identity with 3 identifiers, and reviewing a\\nchecklist per ___ protocol.\\n\\nUnder ultrasound guidance, an entrance site was selected and the skin was\\nprepped and draped in the usual sterile fashion. 1% lidocaine was instilled\\nfor local anesthesia.\\n\\nA 5 ___ catheter was advanced into the largest fluid pocket in the right\\nlower quadrant and 3.0 L of slightly cloudy, blood-tinged, yellow fluid was\\nremoved.\\n\\nThe patient tolerated the procedure well without immediate complication.\\nEstimated blood loss was minimal.\\n\\nDr. ___ supervised the trainee during the key components of the\\nprocedure and reviewed and agrees with the trainee's findings.\\n\\nIMPRESSION: \\n\\nUltrasound-guided therapeutic paracentesis with removal of 3.0 L of ascitic\\nfluid.\\n  \n",
      "6                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   EXAMINATION:  LIVER OR GALLBLADDER US (SINGLE ORGAN)\\n\\nINDICATION:  History: ___ with cirrhosis, increased abdominal pain\\n\\nTECHNIQUE:  Grey scale and color Doppler ultrasound images of the right upper\\nquadrant were obtained.\\n\\nCOMPARISON:  Abdominal ultrasound from ___\\n\\nFINDINGS: \\n\\nThe liver is extremely course and nodular in echotexture similar to the prior\\nexamination consistent with a history of cirrhosis. Parenchymal heterogeneity\\nlimits detection focal lesions. Note is made of a dominant nodule measuring\\n3.3 x 2.7 cm exerting mass effect on the gallbladder, relatively unchanged\\nfrom the prior examination. There is no intrahepatic biliary dilation. The CBD\\nmeasures 5 mm. Main, right anterior and right posterior portal veins are\\npatent with hepatopetal flow. The gallbladder is normal without stones or wall\\nthickening. Pancreas is not well evaluated. The spleen measures 11.9 cm in\\nlength and has homogeneous echotexture. There is moderate ascites throughout\\nthe abdomen.\\n\\nIMPRESSION: \\n\\n1. Extremely coarse and nodular liver echotexture consistent with cirrhosis.\\n\\n2.  Moderate ascites.\\n\\n3.  Patent portal vein.\\n  \n",
      "7                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 EXAMINATION:  CHEST (PA AND LAT)\\n\\nINDICATION:  History: ___ with shortness of breath\\n\\nTECHNIQUE:  Chest PA and lateral\\n\\nCOMPARISON:  ___\\n\\nFINDINGS: \\n\\nThe cardiac, mediastinal and hilar contours are normal. Pulmonary vasculature\\nis normal.  Lungs are clear. No pleural effusion or pneumothorax is present.\\nMultiple clips are again seen projecting over the left breast.  Remote\\nleft-sided rib fractures are also re- demonstrated.\\n\\nIMPRESSION: \\n\\nNo acute cardiopulmonary abnormality.\\n  \n",
      "8                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           EXAMINATION:  Ultrasound-guided paracentesis.\\n\\nINDICATION:  A ___ woman with cirrhosis and recurrent ascites\\npresenting for therapeutic paracentesis.\\n\\nTECHNIQUE:  Ultrasound guided therapeutic paracentesis.\\n\\nCOMPARISON:  Ultrasound-guided paracentesis performed ___.\\n\\nFINDINGS: \\n\\nLimited grayscale ultrasound imaging of the abdomen demonstrated\\nmoderateascites. A suitable target in the deepest pocket in the right lower\\nquadrant was selected for paracentesis.\\n\\nPROCEDURE:  The procedure, risks, benefits and alternatives were discussed\\nwith the patient and existing annual signed consent was reviewed.\\n\\nA pre-procedure time-out was performed discussing the planned procedure,\\nconfirming the patient's identity with 3 identifiers, and reviewing a\\nchecklist per ___ protocol.\\n\\nUnder ultrasound guidance, an entrance site was selected and the skin was\\nprepped and draped in the usual sterile fashion. 1% lidocaine was instilled\\nfor local anesthesia.\\n\\nA 5 ___ ___ catheter was advanced into the largest fluid pocket in the\\nright lower quadrant and 1.9 L of cloudy, pink-tinged ascites fluid was\\nremoved.\\n\\nThe patient tolerated the procedure well without immediate complication.\\nEstimated blood loss was minimal.\\n\\nDr. ___ was present for and personally supervised the entirety of the\\nprocedure and reviewed and agrees with the trainee's findings.\\n\\nIMPRESSION: \\n\\nUncomplicated ultrasound-guided therapeutic paracentesis via a right lower\\nquadrant approach, yielding 1.9 L of cloudy, pink-tinged ascites fluid.\\n  \n",
      "9  EXAMINATION:  Ultrasound-guided paracentesis.\\n\\nINDICATION:  A ___ woman with cirrhosis recurrent ascites, presenting\\nfor large-volume paracentesis.\\n\\nTECHNIQUE:  Ultrasound-guided therapeutic paracentesis.\\n\\nCOMPARISON:  Ultrasound-guided paracentesis performed ___.\\n\\nFINDINGS: \\n\\nLimited grayscale ultrasound imaging of the abdomen demonstrated a large\\namount of ascites. A suitable target in the deepest pocket in the right flank\\nwas selected for paracentesis.\\n\\nOf note, several transverse images of the right lower quadrant revealed a 1.1\\nx 0.9 cm nodular isoechoic structure along the peritoneal surface\\nanterolaterally without any apparent vascularity, of indeterminate\\nsignificance.  This is probably unchanged since ultrasound obtained ___. This may represents sequela from prior paracenteses. If clinical concern\\nfor malignancy exists, CT could be obtained for further evaluation (or\\nreference can be made to outside hospital CT or MR examinations if such\\nexist).\\n\\nPROCEDURE:  The procedure, risks, benefits and alternatives were discussed\\nwith the patient and existing annual signed consent was reviewed.\\n\\nA pre-procedure time-out was performed discussing the planned procedure,\\nconfirming the patient's identity with 3 identifiers, and reviewing a\\nchecklist per ___ protocol.\\n\\nUnder ultrasound guidance, an entrance site was selected and the skin was\\nprepped and draped in the usual sterile fashion. 1% lidocaine was instilled\\nfor local anesthesia.\\n\\nA 5 ___ catheter was advanced into the largest fluid pocket in the right\\nflank and 4.3 L of cloudy, pink-tinged straw-colored fluid was removed. Two\\nrunning 0 silk stitches were used to help secure entry site and prevent\\nagainst ascites leak.\\n\\nThe patient tolerated the procedure well without immediate complication.\\nEstimated blood loss was minimal.\\n\\nDr. ___ supervised the trainee during the key components of the\\nprocedure and reviewed and agrees with the trainee's findings.\\n\\nIMPRESSION:\\n\\n\\n1. Uncomplicated ultrasound-guided therapeutic paracentesis via a right flank\\napproach.\\n2. 1.1 x 0.9 cm isoechoic nodule seen on the parietal peritoneal surface\\nunderlying the right abdominal wall, may relate to sequela from prior\\nparacenteses, probably unchanged since ___. If concern for malignancy\\nexists, CT may be obtained (or reference can be made to outside hospital CT or\\nMR examinations if such exist).\\n  \n"
     ]
    }
   ],
   "source": [
    "top_10_rows_radiology = df_radiology.head(10)\n",
    "print(top_10_rows_radiology)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8a03226f",
   "metadata": {},
   "outputs": [],
   "source": [
    "last_column_name = df_radiology.columns[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3085f97b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text\n"
     ]
    }
   ],
   "source": [
    "print(last_column_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b14ca568",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/alex/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /Users/alex/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /Users/alex/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "# Download required NLTK data\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6afbd021",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_discharge = pd.read_csv(\"C:/Users/zhi-n/Documents/FIT3163/mimic-iv-note/discharge.csv\")\n",
    "df_discharge = pd.read_csv(\"discharge.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d3c40ab8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    # Remove '\\n' characters and extra spaces\n",
    "    cleaned_text = re.sub(r'\\n+', ' ', text) # replacing '\\n' with a space\n",
    "    cleaned_text = re.sub(r' +', ' ', cleaned_text)\n",
    "    # Optionally, remove any other unwanted characters/words/numbers here\n",
    "    cleaned_text = re.sub(r'[0-9]+', '', cleaned_text) # removing numbers\n",
    "    return cleaned_text.strip()\n",
    "\n",
    "df_discharge['text'] = df_discharge['text'].apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b785d04a",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/Users/alex/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb Cell 16\u001b[0m line \u001b[0;36m8\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/alex/Kaizo%27s%20Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb#X21sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m     filtered_text \u001b[39m=\u001b[39m [word \u001b[39mfor\u001b[39;00m word \u001b[39min\u001b[39;00m word_tokens \u001b[39mif\u001b[39;00m word\u001b[39m.\u001b[39mlower() \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m stop_words]\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/alex/Kaizo%27s%20Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb#X21sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39m'\u001b[39m\u001b[39m \u001b[39m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mjoin(filtered_text)\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/alex/Kaizo%27s%20Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb#X21sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m df_discharge[\u001b[39m'\u001b[39m\u001b[39mtext\u001b[39m\u001b[39m'\u001b[39m] \u001b[39m=\u001b[39m df_discharge[\u001b[39m'\u001b[39;49m\u001b[39mtext\u001b[39;49m\u001b[39m'\u001b[39;49m]\u001b[39m.\u001b[39;49mapply(remove_stopwords)\n",
      "File \u001b[0;32m~/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/my_venv/lib/python3.8/site-packages/pandas/core/series.py:4630\u001b[0m, in \u001b[0;36mSeries.apply\u001b[0;34m(self, func, convert_dtype, args, **kwargs)\u001b[0m\n\u001b[1;32m   4520\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mapply\u001b[39m(\n\u001b[1;32m   4521\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[1;32m   4522\u001b[0m     func: AggFuncType,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   4525\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs,\n\u001b[1;32m   4526\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m DataFrame \u001b[39m|\u001b[39m Series:\n\u001b[1;32m   4527\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   4528\u001b[0m \u001b[39m    Invoke function on values of Series.\u001b[39;00m\n\u001b[1;32m   4529\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   4628\u001b[0m \u001b[39m    dtype: float64\u001b[39;00m\n\u001b[1;32m   4629\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 4630\u001b[0m     \u001b[39mreturn\u001b[39;00m SeriesApply(\u001b[39mself\u001b[39;49m, func, convert_dtype, args, kwargs)\u001b[39m.\u001b[39;49mapply()\n",
      "File \u001b[0;32m~/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/my_venv/lib/python3.8/site-packages/pandas/core/apply.py:1025\u001b[0m, in \u001b[0;36mSeriesApply.apply\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1022\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mapply_str()\n\u001b[1;32m   1024\u001b[0m \u001b[39m# self.f is Callable\u001b[39;00m\n\u001b[0;32m-> 1025\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mapply_standard()\n",
      "File \u001b[0;32m~/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/my_venv/lib/python3.8/site-packages/pandas/core/apply.py:1076\u001b[0m, in \u001b[0;36mSeriesApply.apply_standard\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1074\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m   1075\u001b[0m         values \u001b[39m=\u001b[39m obj\u001b[39m.\u001b[39mastype(\u001b[39mobject\u001b[39m)\u001b[39m.\u001b[39m_values\n\u001b[0;32m-> 1076\u001b[0m         mapped \u001b[39m=\u001b[39m lib\u001b[39m.\u001b[39;49mmap_infer(\n\u001b[1;32m   1077\u001b[0m             values,\n\u001b[1;32m   1078\u001b[0m             f,\n\u001b[1;32m   1079\u001b[0m             convert\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconvert_dtype,\n\u001b[1;32m   1080\u001b[0m         )\n\u001b[1;32m   1082\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(mapped) \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(mapped[\u001b[39m0\u001b[39m], ABCSeries):\n\u001b[1;32m   1083\u001b[0m     \u001b[39m# GH#43986 Need to do list(mapped) in order to get treated as nested\u001b[39;00m\n\u001b[1;32m   1084\u001b[0m     \u001b[39m#  See also GH#25959 regarding EA support\u001b[39;00m\n\u001b[1;32m   1085\u001b[0m     \u001b[39mreturn\u001b[39;00m obj\u001b[39m.\u001b[39m_constructor_expanddim(\u001b[39mlist\u001b[39m(mapped), index\u001b[39m=\u001b[39mobj\u001b[39m.\u001b[39mindex)\n",
      "File \u001b[0;32m~/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/my_venv/lib/python3.8/site-packages/pandas/_libs/lib.pyx:2834\u001b[0m, in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "\u001b[1;32m/Users/alex/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb Cell 16\u001b[0m line \u001b[0;36m4\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/alex/Kaizo%27s%20Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb#X21sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mremove_stopwords\u001b[39m(text):\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/alex/Kaizo%27s%20Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb#X21sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m     word_tokens \u001b[39m=\u001b[39m word_tokenize(text)\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/alex/Kaizo%27s%20Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb#X21sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m     filtered_text \u001b[39m=\u001b[39m [word \u001b[39mfor\u001b[39;00m word \u001b[39min\u001b[39;00m word_tokens \u001b[39mif\u001b[39;00m word\u001b[39m.\u001b[39mlower() \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m stop_words]\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/alex/Kaizo%27s%20Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/test2.ipynb#X21sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39m'\u001b[39m\u001b[39m \u001b[39m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mjoin(filtered_text)\n",
      "File \u001b[0;32m~/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/my_venv/lib/python3.8/site-packages/nltk/tokenize/__init__.py:130\u001b[0m, in \u001b[0;36mword_tokenize\u001b[0;34m(text, language, preserve_line)\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    116\u001b[0m \u001b[39mReturn a tokenized copy of *text*,\u001b[39;00m\n\u001b[1;32m    117\u001b[0m \u001b[39musing NLTK's recommended word tokenizer\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[39m:type preserve_line: bool\u001b[39;00m\n\u001b[1;32m    128\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    129\u001b[0m sentences \u001b[39m=\u001b[39m [text] \u001b[39mif\u001b[39;00m preserve_line \u001b[39melse\u001b[39;00m sent_tokenize(text, language)\n\u001b[0;32m--> 130\u001b[0m \u001b[39mreturn\u001b[39;00m [\n\u001b[1;32m    131\u001b[0m     token \u001b[39mfor\u001b[39;00m sent \u001b[39min\u001b[39;00m sentences \u001b[39mfor\u001b[39;00m token \u001b[39min\u001b[39;00m _treebank_word_tokenizer\u001b[39m.\u001b[39mtokenize(sent)\n\u001b[1;32m    132\u001b[0m ]\n",
      "File \u001b[0;32m~/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/my_venv/lib/python3.8/site-packages/nltk/tokenize/__init__.py:131\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    116\u001b[0m \u001b[39mReturn a tokenized copy of *text*,\u001b[39;00m\n\u001b[1;32m    117\u001b[0m \u001b[39musing NLTK's recommended word tokenizer\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[39m:type preserve_line: bool\u001b[39;00m\n\u001b[1;32m    128\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    129\u001b[0m sentences \u001b[39m=\u001b[39m [text] \u001b[39mif\u001b[39;00m preserve_line \u001b[39melse\u001b[39;00m sent_tokenize(text, language)\n\u001b[1;32m    130\u001b[0m \u001b[39mreturn\u001b[39;00m [\n\u001b[0;32m--> 131\u001b[0m     token \u001b[39mfor\u001b[39;00m sent \u001b[39min\u001b[39;00m sentences \u001b[39mfor\u001b[39;00m token \u001b[39min\u001b[39;00m _treebank_word_tokenizer\u001b[39m.\u001b[39;49mtokenize(sent)\n\u001b[1;32m    132\u001b[0m ]\n",
      "File \u001b[0;32m~/Kaizo's Documents/CompSci/Y3S2/FIT3163/GIT_DSP/Git_DSP/my_venv/lib/python3.8/site-packages/nltk/tokenize/destructive.py:160\u001b[0m, in \u001b[0;36mNLTKWordTokenizer.tokenize\u001b[0;34m(self, text, convert_parentheses, return_str)\u001b[0m\n\u001b[1;32m    157\u001b[0m     text \u001b[39m=\u001b[39m regexp\u001b[39m.\u001b[39msub(substitution, text)\n\u001b[1;32m    159\u001b[0m \u001b[39mfor\u001b[39;00m regexp, substitution \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mPUNCTUATION:\n\u001b[0;32m--> 160\u001b[0m     text \u001b[39m=\u001b[39m regexp\u001b[39m.\u001b[39;49msub(substitution, text)\n\u001b[1;32m    162\u001b[0m \u001b[39m# Handles parentheses.\u001b[39;00m\n\u001b[1;32m    163\u001b[0m regexp, substitution \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mPARENS_BRACKETS\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.8/re.py:325\u001b[0m, in \u001b[0;36m_subx\u001b[0;34m(pattern, template)\u001b[0m\n\u001b[1;32m    322\u001b[0m     template \u001b[39m=\u001b[39m sre_parse\u001b[39m.\u001b[39mparse_template(template, pattern)\n\u001b[1;32m    323\u001b[0m     \u001b[39mreturn\u001b[39;00m sre_parse\u001b[39m.\u001b[39mexpand_template(template, match)\n\u001b[0;32m--> 325\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_subx\u001b[39m(pattern, template):\n\u001b[1;32m    326\u001b[0m     \u001b[39m# internal: Pattern.sub/subn implementation helper\u001b[39;00m\n\u001b[1;32m    327\u001b[0m     template \u001b[39m=\u001b[39m _compile_repl(template, pattern)\n\u001b[1;32m    328\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m template[\u001b[39m0\u001b[39m] \u001b[39mand\u001b[39;00m \u001b[39mlen\u001b[39m(template[\u001b[39m1\u001b[39m]) \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m    329\u001b[0m         \u001b[39m# literal replacement\u001b[39;00m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "def remove_stopwords(text):\n",
    "    word_tokens = word_tokenize(text)\n",
    "    filtered_text = [word for word in word_tokens if word.lower() not in stop_words]\n",
    "    return ' '.join(filtered_text)\n",
    "\n",
    "df_discharge['text'] = df_discharge['text'].apply(remove_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ef51dd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "def lemmatize_text(text):\n",
    "    word_tokens = word_tokenize(text)\n",
    "    lemmatized_text = [lemmatizer.lemmatize(word) for word in word_tokens]\n",
    "    return ' '.join(lemmatized_text)\n",
    "\n",
    "df_discharge['text'] = df_discharge['text'].apply(lemmatize_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ff82461",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)  # To display the full content of the 'text' column\n",
    "print(df_discharge.head(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e5b9cbb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99789b88",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}