testLibrary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                  Job Description  Company Name\n",
      "Industry                                                       \n",
      "-1                                            353           352\n",
      "IT Services                                   325           325\n",
      "Staffing & Outsourcing                        323           323\n",
      "Health Care Services & Hospitals              151           151\n",
      "Consulting                                    111           111\n",
      "...                                           ...           ...\n",
      "Chemical Manufacturing                          1             1\n",
      "Pet & Pet Supplies Stores                       1             1\n",
      "Consumer Product Rental                         1             1\n",
      "Metals Brokers                                  1             1\n",
      "News Outlet                                     1             1\n",
      "\n",
      "[89 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "from textclustering import utilities as ut\n",
    "from textclustering import tfidfModule as tfm\n",
    "\n",
    "#change operating folder      \n",
    "os.chdir(\"/Users/arnabborah/Documents/repositories/textclusteringDBSCAN/scripts/\")\n",
    "#read the .csv data file using the dataProcessor class\n",
    "rp = tfm.dataProcessor(\"../datasets/DataAnalyst.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ ================================================== ] 100.00%\n",
      "Adding term frequency column based on stopsRemoved\n",
      "[ ================================================== ] 100.00%\n",
      "Computing list of words for IDF...\n",
      "\n",
      "Created list of terms for IDF matrix with 27075  terms.\n",
      "\n",
      "Computing global IDF matrix...\n",
      "\n",
      "[ ================================================== ] 100.00%\n",
      "Computing and adding TF-IDF column based on stopsRemoved\n",
      "[ ================================================== ] 100.00%"
     ]
    }
   ],
   "source": [
    "#create a flingTFIDF object around the pre-processed daa\n",
    "ftf = tfm.flingTFIDF(rp.dataInitialSmall,'Job Description')\n",
    "\n",
    "# tokenization, customizable\n",
    "ftf.smartTokenizeColumn()\n",
    "\n",
    "# get Term Frequency of each document, and store add it as an object, in a new column\n",
    "ftf.getTF()\n",
    "\n",
    "# compute Inverse Document Frequencies across the entire vocabulary\n",
    "ftf.computeIDFmatrix()\n",
    "\n",
    "# get TFIDF, and store it as a new column in data, tf-idf\n",
    "ftf.getTFIDF()\n",
    "\n",
    "# compute sum of all tf-idf values and add it as a new column\n",
    "ftf.createDistanceMetadata()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(\"/Users/arnabborah/Documents/repositories/textclusteringDBSCAN/scripts/\")\n",
    "ftf.data.to_pickle('../processFiles/data_tfidf_processed.pkl')\n",
    "os.chdir(\"/Users/arnabborah/Documents/repositories/textclusteringDBSCAN/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "DBSCAN initialized!\n",
      "\n",
      "Loading Glove Model\n",
      "\n",
      "400000  words loaded!\n",
      "\n",
      "GloVe Vectors Loaded!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# load dataset with tf-idf vectors and load pretrained GloVe word vectors\n",
    "from textclustering import flingPretrained as pre\n",
    "import pandas as pd\n",
    "\n",
    "dataProcessed = pd.read_pickle('processFiles/data_tfidf_processed.pkl')\n",
    "fdb = pre.flingPretrained(dataProcessed)\n",
    "fdb.loadPretrainedWordVectors('glove')\n",
    "\n",
    "# adding glove vectors for every document\n",
    "fdb.addDocumentGloveVector()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "flingDBSCAN initialized!\n",
      "\n",
      "computing best distance\n",
      "[ ================================================== ] 100.00%"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOoElEQVR4nO3df4xlZ13H8feHLgUNPwrdsTa7W6eGJVpRoFmxgJEfFdPuGrbGQkpACtm4CVaCQpRVE/HXH9sYKZIgZKUNC1FoBWM3FEJIf6RRbGFrodA2yFCL3bWwC7RV0oAufP3jPoXpMtt7ZubO3Jl93q9kMuc857n3fmZy5zNnzrn3TKoKSdLJ73HTDiBJWh0WviR1wsKXpE5Y+JLUCQtfkjqxYdoBADZu3Fizs7PTjiFJ68ptt9329aqaGTp/TRT+7OwsBw8enHYMSVpXknxlMfM9pCNJnbDwJakTFr4kdcLCl6ROWPiS1AkLX5I6YeFLUicsfEnqhIUvSZ1YE++0lcaZ3XPdkm97794dE0wirV/u4UtSJyx8SeqEhS9JnbDwJakTFr4kdcLCl6ROWPiS1AkLX5I6YeFLUicsfEnqhIUvSZ2w8CWpExa+JHXCwpekTlj4ktSJwYWf5JQktyf5aFs/O8mtSeaSXJ3k1Db+hLY+17bPrlB2SdIiLGYP/03A3fPWLweuqKpnAA8Au9r4LuCBNn5FmydJmrJBhZ9kM7ADeG9bD/BS4MNtyn7gora8s63Ttp/f5kuSpmjoHv47gN8HvtfWTwcerKpjbf0QsKktbwLuA2jbH2rzHyXJ7iQHkxw8evTo0tJLkgYbW/hJfhU4UlW3TfKBq2pfVW2rqm0zMzOTvGtJ0gKG/BPzFwIvT7IdeCLwFOCvgdOSbGh78ZuBw23+YWALcCjJBuCpwDcmnlyStChj9/Cr6g+qanNVzQKXADdU1auBG4GL27RLgWvb8oG2Ttt+Q1XVRFNLkhZtOa/Dfyvw5iRzjI7RX9nGrwROb+NvBvYsL6IkaRKGHNL5vqq6CbipLd8DPG+BOd8GXjGBbJKkCfKdtpLUCQtfkjph4UtSJyx8SeqEhS9JnbDwJakTFr4kdcLCl6ROWPiS1AkLX5I6YeFLUicWdS0daT2a3XPdkm97794dE0wiTZd7+JLUCQtfkjph4UtSJzyGLz2G5Rz/B88BaG1xD1+SOmHhS1InLHxJ6oSFL0mdsPAlqRMWviR1wsKXpE5Y+JLUCQtfkjph4UtSJyx8SeqEhS9JnbDwJakTFr4kdcLCl6ROWPiS1AkLX5I6YeFLUicsfEnqhIUvSZ2w8CWpExa+JHVibOEneWKSTyf5XJI7k/xpGz87ya1J5pJcneTUNv6Etj7Xts+u8NcgSRpgyB7+d4CXVtWzgecAFyQ5D7gcuKKqngE8AOxq83cBD7TxK9o8SdKUjS38GvlWW318+yjgpcCH2/h+4KK2vLOt07afnySTCixJWppBx/CTnJLks8AR4JPAl4EHq+pYm3II2NSWNwH3AbTtDwGnL3Cfu5McTHLw6NGjy/oiJEnjDSr8qvpuVT0H2Aw8D/ip5T5wVe2rqm1VtW1mZma5dydJGmNRr9KpqgeBG4HnA6cl2dA2bQYOt+XDwBaAtv2pwDcmEVaStHRDXqUzk+S0tvwjwMuAuxkV/8Vt2qXAtW35QFunbb+hqmqCmSVJS7Bh/BTOBPYnOYXRL4hrquqjSe4CPpTkL4DbgSvb/CuBDySZA74JXLICuSVJizS28KvqDuC5C4zfw+h4/vHj3wZeMZF0OqnM7rlu2hGkrvlOW0nqhIUvSZ2w8CWpExa+JHXCwpekTlj4ktQJC1+SOmHhS1InLHxJ6oSFL0mdsPAlqRMWviR1YsjVMiUt0XIuGHfv3h0TTCK5hy9J3bDwJakTFr4kdcLCl6ROWPiS1AkLX5I6YeFLUicsfEnqhIUvSZ2w8CWpExa+JHXCwpekTlj4ktQJC1+SOmHhS1InLHxJ6oSFL0mdsPAlqRMWviR1wsKXpE5Y+JLUCQtfkjph4UtSJyx8SeqEhS9JnRhb+Em2JLkxyV1J7kzypjb+9CSfTPKl9vlpbTxJ3plkLskdSc5d6S9CkjTekD38Y8Bbquoc4DzgsiTnAHuA66tqK3B9Wwe4ENjaPnYD7554aknSoo0t/Kq6v6r+rS3/D3A3sAnYCexv0/YDF7XlncD7a+QW4LQkZ046uCRpcTYsZnKSWeC5wK3AGVV1f9v0VeCMtrwJuG/ezQ61sfvnjZFkN6O/ADjrrLMWm1s66c3uuW7Jt713744JJtHJYvBJ2yRPAj4C/E5V/ff8bVVVQC3mgatqX1Vtq6ptMzMzi7mpJGkJBhV+ksczKvu/q6p/bMNfe+RQTft8pI0fBrbMu/nmNiZJmqIhr9IJcCVwd1W9fd6mA8ClbflS4Np5469tr9Y5D3ho3qEfSdKUDDmG/0LgN4DPJ/lsG/tDYC9wTZJdwFeAV7ZtHwO2A3PAw8DrJxlY07Wc48qSpmts4VfVPwM5webzF5hfwGXLzCVJmjDfaStJnbDwJakTFr4kdcLCl6ROWPiS1AkLX5I6YeFLUicsfEnqhIUvSZ2w8CWpExa+JHXCwpekTlj4ktQJC1+SOmHhS1InLHxJ6oSFL0mdsPAlqRMWviR1wsKXpE5Y+JLUCQtfkjph4UtSJyx8SeqEhS9JnbDwJakTFr4kdcLCl6ROWPiS1AkLX5I6YeFLUicsfEnqhIUvSZ3YMO0AkiZvds91S77tvXt3TDCJ1hL38CWpExa+JHXCwpekTlj4ktSJsYWf5KokR5J8Yd7Y05N8MsmX2uentfEkeWeSuSR3JDl3JcNLkoYbsof/PuCC48b2ANdX1Vbg+rYOcCGwtX3sBt49mZiSpOUaW/hVdTPwzeOGdwL72/J+4KJ54++vkVuA05KcOaGskqRlWOox/DOq6v62/FXgjLa8Cbhv3rxDbeyHJNmd5GCSg0ePHl1iDEnSUMs+aVtVBdQSbrevqrZV1baZmZnlxpAkjbHUwv/aI4dq2ucjbfwwsGXevM1tTJI0ZUst/APApW35UuDaeeOvba/WOQ94aN6hH0nSFI29lk6SDwIvBjYmOQS8DdgLXJNkF/AV4JVt+seA7cAc8DDw+hXILElagrGFX1WvOsGm8xeYW8Blyw0lSZo832krSZ2w8CWpExa+JHXCwpekTlj4ktQJ/8WhpEdZzr9HBP9F4lrmHr4kdcLCl6ROWPiS1AkLX5I6YeFLUicsfEnqhIUvSZ2w8CWpExa+JHXCd9pKmqjlvFPXd+muLPfwJakT634P3+t+SNIw677wtTjL/QUpaf3ykI4kdcLCl6ROWPiS1AkLX5I6YeFLUicsfEnqhIUvSZ2w8CWpExa+JHXCwpekTlj4ktQJr6Ujac3w0soryz18SeqEhS9JnbDwJakTFr4kdcKTtuuQ/8RE0lJY+JJOCr7CZzwP6UhSJ1ak8JNckOSLSeaS7FmJx5AkLc7ED+kkOQV4F/Ay4BDwmSQHququST+WJE1CL4eDVuIY/vOAuaq6ByDJh4CdgIUv6aSz3BdRrOYvjJUo/E3AffPWDwG/cPykJLuB3W31W0m+uMTH2wh8fYm3JZcv9ZbLtqzcU2Lm1bMec5t5CZbYQY/k/onF3Ghqr9Kpqn3AvuXeT5KDVbVtApFW1XrMbebVsx5zm3n1LDX3Spy0PQxsmbe+uY1JkqZoJQr/M8DWJGcnORW4BDiwAo8jSVqEiR/SqapjSX4b+ARwCnBVVd056ceZZ9mHhaZkPeY28+pZj7nNvHqWlDtVNekgkqQ1yHfaSlInLHxJ6sS6Kfxxl2tI8oQkV7fttyaZnULM4zONy/zmJHcluSPJ9UkW9ZralTL00hhJfj1JJZn6y9qGZE7yyvb9vjPJ3692xgXyjHt+nJXkxiS3t+fI9mnkPC7TVUmOJPnCCbYnyTvb13RHknNXO+MCmcZlfnXL+vkkn0ry7NXOuJBxuefN+/kkx5JcPPZOq2rNfzA6+ftl4CeBU4HPAeccN+e3gPe05UuAq9dB5pcAP9qW3zDtzENzt3lPBm4GbgG2rfXMwFbgduBpbf3H1kHmfcAb2vI5wL1r4PnxS8C5wBdOsH078HEgwHnAresg8wvmPS8uXAuZh+Se9zy6AfgYcPG4+1wve/jfv1xDVf0v8MjlGubbCexvyx8Gzk+SVcx4vLGZq+rGqnq4rd7C6D0L0zbkew3w58DlwLdXM9wJDMn8m8C7quoBgKo6ssoZjzckcwFPactPBf5rFfMtqKpuBr75GFN2Au+vkVuA05KcuTrpFjYuc1V96pHnBWvn53DI9xrgjcBHgEHP5/VS+AtdrmHTieZU1THgIeD0VUm3sCGZ59vFaM9o2sbmbn+mb6mqtfKfWIZ8r58JPDPJvyS5JckFq5ZuYUMy/wnwmiSHGO3BvXF1oi3LYp/3a81a+TkcK8km4NeAdw+9jf8AZQ1I8hpgG/CiaWcZJ8njgLcDr5tylMXawOiwzosZ7cHdnORnq+rBaYYa41XA+6rqr5I8H/hAkmdV1femHexklOQljAr/F6edZaB3AG+tqu8NPZixXgp/yOUaHplzKMkGRn8Cf2N14i1o0CUmkvwy8EfAi6rqO6uU7bGMy/1k4FnATe1J9uPAgSQvr6qDq5by0YZ8rw8xOjb7f8B/JPl3Rr8APrM6EX/IkMy7gAsAqupfkzyR0UWzpn046rGsy0urJPk54L3AhVU1zd5YjG3Ah9rP4UZge5JjVfVPJ7zFtE9MDDx5sQG4BzibH5zg+pnj5lzGo0/aXrMOMj+X0Ym7rdP+Hi8m93Hzb2L6J22HfK8vAPa35Y2MDjucvsYzfxx4XVv+aUbH8LMGniOznPgE6A4efdL209POOyDzWcAc8IJp51xM7uPmvY8BJ23XxR5+neByDUn+DDhYVQeAKxn9yTvH6ETHJdNLPDjzXwJPAv6h/Zb+z6p6+dRCMzj3mjIw8yeAX0lyF/Bd4PdqintyAzO/BfjbJL/L6ATu66r9dE9Lkg8yOiy2sZ1beBvweICqeg+jcw3bGRXow8Drp5P0BwZk/mNG5/v+pv0cHqs1cAXNAbkXf59Tfv5IklbJenmVjiRpmSx8SeqEhS9JnbDwJakTFr4kdcLCl6ROWPiS1In/B8M4LpzdTiDZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best epsilon computed on GLOVE = 0.6544420699360174 \n",
      "\n",
      "\n",
      "initiating DBSCAN Clustering with glove vectors\n",
      "\n",
      "[                                                    ] 0.04%\n",
      " ----  cluster_1_ assigned to 565 points! ----\n",
      "[                                                    ] 0.09%\n",
      " ----  cluster_2_ assigned to 855 points! ----\n",
      "[                                                    ] 0.18%\n",
      " ----  cluster_3_ assigned to 58 points! ----\n",
      "[                                                    ] 0.31%\n",
      " ----  cluster_4_ assigned to 119 points! ----\n",
      "[                                                    ] 0.53%\n",
      " ----  cluster_5_ assigned to 109 points! ----\n",
      "[                                                    ] 1.07%\n",
      " ----  cluster_6_ assigned to 53 points! ----\n",
      "[                                                    ] 1.91%\n",
      " ----  cluster_7_ assigned to 37 points! ----\n",
      "[ =                                                  ] 2.26%\n",
      " ----  cluster_8_ assigned to 55 points! ----\n",
      "[ ===                                                ] 6.79%\n",
      " ----  cluster_9_ assigned to 35 points! ----\n",
      "[ =======                                            ] 15.85%\n",
      " ----  cluster_10_ assigned to 32 points! ----\n",
      "[ ====================                               ] 41.59%\n",
      " ----  cluster_11_ assigned to 27 points! ----\n",
      "[ ================================================== ] 100.00%\n",
      " 11 clusters formed!\n",
      "Cluster characteristics:\n",
      " -- vectors: glove\n",
      " -- minPts: 25\n",
      " -- EstimatedBestDistance 0.6544420699360174\n",
      " -- 11 clusters formed!\n",
      " -- 1945 points assigned to clusters!\n",
      " -- 308 noise points!\n",
      "\n",
      " -- 13.670661340434975 % noise!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# use DBSCAN clustering on the glove vectors loaded in the previos\n",
    "from textclustering import flingDBSCAN as fdbscan\n",
    "\n",
    "fdbscan1 = fdbscan.flingDBSCAN(fdb.data,None,25,'glove')\n",
    "fdbscan1.dbscanCompute()\n",
    "fdbscan1.addClusterLabel('glove-clusterID')\n",
    "fdbscan1.printClusterInfo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "flingDBSCAN initialized!\n",
      "\n",
      "computing best distance\n",
      "[ ================================================== ] 100.00%"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD7CAYAAACRxdTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAASjklEQVR4nO3df4xdZ53f8fdnjQmoIAxkmnVtZyftuloFVJx06g2iqtJElJBscVZlI6MKDE3lbRtUEKt2DX8UaBspSF3SpT+y8jYpZkVJImAbN4Ru0yQrhFQSJlkT8gPaWTCKLRMPIQlEdFM5++0f8xhuhhnPnblzZ8ZP3i/pap7znOfc871H9mfOPPfcc1NVSJL68gvrXYAkafUZ7pLUIcNdkjpkuEtShwx3SeqQ4S5JHRo63JNsSvInSe5syxckuT/JTJLbkry89Z/Tlmfa+skx1S5JWsRyztw/ADw+sPwJ4Maq+mXgaeDa1n8t8HTrv7GNkyStoQzzIaYk24FDwPXAh4C/C8wCv1hVp5K8GfhYVb0tyR+19v9K8jLg+8BEnWFH5557bk1OTo7+aiTpJeTBBx/8QVVNLLTuZUM+x78F/jnw6rb8euCZqjrVlo8B21p7G/AEQAv+Z9v4Hyz25JOTk0xPTw9ZiiQJIMn3Flu35LRMkl8DTlbVg6tc1P4k00mmZ2dnV/OpJeklb5g597cA70hyFLgVuAz4XWBLm3YB2A4cb+3jwA6Atv41wFPzn7SqDlbVVFVNTUws+FeFJGmFlgz3qvpwVW2vqklgL3BvVf194D7gnW3YPuCO1j7clmnr7z3TfLskafWNcp37bwMfSjLD3Jz6za3/ZuD1rf9DwIHRSpQkLdewb6gCUFV/DPxxa38H2L3AmD8DfmMVapMkrZCfUJWkDhnuktQhw12SOmS4S1KHlvWGqjSKyQNfWvG2R2+4ahUrkfrnmbskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDnn7AS3LKLcQkLR2PHOXpA4Z7pLUoSXDPckrkjyQ5BtJHk3y8db/6STfTXKkPXa1/iT5VJKZJA8nuXjMr0GSNM8wc+7PA5dV1XNJNgNfTfLltu6fVdXn541/O7CzPX4VuKn9lCStkSXP3GvOc21xc3vUGTbZA3ymbfc1YEuSraOXKkka1lBz7kk2JTkCnATurqr726rr29TLjUnOaX3bgCcGNj/W+iRJa2SocK+qF6pqF7Ad2J3kjcCHgV8B/gbwOuC3l7PjJPuTTCeZnp2dXV7VkqQzWtbVMlX1DHAfcEVVnWhTL88D/xnY3YYdB3YMbLa99c1/roNVNVVVUxMTEysqXpK0sGGulplIsqW1Xwm8FfjW6Xn0JAGuBh5pmxwG3tOumrkEeLaqToyhdknSIoa5WmYrcCjJJuZ+GdxeVXcmuTfJBBDgCPCP2vi7gCuBGeAnwPtWvWpJ0hktGe5V9TBw0QL9ly0yvoDrRi9NkrRS3ltGZ4VR7mlz9IarVrES6ezg7QckqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdcjbD6h7o9y6ALx9gc5OnrlLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDg3zBdmvSPJAkm8keTTJx1v/BUnuTzKT5LYkL2/957TlmbZ+csyvQZI0zzBn7s8Dl1XVm4BdwBVJLgE+AdxYVb8MPA1c28ZfCzzd+m9s4yRJa2jJcK85z7XFze1RwGXA51v/IeDq1t7TlmnrL0+S1SpYkrS0oebck2xKcgQ4CdwN/CnwTFWdakOOAdtaexvwBEBb/yzw+lWsWZK0hKHCvapeqKpdwHZgN/Aro+44yf4k00mmZ2dnR306SdKAZV0tU1XPAPcBbwa2JDl9b5rtwPHWPg7sAGjrXwM8tcBzHayqqaqampiYWFn1kqQFDXO1zESSLa39SuCtwOPMhfw727B9wB2tfbgt09bfW1W1ijVLkpYwzF0htwKHkmxi7pfB7VV1Z5LHgFuT/GvgT4Cb2/ibgT9IMgP8ENg7hrolSWewZLhX1cPARQv0f4e5+ff5/X8G/MaqVKexGPUWuJI2Pj+hKkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ8N8QfaOJPcleSzJo0k+0Po/luR4kiPtceXANh9OMpPk20neNs4XIEn6ecN8QfYp4Leq6qEkrwYeTHJ3W3djVf2bwcFJLmTuS7HfAPwl4H8m+atV9cJqFi5JWtySZ+5VdaKqHmrtHwOPA9vOsMke4Naqer6qvgvMsMAXaUuSxmdZc+5JJoGLgPtb1/uTPJzkliSvbX3bgCcGNjvGmX8ZSJJW2dDhnuRVwBeAD1bVj4CbgL8C7AJOAL+znB0n2Z9kOsn07OzscjaVJC1hqHBPspm5YP9sVX0RoKqerKoXqurPgd/nZ1Mvx4EdA5tvb30vUlUHq2qqqqYmJiZGeQ2SpHmGuVomwM3A41X1yYH+rQPDfh14pLUPA3uTnJPkAmAn8MDqlSxJWsowV8u8BXg38M0kR1rfR4B3JdkFFHAU+E2Aqno0ye3AY8xdaXOdV8pI0tpaMtyr6qtAFlh11xm2uR64foS6JEkj8BOqktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoWFuPyC9pE0e+NKKtz16w1WrWIk0PM/cJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0a5guydyS5L8ljSR5N8oHW/7okdyf5P+3na1t/knwqyUySh5NcPO4XIUl6sWHO3E8Bv1VVFwKXANcluRA4ANxTVTuBe9oywNuBne2xH7hp1auWJJ3RkuFeVSeq6qHW/jHwOLAN2AMcasMOAVe39h7gMzXna8CWJFtXu3BJ0uKWNeeeZBK4CLgfOK+qTrRV3wfOa+1twBMDmx1rfZKkNTJ0uCd5FfAF4INV9aPBdVVVQC1nx0n2J5lOMj07O7ucTSVJSxgq3JNsZi7YP1tVX2zdT56ebmk/T7b+48COgc23t74XqaqDVTVVVVMTExMrrV+StIBhrpYJcDPweFV9cmDVYWBfa+8D7hjof0+7auYS4NmB6RtJ0hoY5n7ubwHeDXwzyZHW9xHgBuD2JNcC3wOuaevuAq4EZoCfAO9bzYIlSUtbMtyr6qtAFll9+QLjC7huxLokSSPwE6qS1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjo0zBdk35LkZJJHBvo+luR4kiPtceXAug8nmUny7SRvG1fhkqTFDXPm/mngigX6b6yqXe1xF0CSC4G9wBvaNv8xyabVKlaSNJwlw72qvgL8cMjn2wPcWlXPV9V3gRlg9wj1SZJWYJQ59/cnebhN27y29W0DnhgYc6z1SZLW0MtWuN1NwL8Cqv38HeAfLOcJkuwH9gOcf/75KyxD2tgmD3xpxdseveGqVaxELzUrOnOvqier6oWq+nPg9/nZ1MtxYMfA0O2tb6HnOFhVU1U1NTExsZIyJEmLWFG4J9k6sPjrwOkraQ4De5Ock+QCYCfwwGglSpKWa8lpmSSfAy4Fzk1yDPgocGmSXcxNyxwFfhOgqh5NcjvwGHAKuK6qXhhL5ZKkRS0Z7lX1rgW6bz7D+OuB60cpSpI0Gj+hKkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0Eq/IFvraJQvXZb00uCZuyR1aJjvUL0F+DXgZFW9sfW9DrgNmGTuO1SvqaqnkwT4XeBK4CfAe6vqofGULvVtlL/Qjt5w1SpWorPRMGfunwaumNd3ALinqnYC97RlgLcDO9tjP3DT6pQpSVqOJcO9qr4C/HBe9x7gUGsfAq4e6P9MzfkasCXJ1lWqVZI0pJXOuZ9XVSda+/vAea29DXhiYNyx1idJWkMjv6FaVQXUcrdLsj/JdJLp2dnZUcuQJA1Yabg/eXq6pf082fqPAzsGxm1vfT+nqg5W1VRVTU1MTKywDEnSQlYa7oeBfa29D7hjoP89mXMJ8OzA9I0kaY0Mcynk54BLgXOTHAM+CtwA3J7kWuB7wDVt+F3MXQY5w9ylkO8bQ82SpCUsGe5V9a5FVl2+wNgCrhu1KEnSaPyEqiR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0JK3/JV09pk88KUVb3v0hqtWsRKtF8/cJalDhrskdchwl6QOjTTnnuQo8GPgBeBUVU0leR1wGzAJHAWuqaqnRytTkrQcq3Hm/reraldVTbXlA8A9VbUTuKctS5LW0DimZfYAh1r7EHD1GPYhSTqDUcO9gP+R5MEk+1vfeVV1orW/D5w34j4kScs06nXuf7Oqjif5i8DdSb41uLKqKkkttGH7ZbAf4Pzzz19xAaNczwte0yupTyOduVfV8fbzJPCHwG7gySRbAdrPk4tse7CqpqpqamJiYpQyJEnzrDjck/yFJK8+3Qb+DvAIcBjY14btA+4YtUhJ0vKMMi1zHvCHSU4/z3+pqv+e5OvA7UmuBb4HXDN6mZKk5VhxuFfVd4A3LdD/FHD5KEVJkkbjjcMkvYgXKfTB2w9IUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pC3H1gno37EW9qoRvm37a0LVo9n7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDYwv3JFck+XaSmSQHxrUfSdLPG8t17kk2Af8BeCtwDPh6ksNV9dg49iepD+v1+Y8er68f15n7bmCmqr5TVf8PuBXYM6Z9SZLmGdcnVLcBTwwsHwN+dUz7kqR1tRE/lbtutx9Ish/Y3xafS/LtFT7VucAPVlzHJ1a65YqNVO8as9bxsNbxWHGt65kDI+77lxZbMa5wPw7sGFje3vp+qqoOAgdH3VGS6aqaGvV51srZVK+1joe1joe1vti45ty/DuxMckGSlwN7gcNj2pckaZ6xnLlX1akk7wf+CNgE3FJVj45jX5Kknze2Ofequgu4a1zPP2DkqZ01djbVa63jYa3jYa0DUlXj3ockaY15+wFJ6tBZE+5L3c4gyTlJbmvr708yuQ5lnq5lqVrfm2Q2yZH2+IfrUWer5ZYkJ5M8ssj6JPlUey0PJ7l4rWscqGWpWi9N8uzAcf0Xa13jQC07ktyX5LEkjyb5wAJjNsSxHbLWDXFsk7wiyQNJvtFq/fgCYzZEFgxZ6/iyoKo2/IO5N2X/FPjLwMuBbwAXzhvzT4Dfa+29wG0buNb3Av9+vY9rq+VvARcDjyyy/krgy0CAS4D7N3CtlwJ3rvcxbbVsBS5u7VcD/3uBfwcb4tgOWeuGOLbtWL2qtTcD9wOXzBuzUbJgmFrHlgVny5n7MLcz2AMcau3PA5cnyRrWeNpZdeuFqvoK8MMzDNkDfKbmfA3YkmTr2lT3YkPUumFU1Ymqeqi1fww8ztwntwdtiGM7ZK0bQjtWz7XFze0x/43DDZEFQ9Y6NmdLuC90O4P5//h+OqaqTgHPAq9fk+oWqaNZqFaAv9f+FP98kh0LrN8ohn09G8Wb25/BX07yhvUuBqBNC1zE3JnboA13bM9QK2yQY5tkU5IjwEng7qpa9LiucxYMUyuMKQvOlnDvzX8DJqvqrwF387OzDI3mIeCXqupNwL8D/uv6lgNJXgV8AfhgVf1oves5kyVq3TDHtqpeqKpdzH3yfXeSN65XLUsZotaxZcHZEu5L3s5gcEySlwGvAZ5ak+oWqaNZ6NYLT1XV823xPwF/fY1qW4lhjv2GUFU/Ov1ncM19zmJzknPXq54km5kLy89W1RcXGLJhju1StW60Y9vqeAa4D7hi3qqNkgU/tVit48yCsyXch7mdwWFgX2u/E7i32jsWa2zJWufNq76DuTnOjeow8J52ZcclwLNVdWK9i1pIkl88PbeaZDdz/77X5T91q+Nm4PGq+uQiwzbEsR2m1o1ybJNMJNnS2q9k7jsjvjVv2IbIgmFqHWcWrNtdIZejFrmdQZJ/CUxX1WHm/nH+QZIZ5t5027uBa/2nSd4BnGq1vnc9agVI8jnmroQ4N8kx4KPMvfFDVf0ec58yvhKYAX4CvG99Kh2q1ncC/zjJKeD/AnvX6Rc8wFuAdwPfbHOuAB8BzocNd2yHqXWjHNutwKHMfSHQLwC3V9WdGzELhqx1bFngJ1QlqUNny7SMJGkZDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjr0/wEIe5aNT3zVcQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best epsilon computed on GLOVE-TFIDF = 1.4628292329952732 \n",
      "\n",
      "\n",
      "initiating DBSCAN Clustering with tfidf vectors\n",
      "\n",
      "[                                                    ] 0.04%\n",
      " ----  cluster_1_ assigned to 810 points! ----\n",
      "[                                                    ] 0.09%\n",
      " ----  cluster_2_ assigned to 695 points! ----\n",
      "[                                                    ] 0.31%\n",
      " ----  cluster_3_ assigned to 61 points! ----\n",
      "[                                                    ] 0.93%\n",
      " ----  cluster_4_ assigned to 347 points! ----\n",
      "[ =                                                  ] 3.86%\n",
      " ----  cluster_5_ assigned to 26 points! ----\n",
      "[ =============                                      ] 26.14%\n",
      " ----  cluster_6_ assigned to 44 points! ----\n",
      "[ ================                                   ] 32.45%\n",
      " ----  cluster_7_ assigned to 27 points! ----\n",
      "[ ================================================== ] 100.00%\n",
      " 7 clusters formed!\n",
      "Cluster characteristics:\n",
      " -- vectors: tfidf\n",
      " -- minPts: 25\n",
      " -- EstimatedBestDistance 1.4628292329952732\n",
      " -- 7 clusters formed!\n",
      " -- 1995 points assigned to clusters!\n",
      " -- 258 noise points!\n",
      "\n",
      " -- 11.451398135818907 % noise!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# converting tf-idf into vectors\n",
    "fdb.tfidf2vec('tf-only')\n",
    "fdb.tfidf2vec('tf-idf')\n",
    "\n",
    "# clustering documents based on \n",
    "fdbscan2 = fdbscan.flingDBSCAN(fdb.data,None,25,'tfidf')\n",
    "fdbscan2.dbscanCompute()\n",
    "fdbscan2.addClusterLabel('tfidf-clusterID')\n",
    "fdbscan2.printClusterInfo() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Industry</th>\n",
       "      <th>stopsRemoved</th>\n",
       "      <th>tfMatrix</th>\n",
       "      <th>sumTFIDF</th>\n",
       "      <th>glove-vector</th>\n",
       "      <th>glove-clusterID</th>\n",
       "      <th>tfidf2vec-tf</th>\n",
       "      <th>tfidf2vec-tfidf</th>\n",
       "      <th>tfidf-clusterID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Are you eager to roll up your sleeves and harn...</td>\n",
       "      <td>Vera Institute of Justice\\n3.2</td>\n",
       "      <td>Social Assistance</td>\n",
       "      <td>eager roll sleeves harness data drive policy c...</td>\n",
       "      <td>word  tf     tf-idf\n",
       "0          data...</td>\n",
       "      <td>811.569328</td>\n",
       "      <td>[0.20507256798029552, 0.05984949950738914, 0.0...</td>\n",
       "      <td>cluster_0_</td>\n",
       "      <td>[0.2986073091133004, 0.05040200935960588, 0.09...</td>\n",
       "      <td>[0.26263354824176166, -0.023444644206149418, -...</td>\n",
       "      <td>cluster_0_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Overview\\n\\nProvides analytical and technical ...</td>\n",
       "      <td>Visiting Nurse Service of New York\\n3.8</td>\n",
       "      <td>Health Care Services &amp; Hospitals</td>\n",
       "      <td>overview provides analytical technical support...</td>\n",
       "      <td>word  tf    tf-idf\n",
       "0           dat...</td>\n",
       "      <td>415.287583</td>\n",
       "      <td>[0.23643422682926837, -0.055056957317073156, 0...</td>\n",
       "      <td>cluster_1_</td>\n",
       "      <td>[0.4055475764227641, -0.07285501829268287, 0.1...</td>\n",
       "      <td>[0.35240058786555273, -0.1412004425681622, 0.0...</td>\n",
       "      <td>cluster_1_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Weâre looking for a Senior Data Analyst who ...</td>\n",
       "      <td>Squarespace\\n3.4</td>\n",
       "      <td>Internet</td>\n",
       "      <td>weâre looking senior data analyst love mento...</td>\n",
       "      <td>word  tf    tf-idf\n",
       "0          data ...</td>\n",
       "      <td>439.815932</td>\n",
       "      <td>[0.155861351576923, 0.11735425461538473, -0.05...</td>\n",
       "      <td>cluster_2_</td>\n",
       "      <td>[0.283220747730769, 0.14354892653846157, 0.044...</td>\n",
       "      <td>[0.2563749918506738, 0.17575736117618113, -0.0...</td>\n",
       "      <td>cluster_2_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Requisition NumberRR-0001939\\nRemote:Yes\\nWe c...</td>\n",
       "      <td>Celerity\\n4.1</td>\n",
       "      <td>IT Services</td>\n",
       "      <td>requisition numberrr remoteyes collaborate cre...</td>\n",
       "      <td>word  tf     tf-idf\n",
       "0          â¢  ...</td>\n",
       "      <td>569.217931</td>\n",
       "      <td>[0.2306739880813952, 0.09347254534883724, -0.0...</td>\n",
       "      <td>cluster_2_</td>\n",
       "      <td>[0.29634610203488354, 0.10983982558139535, 0.0...</td>\n",
       "      <td>[0.2966705423736133, 0.028126685382837024, -0....</td>\n",
       "      <td>cluster_2_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABOUT FANDUEL GROUP\\n\\nFanDuel Group is a worl...</td>\n",
       "      <td>FanDuel\\n3.9</td>\n",
       "      <td>Sports &amp; Recreation</td>\n",
       "      <td>fanduel group fanduel group worldclass team br...</td>\n",
       "      <td>word  tf     tf-idf\n",
       "0       fanduel...</td>\n",
       "      <td>420.106719</td>\n",
       "      <td>[0.12914707201834857, 0.11582829587155963, 0.0...</td>\n",
       "      <td>cluster_3_</td>\n",
       "      <td>[0.17368260871559627, 0.10919291513761473, 0.0...</td>\n",
       "      <td>[0.021771101166884813, 0.16355587986765768, -0...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2248</th>\n",
       "      <td>Maintains systems to protect data from unautho...</td>\n",
       "      <td>Avacend, Inc.\\n2.5</td>\n",
       "      <td>Staffing &amp; Outsourcing</td>\n",
       "      <td>maintains systems protect data unauthorized us...</td>\n",
       "      <td>word  tf    tf-idf\n",
       "0            ...</td>\n",
       "      <td>43.940807</td>\n",
       "      <td>[0.2738081315789473, -0.001255321052631562, 0....</td>\n",
       "      <td>None</td>\n",
       "      <td>[0.2949110263157894, 0.029555310526315794, 0.0...</td>\n",
       "      <td>[0.23112386279259817, -0.08318866123802247, -0...</td>\n",
       "      <td>cluster_4_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249</th>\n",
       "      <td>Position:\\nSenior Data Analyst (Corporate Audi...</td>\n",
       "      <td>Arrow Electronics\\n2.9</td>\n",
       "      <td>Wholesale</td>\n",
       "      <td>position senior data analyst corporate audit j...</td>\n",
       "      <td>word  tf     tf-idf\n",
       "0      ...</td>\n",
       "      <td>439.042957</td>\n",
       "      <td>[0.2200468355481728, 0.10710706677740867, 0.04...</td>\n",
       "      <td>cluster_1_</td>\n",
       "      <td>[0.3396034966777404, 0.09931764750830561, 0.09...</td>\n",
       "      <td>[0.3077493047461843, 0.06387599003189207, 0.06...</td>\n",
       "      <td>cluster_1_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2250</th>\n",
       "      <td>Title: Technical Business Analyst (SQL, Data a...</td>\n",
       "      <td>Spiceorb</td>\n",
       "      <td>-1</td>\n",
       "      <td>title technical business analyst sql data anal...</td>\n",
       "      <td>word  tf    tf-idf\n",
       "0        busin...</td>\n",
       "      <td>205.978695</td>\n",
       "      <td>[0.36188271052631577, 0.05400915065789475, 0.0...</td>\n",
       "      <td>cluster_2_</td>\n",
       "      <td>[0.5060029144736842, 0.04490494473684211, 0.11...</td>\n",
       "      <td>[0.45506833532863533, 5.3025424212786644e-05, ...</td>\n",
       "      <td>cluster_2_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251</th>\n",
       "      <td>Summary\\n\\nResponsible for working cross-funct...</td>\n",
       "      <td>Contingent Network Services\\n3.1</td>\n",
       "      <td>Enterprise Software &amp; Network Solutions</td>\n",
       "      <td>summary responsible working crossfunctionally ...</td>\n",
       "      <td>word  tf    tf-idf\n",
       "0            ...</td>\n",
       "      <td>364.177527</td>\n",
       "      <td>[0.25247974618181807, 0.07676844581818185, -0....</td>\n",
       "      <td>cluster_2_</td>\n",
       "      <td>[0.34654995709090924, 0.07137524545454547, 0.0...</td>\n",
       "      <td>[0.27937433353352015, 0.08437047685035409, -0....</td>\n",
       "      <td>cluster_1_</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2252</th>\n",
       "      <td>You.\\n\\nYou bring your body, mind, heart and s...</td>\n",
       "      <td>SCL Health\\n3.4</td>\n",
       "      <td>Health Care Services &amp; Hospitals</td>\n",
       "      <td>bring body mind heart spirit work senior quali...</td>\n",
       "      <td>word  tf    tf-idf\n",
       "0          data ...</td>\n",
       "      <td>366.509859</td>\n",
       "      <td>[0.23890638028806577, 0.1815799016460906, -0.0...</td>\n",
       "      <td>cluster_2_</td>\n",
       "      <td>[0.3220337218518514, 0.22893831193415645, 0.07...</td>\n",
       "      <td>[0.2850343471866271, 0.2451438898926933, -0.08...</td>\n",
       "      <td>cluster_2_</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2253 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        Job Description  \\\n",
       "0     Are you eager to roll up your sleeves and harn...   \n",
       "1     Overview\\n\\nProvides analytical and technical ...   \n",
       "2     Weâre looking for a Senior Data Analyst who ...   \n",
       "3     Requisition NumberRR-0001939\\nRemote:Yes\\nWe c...   \n",
       "4     ABOUT FANDUEL GROUP\\n\\nFanDuel Group is a worl...   \n",
       "...                                                 ...   \n",
       "2248  Maintains systems to protect data from unautho...   \n",
       "2249  Position:\\nSenior Data Analyst (Corporate Audi...   \n",
       "2250  Title: Technical Business Analyst (SQL, Data a...   \n",
       "2251  Summary\\n\\nResponsible for working cross-funct...   \n",
       "2252  You.\\n\\nYou bring your body, mind, heart and s...   \n",
       "\n",
       "                                 Company Name  \\\n",
       "0              Vera Institute of Justice\\n3.2   \n",
       "1     Visiting Nurse Service of New York\\n3.8   \n",
       "2                            Squarespace\\n3.4   \n",
       "3                               Celerity\\n4.1   \n",
       "4                                FanDuel\\n3.9   \n",
       "...                                       ...   \n",
       "2248                       Avacend, Inc.\\n2.5   \n",
       "2249                   Arrow Electronics\\n2.9   \n",
       "2250                                 Spiceorb   \n",
       "2251         Contingent Network Services\\n3.1   \n",
       "2252                          SCL Health\\n3.4   \n",
       "\n",
       "                                     Industry  \\\n",
       "0                           Social Assistance   \n",
       "1            Health Care Services & Hospitals   \n",
       "2                                    Internet   \n",
       "3                                 IT Services   \n",
       "4                         Sports & Recreation   \n",
       "...                                       ...   \n",
       "2248                   Staffing & Outsourcing   \n",
       "2249                                Wholesale   \n",
       "2250                                       -1   \n",
       "2251  Enterprise Software & Network Solutions   \n",
       "2252         Health Care Services & Hospitals   \n",
       "\n",
       "                                           stopsRemoved  \\\n",
       "0     eager roll sleeves harness data drive policy c...   \n",
       "1     overview provides analytical technical support...   \n",
       "2     weâre looking senior data analyst love mento...   \n",
       "3     requisition numberrr remoteyes collaborate cre...   \n",
       "4     fanduel group fanduel group worldclass team br...   \n",
       "...                                                 ...   \n",
       "2248  maintains systems protect data unauthorized us...   \n",
       "2249  position senior data analyst corporate audit j...   \n",
       "2250  title technical business analyst sql data anal...   \n",
       "2251  summary responsible working crossfunctionally ...   \n",
       "2252  bring body mind heart spirit work senior quali...   \n",
       "\n",
       "                                               tfMatrix    sumTFIDF  \\\n",
       "0                word  tf     tf-idf\n",
       "0          data...  811.569328   \n",
       "1                 word  tf    tf-idf\n",
       "0           dat...  415.287583   \n",
       "2                word  tf    tf-idf\n",
       "0          data ...  439.815932   \n",
       "3               word  tf     tf-idf\n",
       "0          â¢  ...  569.217931   \n",
       "4                word  tf     tf-idf\n",
       "0       fanduel...  420.106719   \n",
       "...                                                 ...         ...   \n",
       "2248                word  tf    tf-idf\n",
       "0            ...   43.940807   \n",
       "2249                     word  tf     tf-idf\n",
       "0      ...  439.042957   \n",
       "2250               word  tf    tf-idf\n",
       "0        busin...  205.978695   \n",
       "2251                word  tf    tf-idf\n",
       "0            ...  364.177527   \n",
       "2252             word  tf    tf-idf\n",
       "0          data ...  366.509859   \n",
       "\n",
       "                                           glove-vector glove-clusterID  \\\n",
       "0     [0.20507256798029552, 0.05984949950738914, 0.0...      cluster_0_   \n",
       "1     [0.23643422682926837, -0.055056957317073156, 0...      cluster_1_   \n",
       "2     [0.155861351576923, 0.11735425461538473, -0.05...      cluster_2_   \n",
       "3     [0.2306739880813952, 0.09347254534883724, -0.0...      cluster_2_   \n",
       "4     [0.12914707201834857, 0.11582829587155963, 0.0...      cluster_3_   \n",
       "...                                                 ...             ...   \n",
       "2248  [0.2738081315789473, -0.001255321052631562, 0....            None   \n",
       "2249  [0.2200468355481728, 0.10710706677740867, 0.04...      cluster_1_   \n",
       "2250  [0.36188271052631577, 0.05400915065789475, 0.0...      cluster_2_   \n",
       "2251  [0.25247974618181807, 0.07676844581818185, -0....      cluster_2_   \n",
       "2252  [0.23890638028806577, 0.1815799016460906, -0.0...      cluster_2_   \n",
       "\n",
       "                                           tfidf2vec-tf  \\\n",
       "0     [0.2986073091133004, 0.05040200935960588, 0.09...   \n",
       "1     [0.4055475764227641, -0.07285501829268287, 0.1...   \n",
       "2     [0.283220747730769, 0.14354892653846157, 0.044...   \n",
       "3     [0.29634610203488354, 0.10983982558139535, 0.0...   \n",
       "4     [0.17368260871559627, 0.10919291513761473, 0.0...   \n",
       "...                                                 ...   \n",
       "2248  [0.2949110263157894, 0.029555310526315794, 0.0...   \n",
       "2249  [0.3396034966777404, 0.09931764750830561, 0.09...   \n",
       "2250  [0.5060029144736842, 0.04490494473684211, 0.11...   \n",
       "2251  [0.34654995709090924, 0.07137524545454547, 0.0...   \n",
       "2252  [0.3220337218518514, 0.22893831193415645, 0.07...   \n",
       "\n",
       "                                        tfidf2vec-tfidf tfidf-clusterID  \n",
       "0     [0.26263354824176166, -0.023444644206149418, -...      cluster_0_  \n",
       "1     [0.35240058786555273, -0.1412004425681622, 0.0...      cluster_1_  \n",
       "2     [0.2563749918506738, 0.17575736117618113, -0.0...      cluster_2_  \n",
       "3     [0.2966705423736133, 0.028126685382837024, -0....      cluster_2_  \n",
       "4     [0.021771101166884813, 0.16355587986765768, -0...            None  \n",
       "...                                                 ...             ...  \n",
       "2248  [0.23112386279259817, -0.08318866123802247, -0...      cluster_4_  \n",
       "2249  [0.3077493047461843, 0.06387599003189207, 0.06...      cluster_1_  \n",
       "2250  [0.45506833532863533, 5.3025424212786644e-05, ...      cluster_2_  \n",
       "2251  [0.27937433353352015, 0.08437047685035409, -0....      cluster_1_  \n",
       "2252  [0.2850343471866271, 0.2451438898926933, -0.08...      cluster_2_  \n",
       "\n",
       "[2253 rows x 11 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdb.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}