DataScienceIBM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook will be mainly used for the capstone project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello Capstone Project Course!\n"
     ]
    }
   ],
   "source": [
    "print \"Hello Capstone Project Course!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!conda install -c anaconda beautifulsoup4 \n",
    "!conda install -c anaconda lxml\n",
    "!conda install -c anaconda html5lib\n",
    "!conda install -c anaconda requests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import requests\n",
    "import pandas as pd\n",
    "website_url = requests.get('https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M').text\n",
    "from bs4 import BeautifulSoup\n",
    "soup = BeautifulSoup(website_url,'lxml')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Insert Wikipedia data into table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "My_table = soup.find('table',{'class':'wikitable sortable'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parse table and create dataframe according to the given conditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table_rows = My_table.find_all('tr')\n",
    "\n",
    "PostCode =[]\n",
    "Borough = []\n",
    "Neighbourhood = []\n",
    "\n",
    "for tr in table_rows:\n",
    "    td = tr.find_all('td')\n",
    "    if (len(td) > 0 and td[1].text != 'Not assigned'): #check if it valid row and if borough!=\"Not Assigned\"\n",
    "        if td[0].text not in PostCode: #if postcode not present before then create new\n",
    "            PostCode.append((td[0].text)) #append postcode in array\n",
    "            a = td[1].find_all('a')\n",
    "            if(len(a) > 0):\n",
    "                Borough.append((((td[1].find('a')).get('title')).split(','))[0]) #append borough in array for hyperlink\n",
    "            else:\n",
    "                Borough.append((((td[1].text)).split(','))[0]) #append borough in array for non-hyperlink\n",
    "            a = td[2].find_all('a')\n",
    "            if(len(a) > 0):\n",
    "                Neighbourhood.append((((td[2].find('a')).get('title')).split(','))[0]) #append neighbourhood in array for hyperlink\n",
    "            else:   #append neighbourhood in array for non-hyperlink\n",
    "                if td[2].text == 'Not assigned\\n': \n",
    "                    Neighbourhood.append(((td[1].text).split(','))[0])#append borough if non assigned\n",
    "                else:\n",
    "                    Neighbourhood.append((td[2].text).replace('\\n', '')) # append neighbourhood if non-hyperlink\n",
    "        else: #if postcode is present in the array\n",
    "            ind = PostCode.index(td[0].text)\n",
    "            a = td[2].find_all('a')\n",
    "            if(len(a) > 0):\n",
    "                testneighbourhood = (((td[2].find('a')).get('title')).split(','))[0] #append neighbourhood in array for hyperlink\n",
    "            else:   #append neighbourhood in array for non-hyperlink\n",
    "                if td[2].text != 'Not assigned\\n':\n",
    "                    testneighbourhood = (td[2].text).replace('\\n', '') # append neighbourhood if non-hyperlink\n",
    "            if Borough[ind] == Neighbourhood[ind]:\n",
    "                Neighbourhood[ind] = testneighbourhood\n",
    "            else:\n",
    "                Neighbourhood[ind] = Neighbourhood[ind] + ', ' + testneighbourhood           \n",
    "        \n",
    "df=pd.DataFrame()\n",
    "df['PostCode'] = PostCode\n",
    "df['Borough'] = Borough\n",
    "df['Neighbourhood'] = Neighbourhood"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check shape and dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df.shape\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using geocoder to import latitude and longitude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!conda install -c conda-forge geocoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Populate latitude, longitude using geocoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geocoder # import geocoder\n",
    "\n",
    "# initialize your variable to None\n",
    "lat_lng_coords = None\n",
    "latitude = []\n",
    "longitude = []\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    # loop until you get the coordinates\n",
    "    while(lat_lng_coords is None):\n",
    "        g = geocoder.google('{}, Toronto, Ontario'.format(row['PostCode']))\n",
    "        lat_lng_coords = g.latlng\n",
    "    latitude.append(g.latlng[0])\n",
    "    longitude.append(g.latlng[1])\n",
    "df['Latitude'] = latitude\n",
    "df['Latitude'] = longitude"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Populate latitide, longitude using excel sheet¶\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "latitude = []\n",
    "longitude = []\n",
    "latlong = pd.read_csv('http://cocl.us/Geospatial_data')\n",
    "#print((latlong.loc[latlong['Postal Code'] == 'M3A'])['Postal Code'])\n",
    "for index, row in df.iterrows():\n",
    "    templatlong = latlong[latlong['Postal Code'].str.match(row['PostCode'])]\n",
    "    latitude.append((templatlong['Latitude'].values)[0])\n",
    "    longitude.append((templatlong['Longitude'].values)[0])\n",
    "df['Latitude'] = latitude\n",
    "df['Longitude'] = longitude"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering Toronto Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a dataframe with borough's in Toronto¶\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toronto_data = df[df['Borough'].str.contains('Toronto')].reset_index(drop=True)\n",
    "toronto_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "toronto_data_grouped = toronto_data.groupby('Neighbourhood').mean().reset_index()\n",
    "toronto_data_grouped.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K Means Clustering¶\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set number of clusters\n",
    "from sklearn.cluster import KMeans\n",
    "kclusters = 5\n",
    "\n",
    "toronto_grouped_clustering = toronto_data_grouped.drop('Neighbourhood', 1)\n",
    "\n",
    "# run k-means clustering\n",
    "kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(toronto_grouped_clustering)\n",
    "\n",
    "# check cluster labels generated for each row in the dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insert cluster information to original dataframe¶\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toronto_data['Cluster Labels']=kmeans.labels_\n",
    "toronto_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "address = 'Toronto, Ontario'\n",
    "geolocator = Nominatim(user_agent=\"tor_explorer\")\n",
    "location = geolocator.geocode(address)\n",
    "latitude = location.latitude\n",
    "longitude = location.longitude\n",
    "print('The geographical coordinate of Toronto are {}, {}.'.format(latitude, longitude))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing libraries for maps¶\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!conda install -c conda-forge folium=0.5.0 --yes \n",
    "import folium \n",
    "!conda install -c conda-forge geopy --yes \n",
    "from geopy.geocoders import Nominatim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting clusters on the map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.colors as colors\n",
    "\n",
    "map_clusters = folium.Map(location=[latitude, longitude], zoom_start=11)\n",
    "\n",
    "# set color scheme for the clusters\n",
    "x = np.arange(kclusters)\n",
    "ys = [i + x + (i*x)**2 for i in range(kclusters)]\n",
    "colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))\n",
    "rainbow = [colors.rgb2hex(i) for i in colors_array]\n",
    "\n",
    "# add markers to the map\n",
    "markers_colors = []\n",
    "for lat, lon, poi, cluster in zip(toronto_data['Latitude'], toronto_data['Longitude'], toronto_data['Neighbourhood'], toronto_data['Cluster Labels']):\n",
    "    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)\n",
    "    folium.CircleMarker(\n",
    "        [lat, lon],\n",
    "        radius=5,\n",
    "        popup=label,\n",
    "        color=rainbow[cluster-1],\n",
    "        fill=True,\n",
    "        fill_color=rainbow[cluster-1],\n",
    "        fill_opacity=0.7).add_to(map_clusters)\n",
    "       \n",
    "map_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}