Missing_data_handling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Handling Missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"weather_data.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(df.day[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"weather_data.csv\", parse_dates=[\"day\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(df.day[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.set_index('day', inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace all NaN elements with 0s.\n",
    "new_df = df.fillna(0)\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace all NaN elements according to different columns\n",
    "new_df = df.fillna({\n",
    "    'temperature':0,\n",
    "    'windspeed':0,\n",
    "    'event': 'no event'\n",
    "})\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# propagate non-null values forward.\n",
    "new_df=df.fillna(method =\"ffill\")\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# propagate non-null values backward.\n",
    "new_df = df.fillna(method=\"bfill\")\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace all NaN elements as the next column\n",
    "new_df = df.fillna(method=\"bfill\", axis=\"columns\")\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace the first NaN element.\n",
    "new_df = df.fillna(method=\"bfill\", limit=1)\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = df.interpolate()\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Interpolate based on time.\n",
    "new_df = df.interpolate(method=\"time\")\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop the rows where at least one element is missing.\n",
    "new_df = df.dropna()\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop the columns where at least one element is missing.\n",
    "new_df=df.dropna(axis=\"columns\")\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop the rows where all elements are missing.\n",
    "new_df = df.dropna(how=\"all\")\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep only the rows with at least 2 non-NA values.\n",
    "new_df = df.dropna(thresh = 2)\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define in which columns to look for missing values.\n",
    "new_df = df.dropna(subset=['temperature', 'windspeed'])\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# insert missing dates\n",
    "dt = pd.date_range(\"2018-01-01\",\"2018-01-11\")\n",
    "idx = pd.DatetimeIndex(dt)\n",
    "new_df = df.reindex(idx)\n",
    "new_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}