stop-and-search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "import numpy as np\n",
    "import scipy.optimize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Police dataset\n",
    "This dataset is from [https://data.police.uk/](https://data.police.uk/). Code for retrieving the dataset is given at the bottom."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file already downloaded\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "if os.path.exists('stop-and-search.csv'):\n",
    "    print(\"file already downloaded\")\n",
    "else:\n",
    "    !wget \"https://www.cl.cam.ac.uk/teaching/2021/DataSci/data/stop-and-search.csv\"\n",
    "police = pandas.read_csv('stop-and-search.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic tabulations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A no further action disposal                                    467106\n",
      "False                                                           239660\n",
      "Arrest                                                           93384\n",
      "Suspect arrested                                                 63191\n",
      "Community resolution                                             35319\n",
      "Offender given drugs possession warning                          26030\n",
      "Khat or Cannabis warning                                         18668\n",
      "Summons / charged by post                                        11168\n",
      "Penalty Notice for Disorder                                      10628\n",
      "Local resolution                                                  8164\n",
      "Article found - Detailed outcome unavailable                      6705\n",
      "Suspect summonsed to court                                        5758\n",
      "Offender given penalty notice                                     5623\n",
      "Caution (simple or conditional)                                   2947\n",
      "Offender cautioned                                                1747\n",
      "Suspected psychoactive substances seized - No further action        17\n",
      "Name: outcome, dtype: int64\n",
      "Missing values: 17800\n"
     ]
    }
   ],
   "source": [
    "print(police['outcome'].value_counts())\n",
    "print('Missing values:', np.sum(pandas.isna(police['outcome'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "White    532584\n",
      "Black    253315\n",
      "Asian    125646\n",
      "Other     27809\n",
      "Mixed      1644\n",
      "Name: officer_defined_ethnicity, dtype: int64\n",
      "Missing values: 72917\n"
     ]
    }
   ],
   "source": [
    "print(police['officer_defined_ethnicity'].value_counts())\n",
    "print('Missing values:', np.sum(pandas.isna(police['officer_defined_ethnicity'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Male      861605\n",
      "Female     76568\n",
      "Other       2467\n",
      "Name: gender, dtype: int64\n",
      "Missing values: 73275\n"
     ]
    }
   ],
   "source": [
    "print(police['gender'].value_counts())\n",
    "print('Missing values:', np.sum(pandas.isna(police['gender'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4 Analysis of gender + ethnicity as they affect police actions\n",
    "\n",
    "### Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's treat outcomes \"A no further action disposal\" and \"False\" as nothing found\n",
    "# Prepare vectors y=1[something found], eth, and gender,\n",
    "# and remove rows with missing values.\n",
    "\n",
    "df = police[['outcome', 'officer_defined_ethnicity', 'gender']].dropna()\n",
    "df['outcome'] = np.where(df['outcome'].isin(['False','A no further action disposal']), 'nothing', 'find')\n",
    "\n",
    "y = np.where(df['outcome']=='find', 1, 0)\n",
    "eth = df['officer_defined_ethnicity']\n",
    "gender = df['gender']\n",
    "\n",
    "# Define e and g to be integer-encoded versions of ethnicity and gender.\n",
    "# (For the purposes of this analysis, it'll be more useful than one-hot encoding.)\n",
    "\n",
    "ethnicity_levels = ['Asian','Black','Mixed','Other','White']\n",
    "gender_levels = ['Female', 'Male', 'Other']\n",
    "assert all(eth.isin(ethnicity_levels))\n",
    "assert all(gender.isin(gender_levels))\n",
    "ethnicity_code = {k:i for i,k in enumerate(ethnicity_levels)}\n",
    "gender_code = {k:i for i,k in enumerate(gender_levels)}\n",
    "e = numpy.array([ethnicity_code[v] for v in eth])\n",
    "g = numpy.array([gender_code[v] for v in gender])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "officer_defined_ethnicity  Asian   Black  Mixed  Other   White\n",
      "outcome gender                                                \n",
      "find    Female              1051    3327     36    346   15789\n",
      "        Male               31557   66128    468   7293  138926\n",
      "        Other                 21      40      0     32     135\n",
      "nothing Female              2375    8189    108    706   41064\n",
      "        Male               79242  172521    996  17356  317957\n",
      "        Other                 97     157      3     71     460\n"
     ]
    }
   ],
   "source": [
    "x = df.groupby(['outcome','gender','officer_defined_ethnicity']).apply(len).unstack(fill_value=0)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A simple Bernoulli model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-71-d0cb5bccaf20>:3: RuntimeWarning: invalid value encountered in log\n",
      "  return np.sum(np.log(np.where(y==1, ξ, 1-ξ)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 547602.977403\n",
      "         Iterations: 334\n",
      "         Function evaluations: 549\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Asian    0.285352\n",
       "Black    0.277579\n",
       "Mixed    0.312868\n",
       "Other    0.297266\n",
       "White    0.301073\n",
       "dtype: float64"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def loglik1(β):\n",
    "    ξ = β[e] # get a vector [β[e_1], β[e_2], ..., β[e_n]]\n",
    "    return np.sum(np.log(np.where(y==1, ξ, 1-ξ)))\n",
    "\n",
    "initial_guess = np.full(5,.1)\n",
    "βhat = scipy.optimize.fmin(lambda β: -loglik1(β), initial_guess)\n",
    "\n",
    "pandas.Series(βhat, index=ethnicity_levels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Asian     Black     Mixed     Other     White\n",
      " 0.28233  0.281194  0.319455  0.266409  0.301475\n"
     ]
    }
   ],
   "source": [
    "x = pandas.Series(βhat, index=ethnicity_levels)\n",
    "print(x.to_frame().T.to_string(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A Bernoulli model with interaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 547547.227247\n",
      "         Iterations: 675\n",
      "         Function evaluations: 999\n"
     ]
    }
   ],
   "source": [
    "def loglik2(θ):\n",
    "    β, γ = θ[:5], θ[5:]\n",
    "    ξ = β[e] + γ[g]\n",
    "    return np.sum(np.log(np.where(y==1, ξ, 1-ξ)))\n",
    "\n",
    "initial_guess = np.full(8, .2)\n",
    "mle = scipy.optimize.fmin(lambda θ: -loglik2(θ), initial_guess)\n",
    "βhat, γhat = mle[:5], mle[5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.34922542, 0.34169947, 0.37748171, 0.3616269 , 0.36619801]),\n",
       " array([-0.07964008, -0.0632652 , -0.13476383]))"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "βhat, γhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Asian  Black  Mixed  Other  White\n",
      "Female  0.270  0.262  0.298  0.282  0.287\n",
      "Male    0.286  0.278  0.314  0.298  0.303\n",
      "Other   0.214  0.207  0.243  0.227  0.231\n"
     ]
    }
   ],
   "source": [
    "ξhat = βhat[np.newaxis, :] + γhat[:, np.newaxis]\n",
    "print(pandas.DataFrame(np.round(ξhat,3), index=gender_levels, columns=ethnicity_levels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 547592.263723\n",
      "         Iterations: 423\n",
      "         Function evaluations: 678\n"
     ]
    }
   ],
   "source": [
    "# Identifiable version of the model\n",
    "\n",
    "def loglik2b(θ):\n",
    "    α, β, γ = θ[0], θ[1:5], θ[5:]\n",
    "    β, γ = np.insert(β,0,0), np.insert(γ,0,0)\n",
    "    ξ = α + β[e] + γ[g]\n",
    "    return np.sum(np.log(np.where(y==1, ξ, 1-ξ)))\n",
    "\n",
    "initial_guess = np.array([.5, 0,0,0,0, 0,0])\n",
    "θhat = scipy.optimize.fmin(lambda θ: -loglik2b(θ), initial_guess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Asian  Black  Mixed  Other  White\n",
      "Female  0.270  0.265  0.261  0.262  0.287\n",
      "Male    0.286  0.281  0.277  0.277  0.303\n",
      "Other   0.271  0.266  0.262  0.262  0.288\n"
     ]
    }
   ],
   "source": [
    "α,β,γ = θhat[0], θhat[1:5], θhat[5:]\n",
    "β,γ = np.insert(β,0,0), np.insert(γ,0,0)\n",
    "ξ = α + β[np.newaxis,:] + γ[:,np.newaxis]\n",
    "print(pandas.DataFrame(np.round(ξ,3), index=gender_levels, columns=ethnicity_levels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Natural parameters (full logistic regression)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 547581.066001\n",
      "         Iterations: 640\n",
      "         Function evaluations: 984\n"
     ]
    }
   ],
   "source": [
    "def loglik3(θ):\n",
    "    α, β, γ = θ[0], θ[1:5], θ[5:]\n",
    "    β, γ = np.insert(β,0,0), np.insert(γ,0,0)\n",
    "    ξ = α + β[e] + γ[g]\n",
    "    p1 = np.exp(ξ) / (1 + np.exp(ξ))\n",
    "    return np.sum(np.where(y==1, np.log(p1), np.log(1-p1)))\n",
    "    \n",
    "initial_guess = np.array([.5, 0,0,0,0, 0,0])\n",
    "θhat = scipy.optimize.fmin(lambda θ: -loglik3(θ), initial_guess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Asian  Black  Mixed  Other  White\n",
      "Female  0.278  0.264  0.287  0.271  0.288\n",
      "Male    0.291  0.278  0.301  0.284  0.302\n",
      "Other   0.291  0.278  0.301  0.284  0.302\n"
     ]
    }
   ],
   "source": [
    "α,β,γ = θhat[0], θhat[1:5], θhat[5:]\n",
    "β,γ = np.insert(β,0,0), np.insert(γ,0,0)\n",
    "ξ = α + β[np.newaxis,:] + γ[:,np.newaxis]\n",
    "p = np.exp(ξ) / (1 + np.exp(ξ))\n",
    "print(pandas.DataFrame(numpy.round(p,3), index=gender_levels, columns=ethnicity_levels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Asian     Black     Mixed     Other     White\n",
      "   0.0 -0.066871  0.047821 -0.034717  0.052221\n"
     ]
    }
   ],
   "source": [
    "x = pandas.Series(β, index=ethnicity_levels)\n",
    "print(x.to_frame().T.to_string(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Female      Male     Other\n",
      "    0.0  0.067568  0.067881\n"
     ]
    }
   ],
   "source": [
    "x = pandas.Series(γ, index=gender_levels)\n",
    "print(x.to_frame().T.to_string(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieving the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas\n",
    "import requests, requests_cache, urllib, io\n",
    "import json, re\n",
    "import collections, numbers\n",
    "requests_cache.install_cache('data/cache')\n",
    "BASE_URL = 'https://data.police.uk/api/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetch all available forces*dates, and cache them\n",
    "availability_url = urllib.parse.urljoin(BASE_URL, 'crimes-street-dates')\n",
    "availability = requests.get(availability_url).json()\n",
    "for df in availability:\n",
    "    date = df['date']\n",
    "    print(date)\n",
    "    for force in df['stop-and-search']:\n",
    "        p = [('force',force), ('date',date)]\n",
    "        url = urllib.parse.urljoin(BASE_URL, 'stops-force') + '?' + urllib.parse.urlencode(p)\n",
    "        requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetch a list of all forces * dates\n",
    "availability_url = urllib.parse.urljoin(BASE_URL, 'crimes-street-dates')\n",
    "availability = requests.get(availability_url).json()\n",
    "availability = [(force, df['date']) for df in availability for force in df['stop-and-search']]\n",
    "\n",
    "def get_dataframe(force, date):\n",
    "    p = [('force',force), ('date',date)]\n",
    "    url = urllib.parse.urljoin(BASE_URL, 'stops-force') + '?' + urllib.parse.urlencode(p)\n",
    "    response = requests.get(url)\n",
    "    df = response.json()\n",
    "    df = pandas.json_normalize(df, sep='_')\n",
    "    df.insert(0, 'month', date)\n",
    "    df.insert(0, 'force', force)\n",
    "    return df\n",
    "df = [get_dataframe(*x) for x in availability]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "police = pandas.concat(df, axis=0, ignore_index=True, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "police.to_csv('data/stop-and-search.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}