-
Notifications
You must be signed in to change notification settings - Fork 0
/
dc2018.py
109 lines (69 loc) · 3.81 KB
/
dc2018.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#!/usr/bin/env python
# coding: utf-8
# - Add columns {'date', 'county_fips', 'jurisdiction_fips', 'magnitude', 'readme_check'}.
# - Remove periods and commas from candidate names. Be careful of double initial candidates, make sure the names remain separated (e.g. D.L. HUMPHREY to D L HUMPHREY).
# - Replace parentheses surrounding nicknames with double quotation marks (e.g. JOYCE (CHESTNUT) ROBINSON-PAUL to JOYCE "CHESTNUT" ROBINSON-PAUL). Similarly for single quotation marks (e.g. DUSTIN 'DC' CANTER to DUSTIN "DC" CANTER).
# - Replace candidate name [WRITE-IN] with WRITEIN.
# - Remove the double space in candidate name RENEE L. BOWSER (so RENEE L BOWSER).
# - Investigate GORDON - ANDREW FLETCHER and JOYCE ROBINSON - PAUL and see if they are double names (in which case, you'd want to remove the spaces surrounding the -), or a coding error.
# In[1]:
import pandas as pd
import numpy as np
import os
import csv
df = pd.read_csv('2018-dc-precinct-autoadapted.csv')
df = df.fillna("")
# In[5]:
df
# In[3]:
df['date'] = '2018-11-06'
df['readme_check'] = 'FALSE'
df['magnitude'] = 1
# In[4]:
# After county name fix, append on fips codes
fips = pd.read_csv('../../../help-files/county-fips-codes.csv')
fips = fips.applymap(str)
fips['state'] = fips['state'].str.upper()
df=df.applymap(lambda x: x.strip() if type(x)==str else x)
df = pd.merge(df, fips, on = ['state','county_name'],
how = 'left')
# In[6]:
df['jurisdiction_fips'] = df['county_fips']
# In[9]:
df['candidate'] = df['candidate'].str.replace(".", " ",regex=True)
df['candidate'] = df['candidate'].str.replace(",", " ",regex=True)
df['candidate'] = df['candidate'].str.replace(')', '"',regex=True)
df['candidate'] = df['candidate'].str.replace('(', '"',regex=True)
df['candidate'] = df['candidate'].str.replace("' ", '" ',regex=True)
df['candidate'] = df['candidate'].str.replace(" '", ' "',regex=True)
df['candidate'] = df['candidate'].str.replace('\s+', ' ',regex=True)
# In[10]:
df['candidate'] = df['candidate'].replace({'[WRITE-IN]': 'WRITEIN',
'JOYCE ROBINSON - PAUL': 'JOYCE ROBINSON-PAUL',
'GORDON - ANDREW FLETCHER': 'GORDON-ANDREW FLETCHER',
'"TERRY" TERESA STITH':'TERESA "TERRY" STITH'})
### DC fixes ##################################################################
# district fixes
df['district'] = df['district'].replace(['WARD 6','WARD 1','WARD 3','WARD 5'],
['006','001','003','005'])
# dataverse fixes
df.loc[df['office'] == 'US HOUSE','dataverse'] = 'HOUSE'
df.loc[df['dataverse'] == 'HOUSE','district'] = '000'
df.loc[df['office'] == 'US SENATE','dataverse'] = 'SENATE'
df.loc[df['office'] == 'BALLOTS CAST','dataverse'] = ''
# mag
df.loc[df['office'] == 'BALLOTS CAST','magnitude'] = 0
#party
df.loc[df['party_detailed']=='NON-PARTISAN','party_detailed'] = 'NONPARTISAN'
df.loc[df['party_detailed']=='NONPARTISAN','party_simplified'] = 'NONPARTISAN'
# bools
df = df.replace([True,False],['TRUE','FALSE'])
df.loc[df['office'].isin(['MAYOR', 'CHAIRMAN OF THE DC COUNCIL',
'ATTORNEY GENERAL', 'US SENATE']),'district'] = 'STATEWIDE'
#################################################################################
# Final step: Remove all trailing white space and put columns in correct order.
df=df.applymap(lambda x: x.strip() if type(x)==str else x)
df=df[["precinct", "office", "party_detailed", "party_simplified", "mode", "votes", "county_name", "county_fips", "jurisdiction_name",
"jurisdiction_fips", "candidate", "district", "dataverse", "year", "stage", "state", "special", "writein", "state_po",
"state_fips", "state_cen", "state_ic", "date", "readme_check", "magnitude"]]
df.to_csv('2018-dc-precinct-general-updated.csv',quoting=csv.QUOTE_NONNUMERIC,index=False)