forked from CoD-Segfault/covid_il_bot
-
Notifications
You must be signed in to change notification settings - Fork 0
/
get_data.py
149 lines (117 loc) · 6.66 KB
/
get_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
import requests
from datetime import datetime, date, timedelta
import time
# formats date to ISO 8601
def format_date(date):
return date.strftime("%Y-%m-%d")
# takes date from IDPH source and strips time value
def import_date(date, add_day=False, cdc=False):
if cdc:
imported_date = datetime.strptime(date,"%Y-%m-%dT%H:%M:%S.%f")
else:
imported_date = datetime.strptime(date,"%Y-%m-%dT%H:%M:%S")
if add_day:
imported_date = imported_date + timedelta(1)
formatted_date = format_date(imported_date)
return formatted_date
# Pulls all data from IDPH and combines them to a single dictionary using the date as the key
def get_idph_data():
# Get today's date and format it how needed
today = date.today()
today_formatted = format_date(today)
# data source for tests and deaths
test_url = "https://idph.illinois.gov/DPHPublicInformation/api/COVIDExport/GetIllinoisCases"
# data source for hospital, ICU, and ventilator utilization
hospital_url = "https://idph.illinois.gov/DPHPublicInformation/api/COVIDExport/GetHospitalUtilizationResults"
# data source for vaccination info
#vaccine_url = "https://idph.illinois.gov/DPHPublicInformation/api/COVIDExport/GetVaccineAdministration?countyname="
# CDC vaccination data source
cdc_vaccine_url = "https://data.cdc.gov/resource/unsk-b7fc.json?location=IL"
# grab all the data sources
test_data = requests.get(test_url)
hospital_data = requests.get(hospital_url)
#vaccine_data = requests.get(vaccine_url)
cdc_vaccine_data = requests.get(cdc_vaccine_url)
# create a dictionary to combine all data by date
combined_data = dict()
# get relevant info for tests and deaths
for day in test_data.json():
day_date = day['testDate']
day_cases = day['cases_change']
day_deaths = day['deaths_change']
day_tested = day['tested_change']
normalized_date = import_date(day_date)
# add day if it doesn't exist
if normalized_date not in combined_data:
combined_data[normalized_date] = dict()
combined_data[normalized_date]['cases'] = day_cases
combined_data[normalized_date]['deaths'] = day_deaths
combined_data[normalized_date]['tested'] = day_tested
# get relevant info for hospitalizations, etc.
for day in hospital_data.json():
day_date = day['ReportDate']
day_covid_vent = day['VentilatorInUseCOVID']
day_covid_icu = day['ICUInUseBedsCOVID']
day_covid_beds = day['TotalInUseBedsCOVID']
# data delayed by one day, adjusting to match official reports
normalized_date = import_date(day_date, add_day=True)
# add day if it doesn't exist
if normalized_date not in combined_data:
combined_data[normalized_date] = dict()
combined_data[normalized_date]['covid_vent'] = day_covid_vent
combined_data[normalized_date]['covid_icu'] = day_covid_icu
combined_data[normalized_date]['covid_beds'] = day_covid_beds
# get relevant info for vaccinations. switching to CDC data
""" for day in vaccine_data.json():
day_date = day['Report_Date']
day_vaccines_administered = day['AdministeredCountChange']
day_vaccines_rolling_avg = day['AdministeredCountRollAvg']
day_percent_vaccinated = day['PctVaccinatedPopulation']
# percent vaccinated is provided as a ratio, converting to actual percent to 2 decimal places
day_percent_vaccinated = round(day_percent_vaccinated * 100, 2)
# data delayed by one day, adjusting to match official reports
normalized_date = import_date(day_date, add_day=True)
# add day if it doesn't exist
if normalized_date not in combined_data:
combined_data[normalized_date] = dict()
combined_data[normalized_date]['vaccine_doses'] = day_vaccines_administered
combined_data[normalized_date]['vaccine_rolling_average'] = day_vaccines_rolling_avg
combined_data[normalized_date]['total_population_percent_vaccinated'] = day_percent_vaccinated
"""
# Check to make sure that the data for today is available, otherwise try again in 30 seconds.
if today_formatted not in combined_data:
print("Data not abailable yet, pausing 30 seconds.")
time.sleep(30)
combined_data = get_idph_data()
# Ingest CDC data
for day in cdc_vaccine_data.json():
day_date = day['date']
day_vaccines_administered_total = day['administered']
day_vaccines_administered_12plus = day['administered_12plus']
day_vaccines_administered_18plus = day['administered_18plus']
day_vaccines_administered_65plus = day['administered_65plus']
first_dose_percent_total = day['administered_dose1_pop_pct']
first_dose_percent_12plus = day['administered_dose1_recip_2']
first_dose_percent_18plus = day['administered_dose1_recip_4']
first_dose_percent_65plus = day['administered_dose1_recip_6']
fully_vaccinated_total = day['series_complete_pop_pct']
fully_vaccinated_12plus = day['series_complete_12pluspop']
fully_vaccinated_18plus = day['series_complete_18pluspop']
fully_vaccinated_65plus = day['series_complete_65pluspop']
normalized_date = import_date(day_date, add_day=True, cdc=True)
# add day if it doesn't exist
if normalized_date not in combined_data:
combined_data[normalized_date] = dict()
combined_data[normalized_date]['vaccines_administered_total'] = day_vaccines_administered_total
combined_data[normalized_date]['vaccines_administered_12plus'] = day_vaccines_administered_12plus
combined_data[normalized_date]['vaccines_administered_18plus'] = day_vaccines_administered_18plus
combined_data[normalized_date]['vaccines_administered_65plus'] = day_vaccines_administered_65plus
combined_data[normalized_date]['vaccines_first_dose_percent_total'] = first_dose_percent_total
combined_data[normalized_date]['vaccines_first_dose_percent_12plus'] = first_dose_percent_12plus
combined_data[normalized_date]['vaccines_first_dose_percent_18plus'] = first_dose_percent_18plus
combined_data[normalized_date]['vaccines_first_dose_percent_65plus'] = first_dose_percent_65plus
combined_data[normalized_date]['fully_vaccinated_percent_total'] = fully_vaccinated_total
combined_data[normalized_date]['fully_vaccinated_percent_12plus'] = fully_vaccinated_12plus
combined_data[normalized_date]['fully_vaccinated_percent_18plus'] = fully_vaccinated_18plus
combined_data[normalized_date]['fully_vaccinated_percent_65plus'] = fully_vaccinated_65plus
return combined_data