-
Notifications
You must be signed in to change notification settings - Fork 0
/
run.sh
executable file
·171 lines (108 loc) · 7.19 KB
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
#!/usr/bin/env bash
# Cleans the local environment
./clearAll.sh
# Creates the output subdirectories
mkdir results
mkdir results/create_tables
mkdir results/insert_queries
mkdir results/insert_queries/raw
###############################################
############## AREECENSIMENTO #################
###############################################
# Parses the CSV file and specifies the column division
python parseCSV.py R04_indicatori_2011_areecensimento.csv 6,6,-1 83,7_6,6 98,7_6,6 104,7_6,6 116,7_6,6 147,7_6,6
# Merges the files contained in the first subdirectory
python mergeFiles.py results/create_tables/ .sql
# Merges the files contained in the second subdirectory
python mergeFiles.py results/insert_queries/ .sql
#Remove duplicates
python removeDuplicates.py results/insert_queries/DIRECTORY_MERGE.sql
#Remapping table names
python remap.py results/create_tables/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:POPOLAZIONE_RESIDENTE census_area_2:STRANIERI_RESIDENTI census_area_3:ABITAZIONI census_area_4:FAMIGLIE census_area_5:EDIFICI
python remap.py results/insert_queries/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:POPOLAZIONE_RESIDENTE census_area_2:STRANIERI_RESIDENTI census_area_3:ABITAZIONI census_area_4:FAMIGLIE census_area_5:EDIFICI
#Merge all DIRECTORY_MERGE
python mergeAll.py DIRECTORY_MERGE.sql DB_areecensimento.sql drop_all_tables.sql
# Gets raw data from the insert queries
python getRawData.py results/insert_queries/
###############################################
################# LOCALITA ####################
###############################################
./clearResults.sh
python parseCSV.py R04_indicatori_2011_localita.csv 6,6,-1 13,7,6 89,7,7 104,7,7 110,7,7 122,7,7 153,7,7
# Merges the files contained in the first subdirectory
python mergeFiles.py results/create_tables/ .sql
# Merges the files contained in the second subdirectory
python mergeFiles.py results/insert_queries/ .sql
#Remove duplicates
python removeDuplicates.py results/insert_queries/DIRECTORY_MERGE.sql
#Remapping table names
python remap.py results/create_tables/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:LOCALITA census_area_2:POPOLAZIONE_RESIDENTE census_area_3:STRANIERI_RESIDENTI census_area_4:ABITAZIONI census_area_5:FAMIGLIE census_area_6:EDIFICI
python remap.py results/insert_queries/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:LOCALITA census_area_2:POPOLAZIONE_RESIDENTE census_area_3:STRANIERI_RESIDENTI census_area_4:ABITAZIONI census_area_5:FAMIGLIE census_area_6:EDIFICI
#Merge all DIRECTORY_MERGE
python mergeAll.py DIRECTORY_MERGE.sql DB_località.sql drop_all_tables.sql
###############################################
################## SEZIONI ####################
###############################################
./clearResults.sh
python parseCSV.py R04_indicatori_2011_sezioni.csv 6,6,-1 11,7,6 87,7,7 102,7,7 108,7,7 120,7,7 151,7,7
# Merges the files contained in the first subdirectory
python mergeFiles.py results/create_tables/ .sql
# Merges the files contained in the second subdirectory
python mergeFiles.py results/insert_queries/ .sql
#Remove duplicates
python removeDuplicates.py results/insert_queries/DIRECTORY_MERGE.sql
#Remapping table names
python remap.py results/create_tables/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:SEZIONI census_area_2:POPOLAZIONE_RESIDENTE census_area_3:STRANIERI_RESIDENTI census_area_4:ABITAZIONI census_area_5:FAMIGLIE census_area_6:EDIFICI
python remap.py results/insert_queries/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:SEZIONI census_area_2:POPOLAZIONE_RESIDENTE census_area_3:STRANIERI_RESIDENTI census_area_4:ABITAZIONI census_area_5:FAMIGLIE census_area_6:EDIFICI
#Merge all DIRECTORY_MERGE
python mergeAll.py DIRECTORY_MERGE.sql DB_sezioni.sql drop_all_tables.sql
###############################################
############### EXTRA INFO ####################
###############################################
./clearResults.sh
cp Elenco_comuni_2011.csv results/.
python number_format.py results/Elenco_comuni_2011.csv
python parseCSV.py results/Elenco_comuni_2011.csv 24,6,-1
# Merges the files contained in the first subdirectory
python mergeFiles.py results/create_tables/ .sql
# Merges the files contained in the second subdirectory
python mergeFiles.py results/insert_queries/ .sql
#Remove duplicates
python removeDuplicates.py results/insert_queries/DIRECTORY_MERGE.sql
#Remapping table names
python remap.py results/create_tables/DIRECTORY_MERGE.sql census_area_0:LOC_EXTRA "SUPERFICIE (in KMQ):SUPERFICIE_KMQ"
python remap.py results/insert_queries/DIRECTORY_MERGE.sql census_area_0:LOC_EXTRA "SUPERFICIE (in KMQ):SUPERFICIE_KMQ" SUPERFICIEinKMQ:SUPERFICIE_KMQ
#Merge all DIRECTORY_MERGE
python mergeAll.py DIRECTORY_MERGE.sql DB_EXTRA_elenco_comuni.sql
###############################################
################# LOCALITA ####################
###############################################
./clearResults.sh
python parseCSV.py joint_tables/joint_loc_coor.csv 6,6,-1 13,7,6 89,7,7 104,7,7 110,7,7 122,7,7 153,7,7 176,6,6
# Merges the files contained in the first subdirectory
python mergeFiles.py results/create_tables/ .sql
# Merges the files contained in the second subdirectory
python mergeFiles.py results/insert_queries/ .sql
#Remove duplicates
python removeDuplicates.py results/insert_queries/DIRECTORY_MERGE.sql
#Remapping table names
python remap.py results/create_tables/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:LOCALITA census_area_2:POPOLAZIONE_RESIDENTE census_area_3:STRANIERI_RESIDENTI census_area_4:ABITAZIONI census_area_5:FAMIGLIE census_area_6:EDIFICI census_area_7:INFO_LOCALITA
python remap.py results/insert_queries/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:LOCALITA census_area_2:POPOLAZIONE_RESIDENTE census_area_3:STRANIERI_RESIDENTI census_area_4:ABITAZIONI census_area_5:FAMIGLIE census_area_6:EDIFICI census_area_7:INFO_LOCALITA
#Merge all DIRECTORY_MERGE
python mergeAll.py DIRECTORY_MERGE.sql DB_località_final.sql drop_all_tables.sql
###############################################
################## SEZIONI ####################
###############################################
./clearResults.sh
python parseCSV.py joint_tables/joint_sez_coor.csv 6,6,-1 11,7,6 87,7,7 102,7,7 108,7,7 120,7,7 151,7,7 174,7,7
# Merges the files contained in the first subdirectory
python mergeFiles.py results/create_tables/ .sql
# Merges the files contained in the second subdirectory
python mergeFiles.py results/insert_queries/ .sql
#Remove duplicates
python removeDuplicates.py results/insert_queries/DIRECTORY_MERGE.sql
#Remapping table names
python remap.py results/create_tables/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:SEZIONI census_area_2:POPOLAZIONE_RESIDENTE census_area_3:STRANIERI_RESIDENTI census_area_4:ABITAZIONI census_area_5:FAMIGLIE census_area_6:EDIFICI census_area_7:INFO_SEZIONI
python remap.py results/insert_queries/DIRECTORY_MERGE.sql census_area_0:COMUNI census_area_1:SEZIONI census_area_2:POPOLAZIONE_RESIDENTE census_area_3:STRANIERI_RESIDENTI census_area_4:ABITAZIONI census_area_5:FAMIGLIE census_area_6:EDIFICI census_area_7:INFO_SEZIONI
#Merge all DIRECTORY_MERGE
python mergeAll.py DIRECTORY_MERGE.sql DB_sezioni_final.sql drop_all_tables.sql