Enhancements

docker/Dockerfile

@@ -9,15 +9,16 @@
 # Push to docker hub
 # docker login
 # docker tag globalhealthyliveablecities/global-indicators globalhealthyliveablecities/global-indicators:latest
-# docker push globalhealthyliveablecities/global-indicators
+# docker tag globalhealthyliveablecities/global-indicators globalhealthyliveablecities/global-indicators:yyyy-mm-dd
+# docker push --all-tags globalhealthyliveablecities/global-indicators
 #
 # Stop/delete all local docker containers/images:
 # >>> docker stop $(docker ps -aq)
 # >>> docker rm $(docker ps -aq)
 # >>> docker rmi $(docker images -q) --force
 ##############################################################################
 
-FROM --platform=$BUILDPLATFORM continuumio/miniconda3:4.10.3-alpine as build
+FROM continuumio/miniconda3:4.10.3-alpine as build
 LABEL maintainer="Global Healthy Liveable City Indicator Study Collaboration Group"
 LABEL url="https://github.com/global-healthy-liveable-cities/global-indicators"
 

docker/environment.yml

@@ -11,10 +11,12 @@ dependencies:
   - babel
   - cartopy
   - contextily
+  - jupyter
   - geoalchemy2
   - pandana
   - psycopg2
   - pyyaml
   - rasterio
   - urbanaccess
   - tqdm
+  - pygeometa

docker/requirements.txt

@@ -1,25 +1,41 @@
 affine==2.4.0
+anyio==3.6.2
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+asttokens==2.2.1
 attrs==22.2.0
 Babel==2.12.1
+backcall==0.2.0
+backports.functools-lru-cache==1.6.4
+beautifulsoup4==4.12.1
+bleach==6.0.0
 branca==0.6.0
 brotlipy==0.7.0
 Cartopy==0.21.1
 certifi==2022.12.7
 cffi==1.15.1
-charset-normalizer==2.1.1
+charset-normalizer==3.1.0
 click==8.1.3
 click-plugins==1.1.1
 cligj==0.7.2
 colorama==0.4.6
+comm==0.1.3
 contextily==1.3.0
 contourpy==1.0.7
 cryptography==39.0.2
 cycler==0.11.0
+dataclasses==0.8
+debugpy==1.6.6
+decorator==5.1.1
 defusedxml==0.7.1
+entrypoints==0.4
 et-xmlfile==1.1.0
+executing==1.2.0
+fastjsonschema==2.16.3
 Fiona==1.9.2
+flit_core==3.8.0
 folium==0.14.0
-fonttools==4.39.2
+fonttools==4.39.3
 fpdf2==2.6.1
 GDAL==3.6.3
 GeoAlchemy2==0.13.1
@@ -29,55 +45,119 @@ geopy==2.3.0
 greenlet==2.0.2
 idna==3.4
 importlib-metadata==6.1.0
+importlib-resources==5.12.0
+ipykernel==6.22.0
+ipython==8.12.0
+ipython-genutils==0.2.0
+ipywidgets==8.0.6
+jedi==0.18.2
 Jinja2==3.1.2
 joblib==1.2.0
+jsonschema==4.17.3
+jupyter==1.0.0
+jupyter_client==8.1.0
+jupyter-console==6.6.3
+jupyter_core==5.3.0
+jupyter-events==0.6.3
+jupyter_server==2.5.0
+jupyter_server_terminals==0.4.4
+jupyterlab-pygments==0.2.2
+jupyterlab-widgets==3.0.7
 kiwisolver==1.4.4
+lxml==4.9.2
 mapclassify==2.5.0
 MarkupSafe==2.1.2
 matplotlib==3.7.1
+matplotlib-inline==0.1.6
 mercantile==1.2.1
+mistune==2.0.5
 munch==2.5.0
 munkres==1.1.4
-networkx==3.0
-numexpr==2.8.3
+nbclassic==0.5.5
+nbclient==0.7.3
+nbconvert==7.3.0
+nbformat==5.8.0
+nest-asyncio==1.5.6
+networkx==3.1
+notebook==6.5.3
+notebook_shim==0.2.2
+numexpr==2.8.4
 numpy==1.24.2
 openpyxl==3.1.1
 osmnet==0.1.6
 osmnx==1.3.0
+OWSLib==0.28.1
 packaging==23.0
 pandana==0.6.1
-pandas==1.5.3
-Pillow==9.4.0
+pandas==2.0.0
+pandocfilters==1.5.0
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.5.0
 pip==23.0.1
-platformdirs==3.1.1
+pkgutil_resolve_name==1.3.10
+platformdirs==3.2.0
+ply==3.11
 pooch==1.7.0
+prometheus-client==0.16.0
+prompt-toolkit==3.0.38
+psutil==5.9.4
 psycopg2==2.9.3
+ptyprocess==0.7.0
+pure-eval==0.2.2
 pycparser==2.21
-pyOpenSSL==23.0.0
+pygeometa==0.13.1
+Pygments==2.14.0
+pyOpenSSL==23.1.1
 pyparsing==3.0.9
-pyproj==3.4.1
+pyproj==3.5.0
+PyQt5==5.15.7
+PyQt5-sip==12.11.0
+pyrsistent==0.19.3
 pyshp==2.3.1
 PySocks==1.7.1
 python-dateutil==2.8.2
-pytz==2022.7.1
+python-json-logger==2.0.7
+pytz==2023.3
 PyYAML==6.0
+pyzmq==25.0.2
+qtconsole==5.4.2
+QtPy==2.3.1
 rasterio==1.3.6
 requests==2.28.2
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
 Rtree==1.0.1
 scikit-learn==1.2.2
 scipy==1.10.1
-setuptools==67.6.0
+Send2Trash==1.8.0
+setuptools==67.6.1
 shapely==2.0.1
+sip==6.7.7
 six==1.16.0
+sniffio==1.3.0
 snuggs==1.4.7
+soupsieve==2.3.2.post1
 SQLAlchemy==1.4.46
+stack-data==0.6.2
 tables==3.7.0
+terminado==0.17.1
 threadpoolctl==3.1.0
+tinycss2==1.2.1
+toml==0.10.2
+tornado==6.2
 tqdm==4.65.0
+traitlets==5.9.0
 typing_extensions==4.5.0
+tzdata==2023.3
 unicodedata2==15.0.0
 urbanaccess==0.2.2
 urllib3==1.26.15
+wcwidth==0.2.6
+webencodings==0.5.1
+websocket-client==1.5.1
 wheel==0.40.0
+widgetsnbextension==4.0.7
 xyzservices==2023.2.0
 zipp==3.15.0

process/1_create_project_configuration_files.py

@@ -8,6 +8,8 @@
 import shutil
 import sys
 
+from subprocesses._utils import get_terminal_columns, print_autobreak
+
 source_folder = './configuration/templates'
 dest_folder = './configuration'
 region_names = [
@@ -55,7 +57,7 @@
 notes: Notes for this region
 """
 
-print(
+print_autobreak(
     'Creating project configuration files, if not already existing in the configuration folder...',
 )
 try:
@@ -73,17 +75,18 @@
 if len(sys.argv) >= 2:
     codename = sys.argv[1]
     if os.path.exists(f'./configuration/regions/{codename}.yml'):
-        print(
+        print_autobreak(
             f"\nConfiguration file for the specified study region codename '{codename}' already exists: \nconfiguration/regions/{codename}.yml.\n\nPlease open and edit this file in a text editor following the provided example directions in order to complete configuration for your study region.  A completed example study region configuration can be viewed in the file 'configuration/regions/example_ES_Las_Palmas_2023.yml'.\n\nTo view additional guidance on configuration, run this script again without a codename. Once configuration has been completed, to proceed to analysis for this city, enter:\npython 2_analyse_region.py {codename}\n",
         )
     else:
         with open(f'./configuration/regions/{codename}.yml', 'w') as f:
             f.write(region_template)
-        print(
+        print_autobreak(
             f"\nNew region configuration file has been initialised using the codename, '{codename}', in the folder:\nconfiguration/regions/{codename}.yml\n\nPlease open and edit this file in a text editor following the provided example directions in order to complete configuration for your study region.  A completed example study region configuration can be viewed in the file 'configuration/regions/example_ES_Las_Palmas_2023.yml'.\n\nTo view additional guidance on configuration, run this script again without a codename.\n",
         )
 else:
-    print(
+    list_seperation = '\n  '
+    print_autobreak(
         f"""
 Before commencing analysis, your study regions will need to be configured.
 
@@ -102,7 +105,7 @@
 Optional configuration of other parameters is also possible.  Please visit our tool's website for further guidance on how to use this tool:
 https://global-healthy-liveable-cities.github.io/
 
-Currently configured study regions : {', '.join(region_names)}
+Currently configured study regions : {list_seperation}{list_seperation.join(region_names)}
 
 To initialise a new study region configuration file, you can run this script again providing your choice of codename, for example:
 

process/2_analyse_region.py

@@ -25,6 +25,7 @@
     time,
     version,
 )
+from subprocesses._utils import get_terminal_columns, print_autobreak
 from tqdm.auto import tqdm
 
 # Create study region folder if not exists
@@ -55,8 +56,8 @@
         current_parameters['project'] == saved_parameters['project']
         and current_parameters[codename] == saved_parameters[codename]
     ):
-        print(
-            f"The saved copy of region and project parameters from a previous analysis dated {saved_parameters['date'].replace('_',' at ')} at {region_dir}/_parameters_{saved_parameters['date']}.yml matches the current configuration parameters and will be retained.\n\n",
+        print_autobreak(
+            f"The copy of region and project parameters from a previous analysis dated {saved_parameters['date'].replace('_',' at ')} saved in the output directory as _parameters_{saved_parameters['date']}.yml matches the current configuration parameters and will be retained.\n\n",
         )
     else:
         shutil.copyfile(
@@ -72,8 +73,8 @@
                 sort_keys=False,
                 width=float('inf'),
             )
-        print(
-            f"Project or region parameters from a previous analysis dated {saved_parameters['date'].replace('_',' at ')} appear to have been modified. The previous parameter record file has been copied to {region_dir}/_parameters_{saved_parameters['date']}.yml, while the current ones have been saved as {region_dir}/_parameters.yml.\n\n",
+        print_autobreak(
+            f"Project or region parameters from a previous analysis dated {saved_parameters['date'].replace('_',' at ')} appear to have been modified. The previous parameter record file has been copied to the output directory as _parameters_{saved_parameters['date']}.yml, while the current ones have been saved as _parameters.yml.\n",
         )
 else:
     with open(f'{region_dir}/_parameters.yml', 'w') as f:
@@ -85,15 +86,17 @@
             sort_keys=False,
             width=float('inf'),
         )
-    print(
-        f'A dated copy of project and region parameters has been saved as {region_dir}/_parameters.yml.\n\n',
+    print_autobreak(
+        f'A dated copy of project and region parameters has been saved as {region_dir}/_parameters.yml.'.replace(
+            '/home/ghsci/work/', '',
+        ),
     )
 
-print(
-    f'Analysis time zone: {analysis_timezone} (to set time zone for where you are, edit config.yml)',
+print_autobreak(
+    f'\nAnalysis time zone: {analysis_timezone} (to set time zone for where you are, edit config.yml)\n\n',
 )
 start_analysis = time.time()
-print(f"Analysis start: {time.strftime('%Y-%m-%d_%H%M')}")
+print(f"Analysis start:\t{time.strftime('%Y-%m-%d_%H%M')}")
 study_region_setup = {
     '_00_create_database.py': 'Create database',
     '_01_create_study_region.py': 'Create study region',
@@ -106,9 +109,8 @@
     '_08_destination_summary.py': 'Summarise spatial distribution',
     '_09_urban_covariates.py': 'Collate urban covariates',
     '_10_gtfs_analysis.py': 'Analyse GTFS Feeds',
-    '_11_export_gpkg.py': 'Export geopackage',
-    '_12_neighbourhood_analysis.py': 'Analyse neighbourhoods',
-    '_13_aggregation.py': 'Aggregate region summary analyses',
+    '_11_neighbourhood_analysis.py': 'Analyse neighbourhoods',
+    '_12_aggregation.py': 'Aggregate region summary analyses',
 }
 pbar = tqdm(
     study_region_setup,
@@ -130,11 +132,11 @@
             stdout=append_to_log_file,
         )
 except Exception as e:
-    print(
+    print_autobreak(
         f'\n\nProcessing {step} failed: {e}\n\n Please review the processing log file for this study region for more information on what caused this error and how to resolve it. The file __{name}__{codename}_processing_log.txt is located in the output directory and may be opened for viewing in a text editor.',
     )
 finally:
     duration = (time.time() - start_analysis) / 60
     print(
-        f'{time.strftime("%Y-%m-%d_%H%M")} (analysis duration of approximately {duration:.1f} minutes)\n\nOnce the setup of study region resources has been successfully completed, we encourage you to inspect the region-specific resources located in the output directory (e.g. text log file, geopackage output, csv files, PDF report and image files).',
+        f'Analysis end:\t{time.strftime("%Y-%m-%d_%H%M")} (approximately {duration:.1f} minutes)\n',
     )