From ebcb2bf73f732944c0ade594d2e0b0dce92bf307 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 8 Jun 2023 17:16:37 -0500 Subject: [PATCH 01/17] Revert "hotfix/disable questionnaire (#127)" This reverts commit 698da33ccca6327b77162ac77de617d1ffbae397. --- geoapi/services/features.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 25f115ff..46f2820d 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -61,12 +61,10 @@ class FeaturesService: ) ALLOWED_GEOSPATIAL_EXTENSIONS = IMAGE_FILE_EXTENSIONS + GPX_FILE_EXTENSIONS + GEOJSON_FILE_EXTENSIONS\ - + SHAPEFILE_FILE_EXTENSIONS - # RAPP_FILE_EXTENSIONS to be added in https://jira.tacc.utexas.edu/browse/DES-2462 + + SHAPEFILE_FILE_EXTENSIONS + RAPP_FILE_EXTENSIONS ALLOWED_EXTENSIONS = IMAGE_FILE_EXTENSIONS + VIDEO_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS + GPX_FILE_EXTENSIONS\ - + GEOJSON_FILE_EXTENSIONS + SHAPEFILE_FILE_EXTENSIONS + INI_FILE_EXTENSIONS - # RAPP_FILE_EXTENSIONS to be added in https://jira.tacc.utexas.edu/browse/DES-2462 + + GEOJSON_FILE_EXTENSIONS + SHAPEFILE_FILE_EXTENSIONS + INI_FILE_EXTENSIONS + RAPP_FILE_EXTENSIONS @staticmethod def get(featureId: int) -> Feature: @@ -344,7 +342,7 @@ def fromFileObj(projectId: int, fileObj: IO, metadata: Dict, original_path: str return FeaturesService.fromShapefile(projectId, fileObj, {}, additional_files, original_path) elif ext in FeaturesService.INI_FILE_EXTENSIONS: return FeaturesService.fromINI(projectId, fileObj, {}, original_path) - elif False and ext in FeaturesService.RAPP_FILE_EXTENSIONS: # Activate for https://jira.tacc.utexas.edu/browse/DES-2462 + elif ext in FeaturesService.RAPP_FILE_EXTENSIONS: return FeaturesService.fromRAPP(projectId, fileObj, {}, original_path) else: raise ApiException("Filetype not supported for direct upload. Create a feature and attach as an asset?") From 14a93fdebcd0f03e90aa9f93a14e85967cf2588c Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 29 Jun 2023 07:40:49 -0500 Subject: [PATCH 02/17] Add importing of qeustionnaire assets --- geoapi/services/features.py | 21 +- geoapi/tasks/external_data.py | 100 ++-- .../tests/api_tests/test_feature_service.py | 18 +- geoapi/tests/conftest.py | 13 +- .../external_data_tests/test_external_data.py | 36 +- geoapi/tests/fixtures/questionnaire.rq | 215 +++++++- .../questionnaire_with_assets.rq | 56 ++ .../questionnaire_with_assets2.rq | 481 ++++++++++++++++++ 8 files changed, 890 insertions(+), 50 deletions(-) create mode 100644 geoapi/tests/fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq create mode 100644 geoapi/tests/fixtures/questionnaire_with_assets2.rqa/questionnaire_with_assets2.rq diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 46f2820d..4ae6e660 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -239,12 +239,17 @@ def fromShapefile(projectId: int, fileObj: IO, metadata: Dict, additional_files: return features @staticmethod - def fromRAPP(projectId: int, fileObj: IO, metadata: Dict, original_path: str = None) -> Feature: + def from_rapp_questionnaire(projectId: int, fileObj: IO, additional_files: List[IO], original_path: str = None) -> Feature: """ + Import RAPP questionnaire + + RAPP questionnaire is imported along with any asset images that it + refers to. The asset images are assumed to reside in the same directory + as the questionnaire .rq file. :param projectId: int - :param fileObj: file descriptor - :param metadata: Dict of pairs + :param fileObj: file + :param additional_files: list of file objs :param original_path: str path of original file location :return: Feature """ @@ -263,9 +268,17 @@ def fromRAPP(projectId: int, fileObj: IO, metadata: Dict, original_path: str = N pathlib.Path(questionnaire_path).mkdir(parents=True, exist_ok=True) asset_path = os.path.join(questionnaire_path, 'questionnaire.rq') + # write questionnaire file (rq) with open(asset_path, 'w') as tmp: tmp.write(json.dumps(data)) + # write all asset files (i.e jpgs) + if additional_files is not None: + for file_obj in additional_files: + image_asset_path = os.path.join(questionnaire_path, pathlib.Path(file_obj.name).name) + with open(image_asset_path, 'wb') as image_asset: + image_asset.write(file_obj.read()) + fa = FeatureAsset( uuid=asset_uuid, asset_type="questionnaire", @@ -343,7 +356,7 @@ def fromFileObj(projectId: int, fileObj: IO, metadata: Dict, original_path: str elif ext in FeaturesService.INI_FILE_EXTENSIONS: return FeaturesService.fromINI(projectId, fileObj, {}, original_path) elif ext in FeaturesService.RAPP_FILE_EXTENSIONS: - return FeaturesService.fromRAPP(projectId, fileObj, {}, original_path) + return FeaturesService.from_rapp_questionnaire(projectId, fileObj, additional_files, original_path) else: raise ApiException("Filetype not supported for direct upload. Create a feature and attach as an asset?") diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index ae2e867d..c2373dda 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -22,6 +22,7 @@ from geoapi.db import db_session from geoapi.services.notifications import NotificationsService from geoapi.services.users import UserService +from dataclasses import dataclass class ImportState(Enum): @@ -30,6 +31,13 @@ class ImportState(Enum): RETRYABLE_FAILURE = 3 +@dataclass +class AdditionalFile: + """Represents an additional file with its path and and if its required (i.e. not optional).""" + path: str + required: bool + + def _parse_rapid_geolocation(loc): coords = loc[0] lat = coords["latitude"] @@ -58,47 +66,73 @@ def get_file(client, system_id, path, required): return system_id, path, required, result_file, error -def get_additional_files(systemId: str, path: str, client, available_files=None): + + +def get_additional_files(current_file, system_id: str, path: str, client, available_files=None): """ - Get any additional files needed for processing - :param systemId: str - :param path: str - :param client + Get any additional files needed for processing the current file being imported + + Note `available_files` is optional. if provided, then it can be used to fail early if it is known + that a required file is missing + + :param str current_file: active file that is being imported + :param str system_id: system of active file + :param path: path of active file + :param client: :param available_files: list of files that exist (optional) :return: list of additional files """ - path = Path(path) - if path.suffix.lower().lstrip('.') == "shp": - paths_to_get = [] + additional_files_to_get = [] + + current_file_path = Path(path) + file_suffix = current_file_path.suffix.lower().lstrip('.') + if file_suffix == "shp": + logger.info(f'Determining which shapefile-related files need to be downloaded for file {current_file.filename}') for extension, required in SHAPEFILE_FILE_ADDITIONAL_FILES.items(): - additional_file_path = path.with_suffix(extension) + additional_file_path = current_file_path.with_suffix(extension) if available_files and str(additional_file_path) not in available_files: if required: - logger.error("Could not import required shapefile-related file: " - "agave: {} :: {}".format(systemId, additional_file_path)) - raise Exception("Required file ({}) missing".format(additional_file_path)) + logger.error(f"Could not import required shapefile-related file: agave: {system_id}/{additional_file_path}") + raise Exception(f'Required file ({system_id}/{additional_file_path}) missing') else: continue - paths_to_get.append(additional_file_path) - - additional_files = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - getting_files_futures = [executor.submit(get_file, client, systemId, additional_file_path, required) - for additional_file_path in paths_to_get] - for future in concurrent.futures.as_completed(getting_files_futures): - _, additional_file_path, required, result_file, error = future.result() - if not result_file and required: - logger.error("Could not import a required shapefile-related file: " - "agave: {} :: {} ---- error: {}".format(systemId, additional_file_path, error)) - if not result_file: - logger.debug("Unable to get non-required shapefile-related file: " - "agave: {} :: {}".format(systemId, additional_file_path)) - continue - result_file.filename = Path(additional_file_path).name - additional_files.append(result_file) + additional_files_to_get.append(AdditionalFile(path=additional_file_path, required=required)) + elif file_suffix == "rq": + logger.info(f'Parsing rq file {current_file.filename} to see what assets need to be downloaded ') + data = json.load(current_file) + for section in data["sections"]: + for question in section["questions"]: + for asset in question.get("assets", []): + # determine full path for this asset and add to list + additional_file_path = current_file_path.with_name(asset["filename"]) + additional_files_to_get.append(AdditionalFile(path=additional_file_path, required=True)) + logger.info(f'{len(additional_files_to_get)} assets were found for rq file {current_file.filename}') + + # Seek back to start of file + current_file.seek(0) else: - additional_files = None - return additional_files + return None + + + # Try to get all additional files. + additional_files_result = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + getting_files_futures = [executor.submit(get_file, client, system_id, additional_file.path, additional_file.required) + for additional_file in additional_files_to_get] + for future in concurrent.futures.as_completed(getting_files_futures): + _, additional_file_path, required, result_file, error = future.result() + if not result_file and required: + logger.error(f'Could not import a required {file_suffix}-related file: ' + f'agave: {system_id} :: {additional_file_path} ---- error: {error}') + raise Exception(f'Required file ({system_id}/{additional_file_path}) missing') + if not result_file: + logger.error(f'Unable to get non-required {file_suffix}-related file: ' + f'agave: {system_id} :: {additional_file_path} ---- error: {error}') + + continue + result_file.filename = Path(additional_file_path).name + additional_files_result.append(result_file) + return additional_files_result @app.task(rate_limit="10/s") @@ -113,7 +147,7 @@ def import_file_from_agave(userId: int, systemId: str, path: str, projectId: int try: tmpFile = client.getFile(systemId, path) tmpFile.filename = Path(path).name - additional_files = get_additional_files(systemId, path, client) + additional_files = get_additional_files(tmpFile, systemId, path, client) FeaturesService.fromFileObj(projectId, tmpFile, {}, original_path=path, additional_files=additional_files) NotificationsService.create(user, "success", "Imported {f}".format(f=path)) tmpFile.close() @@ -315,7 +349,7 @@ def import_from_agave(tenant_id: str, userId: int, systemId: str, path: str, pro logger.info("importing:{} for user:{}".format(item_system_path, user.username)) tmpFile = client.getFile(systemId, item.path) tmpFile.filename = Path(item.path).name - additional_files = get_additional_files(systemId, item.path, client, filenames_in_directory) + additional_files = get_additional_files(tmpFile, systemId, item.path, client, available_files=filenames_in_directory) FeaturesService.fromFileObj(projectId, tmpFile, {}, original_path=item_system_path, additional_files=additional_files) NotificationsService.create(user, "success", "Imported {f}".format(f=item_system_path)) diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index 9c6409f1..6d08e2ea 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -208,10 +208,24 @@ def test_create_tile_server_from_file(projects_fixture, tile_server_ini_file_fix def test_create_questionnaire_feature(projects_fixture, questionnaire_file_fixture): - feature = FeaturesService.fromRAPP(projects_fixture.id, questionnaire_file_fixture, metadata={}) + feature = FeaturesService.from_rapp_questionnaire(projects_fixture.id, questionnaire_file_fixture, additional_files=None) assert feature.project_id == projects_fixture.id assert len(feature.assets) == 1 assert db_session.query(Feature).count() == 1 assert db_session.query(FeatureAsset).count() == 1 assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 1 - assert os.path.isfile(os.path.join(get_project_asset_dir(projects_fixture.id), str(feature.assets[0].uuid) + "/questionnaire.rq")) + assert len(os.listdir(get_asset_path(feature.assets[0].path))) == 1 + assert os.path.isfile(get_asset_path(feature.assets[0].path, "questionnaire.rq")) + + +def test_create_questionnaire_feature_with_assets(projects_fixture, questionnaire_file_with_assets_fixture, image_file_fixture): + assets = [image_file_fixture] + feature = FeaturesService.from_rapp_questionnaire(projects_fixture.id, questionnaire_file_with_assets_fixture, additional_files=assets) + assert feature.project_id == projects_fixture.id + assert len(feature.assets) == 1 + assert db_session.query(Feature).count() == 1 + assert db_session.query(FeatureAsset).count() == 1 + assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 1 + assert len(os.listdir(get_asset_path(feature.assets[0].path))) == 2 + assert os.path.isfile(get_asset_path(feature.assets[0].path, "questionnaire.rq")) + assert os.path.isfile(get_asset_path(feature.assets[0].path, "image.jpg")) diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index db296ff5..f7c93da5 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -162,6 +162,7 @@ def gpx_file_fixture(): def image_file_fixture(): home = os.path.dirname(__file__) with open(os.path.join(home, 'fixtures/image.jpg'), 'rb') as f: + f.filename = 'image.jpg' yield f @@ -480,5 +481,15 @@ def tile_server_ini_file_fixture(): @pytest.fixture(scope="function") def questionnaire_file_fixture(): home = os.path.dirname(__file__) - with open(os.path.join(home, 'fixtures/questionnaire.rq'), 'rb') as f: + filename = 'fixtures/questionnaire.rq' + with open(os.path.join(home, filename), 'rb') as f: + f.filename = filename + yield f + +@pytest.fixture(scope="function") +def questionnaire_file_with_assets_fixture(): + home = os.path.dirname(__file__) + filename = 'fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq' + with open(os.path.join(home, filename ), 'rb') as f: + f.filename = filename yield f diff --git a/geoapi/tests/external_data_tests/test_external_data.py b/geoapi/tests/external_data_tests/test_external_data.py index 05eff738..560e296e 100644 --- a/geoapi/tests/external_data_tests/test_external_data.py +++ b/geoapi/tests/external_data_tests/test_external_data.py @@ -434,16 +434,16 @@ def test_is_member_of_rapp_project_folder(): assert not is_member_of_rapp_project_folder("/something/test.jpg") -def test_get_additional_files_none(agave_utils_with_geojson_file): - assert not get_additional_files("testSystem", "/testPath/file.jpg", agave_utils_with_geojson_file) +def test_get_additional_files_none(shapefile_fixture, agave_utils_with_geojson_file): + assert not get_additional_files(shapefile_fixture, "testSystem", "/testPath/file.jpg", agave_utils_with_geojson_file) -def test_get_additional_files(agave_utils_with_geojson_file): - files = get_additional_files("testSystem", "/testPath/file.shp", agave_utils_with_geojson_file) +def test_get_additional_shapefiles_files(shapefile_fixture, agave_utils_with_geojson_file): + files = get_additional_files(shapefile_fixture, "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file) assert len(files) == 14 -def test_get_additional_files_with_available_files(agave_utils_with_geojson_file): +def test_get_additional_files_shapefiles_with_available_files(shapefile_fixture, agave_utils_with_geojson_file): available_files = ["/testPath/file.shx", "/testPath/file.dbf", "/testPath/file.sbn", @@ -458,7 +458,8 @@ def test_get_additional_files_with_available_files(agave_utils_with_geojson_file "/testPath/file.prj", "/testPath/file.xml", "/testPath/file.cpg"] - files = get_additional_files("testSystem", + files = get_additional_files(shapefile_fixture, + "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file, available_files=available_files) @@ -467,17 +468,34 @@ def test_get_additional_files_with_available_files(agave_utils_with_geojson_file available_files = ["/testPath/file.shx", "/testPath/file.dbf", "/testPath/file.prj"] - files = get_additional_files("testSystem", + files = get_additional_files(shapefile_fixture, + "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file, available_files=available_files) assert len(files) == 3 -def test_get_additional_files_but_missing_prj(agave_utils_with_geojson_file): +def test_get_additional_files_shapefiles_but_missing_prj(shapefile_fixture, agave_utils_with_geojson_file): available_files_missing_prj = ["/testPath/file.shx", "/testPath/file.dbf"] with pytest.raises(Exception): - get_additional_files("testSystem", + get_additional_files(shapefile_fixture, + "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file, available_files=available_files_missing_prj) + + +def test_get_additional_files_rapid_questionnaire(questionnaire_file_with_assets_fixture, agave_utils_with_geojson_file): + files = get_additional_files(questionnaire_file_with_assets_fixture, + "testSystem", + questionnaire_file_with_assets_fixture.filename, + agave_utils_with_geojson_file) + assert len(files) == 1 + +def test_get_additional_files_rapid_questionnaire_no_assets(questionnaire_file_fixture, agave_utils_with_geojson_file): + files = get_additional_files(questionnaire_file_fixture, + "testSystem", + questionnaire_file_fixture.filename, + agave_utils_with_geojson_file) + assert files == [] diff --git a/geoapi/tests/fixtures/questionnaire.rq b/geoapi/tests/fixtures/questionnaire.rq index 76137a0f..48172d85 100644 --- a/geoapi/tests/fixtures/questionnaire.rq +++ b/geoapi/tests/fixtures/questionnaire.rq @@ -1 +1,214 @@ -{"id":2171,"description":"Yellowstone Test","geolocation":[{"longitude":-122.30502990145563,"timestamp":1658877846.8723259,"latitude":47.652537730540047,"course":-1,"heading":24.99053955078125,"altitude":38.239620208740234}],"uuid":"8288DDDC-E50B-4A72-980E-84D7BD066BA9","version":"8288DDDC-E50B-4A72-980E-84D7BD066BA9","access":"private","owner":"elliot_n","end_uuid":"B20EBBAE-E3E2-42AB-88EA-7F9068BD4502","editable":false,"allow_back":true,"questionUuidsVisited":[],"sections":[{"id":"AA901679-2D1F-47BD-A67B-ABA5FDB0B23C","label":"Researcher Questions","questions":[{"id":"B5B0D1D8-7555-4414-A9FE-C3389AFD000F","heading":"Q1","label":"Are you wearing proper PPE","options":[{"go_to":null,"default":false,"label":"Yes","sub_question":null,"value":"yes"},{"go_to":null,"default":false,"label":"No","sub_question":null,"value":"no"}],"responseIndexes":[0],"decline":null,"type":"Yes \/ No","value":"question","mode":"list","assetUuids":[],"instructions":"Select one","required":true},{"id":"06A57174-5336-4BC4-BF87-BD5E9C6AF9B4","heading":"Is your subject wearing proper PPE?","label":"Enter your question text","options":[{"go_to":null,"default":false,"label":"Yes","sub_question":null,"value":"yes"},{"go_to":null,"default":false,"label":"No","sub_question":null,"value":"no"}],"responseIndexes":[0],"decline":null,"type":"Yes \/ No","value":"enter_your_question_text","mode":"list","assetUuids":[],"instructions":"Select one:","required":true},{"id":"2DAC188A-BBD0-42FF-B0B9-0C88498B2D36","heading":"Q3","label":"Please select one of the following:","options":[{"go_to":null,"default":false,"label":"Uninjured, not displaced","sub_question":null,"value":"uninjured_not_displaced"},{"go_to":null,"default":false,"label":"Minor Injuries, not displaced","sub_question":null,"value":"minor_injuries_not_displaced"},{"go_to":null,"default":false,"label":"Major injuries, not displaced","sub_question":null,"value":"major_injuries_not_displaced"},{"go_to":null,"default":false,"label":"Uninjured, displaced","sub_question":null,"value":"uninjured_displaced"},{"go_to":null,"default":false,"label":"Minor Injuries, displaced","sub_question":null,"value":"minor_injuries_displaced"},{"go_to":null,"default":false,"label":"Major injuries, displaced","sub_question":null,"value":"major_injuries_displaced"}],"responseIndexes":[5],"decline":null,"type":"Single Select","value":"please_select_one_of_the_follo","mode":"list","assetUuids":[],"instructions":"What is the status of your subject?","required":true}]},{"id":"EFFDA218-C78B-4EED-B107-7EB1A96F9C93","label":"Resident Questions","questions":[{"value":"primary_residence","responseIndexes":[0],"id":"2C63F564-BC9F-4BDA-B511-68470AF464B1","instructions":"Where were you during the flooding event?","decline":null,"assetUuids":[],"label":"Please select one or more of the following:","type":"Multi Select","required":true,"heading":"Q1","options":[{"go_to":null,"default":false,"label":"Primary Residence","sub_question":null,"value":"primary_residence"},{"go_to":null,"default":false,"label":"Secondary Residence","sub_question":null,"value":"secondary_residence"},{"go_to":null,"default":false,"label":"In the area, but not at primary\/secondary residence","sub_question":null,"value":"in_the_area_but_not_at_primary"},{"go_to":null,"default":false,"label":"Out of town","sub_question":null,"value":"out_of_town"},{"go_to":null,"default":false,"label":"Other","sub_question":null,"value":"other"}]}]}],"subversion":"1","is_invalid":false,"display_mode":"multi","end_text":"End","name":"EN_Test","errors":[]} +{ + "id": 2171, + "description": "Yellowstone Test", + "geolocation": [ + { + "longitude": -122.30502990145563, + "timestamp": 1658877846.8723259, + "latitude": 47.652537730540047, + "course": -1, + "heading": 24.99053955078125, + "altitude": 38.239620208740234 + } + ], + "uuid": "8288DDDC-E50B-4A72-980E-84D7BD066BA9", + "version": "8288DDDC-E50B-4A72-980E-84D7BD066BA9", + "access": "private", + "owner": "elliot_n", + "end_uuid": "B20EBBAE-E3E2-42AB-88EA-7F9068BD4502", + "editable": false, + "allow_back": true, + "questionUuidsVisited": [], + "sections": [ + { + "id": "AA901679-2D1F-47BD-A67B-ABA5FDB0B23C", + "label": "Researcher Questions", + "questions": [ + { + "id": "B5B0D1D8-7555-4414-A9FE-C3389AFD000F", + "heading": "Q1", + "label": "Are you wearing proper PPE", + "options": [ + { + "go_to": null, + "default": false, + "label": "Yes", + "sub_question": null, + "value": "yes" + }, + { + "go_to": null, + "default": false, + "label": "No", + "sub_question": null, + "value": "no" + } + ], + "responseIndexes": [ + 0 + ], + "decline": null, + "type": "Yes \/ No", + "value": "question", + "mode": "list", + "assetUuids": [], + "instructions": "Select one", + "required": true + }, + { + "id": "06A57174-5336-4BC4-BF87-BD5E9C6AF9B4", + "heading": "Is your subject wearing proper PPE?", + "label": "Enter your question text", + "options": [ + { + "go_to": null, + "default": false, + "label": "Yes", + "sub_question": null, + "value": "yes" + }, + { + "go_to": null, + "default": false, + "label": "No", + "sub_question": null, + "value": "no" + } + ], + "responseIndexes": [ + 0 + ], + "decline": null, + "type": "Yes \/ No", + "value": "enter_your_question_text", + "mode": "list", + "assetUuids": [], + "instructions": "Select one:", + "required": true + }, + { + "id": "2DAC188A-BBD0-42FF-B0B9-0C88498B2D36", + "heading": "Q3", + "label": "Please select one of the following:", + "options": [ + { + "go_to": null, + "default": false, + "label": "Uninjured, not displaced", + "sub_question": null, + "value": "uninjured_not_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Minor Injuries, not displaced", + "sub_question": null, + "value": "minor_injuries_not_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Major injuries, not displaced", + "sub_question": null, + "value": "major_injuries_not_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Uninjured, displaced", + "sub_question": null, + "value": "uninjured_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Minor Injuries, displaced", + "sub_question": null, + "value": "minor_injuries_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Major injuries, displaced", + "sub_question": null, + "value": "major_injuries_displaced" + } + ], + "responseIndexes": [ + 5 + ], + "decline": null, + "type": "Single Select", + "value": "please_select_one_of_the_follo", + "mode": "list", + "assetUuids": [], + "instructions": "What is the status of your subject?", + "required": true + } + ] + }, + { + "id": "EFFDA218-C78B-4EED-B107-7EB1A96F9C93", + "label": "Resident Questions", + "questions": [ + { + "value": "primary_residence", + "responseIndexes": [ + 0 + ], + "id": "2C63F564-BC9F-4BDA-B511-68470AF464B1", + "instructions": "Where were you during the flooding event?", + "decline": null, + "assetUuids": [], + "label": "Please select one or more of the following:", + "type": "Multi Select", + "required": true, + "heading": "Q1", + "options": [ + { + "go_to": null, + "default": false, + "label": "Primary Residence", + "sub_question": null, + "value": "primary_residence" + }, + { + "go_to": null, + "default": false, + "label": "Secondary Residence", + "sub_question": null, + "value": "secondary_residence" + }, + { + "go_to": null, + "default": false, + "label": "In the area, but not at primary\/secondary residence", + "sub_question": null, + "value": "in_the_area_but_not_at_primary" + }, + { + "go_to": null, + "default": false, + "label": "Out of town", + "sub_question": null, + "value": "out_of_town" + }, + { + "go_to": null, + "default": false, + "label": "Other", + "sub_question": null, + "value": "other" + } + ] + } + ] + } + ], + "subversion": "1", + "is_invalid": false, + "display_mode": "multi", + "end_text": "End", + "name": "EN_Test", + "errors": [] +} diff --git a/geoapi/tests/fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq b/geoapi/tests/fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq new file mode 100644 index 00000000..85d16cac --- /dev/null +++ b/geoapi/tests/fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq @@ -0,0 +1,56 @@ +{ + "editable": false, + "uuid": "DE99A39A-B352-4FF6-BA6B-C325BB25CDFF", + "access": "private", + "subversion": "1", + "allow_back": true, + "errors": [], + "geolocation": [ + { + "altitude": 44.7580108642578, + "latitude": 12.3456789, + "course": -1, + "timestamp": 1680540248.33747, + "longitude": -123.456789 + } + ], + "version": "DE99A39A-B352-4FF6-BA6B-C325BB25CDFF", + "end_text": "End", + "description": "Questionnaire with only one question", + "sections": [ + { + "id": "D1893426-9206-4678-9589-B81B9E7042AB", + "label": "", + "questions": [ + { + "decline": null, + "id": "7389C442-05C0-46DA-AB1E-C9AF29A29A0C", + "required": true, + "responseStrings": [ + "5" + ], + "type": "Number", + "mode": "integer", + "heading": null, + "label": "Please enter an integer", + "assets": [ + { + "rappUuid": "9378FA7A-8BDD-4947-A3EC-2D66B938C59F", + "filename": "Q1-Photo-001.jpg" + } + ], + "instructions": null, + "value": "please_enter_an_integer" + } + ] + } + ], + "owner": "adioso", + "questionUuidsVisited": [], + "asset_embedding": true, + "is_invalid": false, + "id": 1918, + "display_mode": "single", + "end_uuid": "C551C60C-AA8D-4BBD-BCE4-87A1A11120E7", + "name": "Single Question" +} \ No newline at end of file diff --git a/geoapi/tests/fixtures/questionnaire_with_assets2.rqa/questionnaire_with_assets2.rq b/geoapi/tests/fixtures/questionnaire_with_assets2.rqa/questionnaire_with_assets2.rq new file mode 100644 index 00000000..baec7257 --- /dev/null +++ b/geoapi/tests/fixtures/questionnaire_with_assets2.rqa/questionnaire_with_assets2.rq @@ -0,0 +1,481 @@ +{ + "id": 2324, + "description": "Example made by Nathan", + "geolocation": [ + { + "timestamp": 1683578336.2153249, + "latitude": 30.319060724035527, + "course": -1, + "longitude": -97.720111181677041, + "altitude": 208.61720085144043 + } + ], + "uuid": "1E0DEBD1-601A-44EB-A40F-AC00EDBF11F2", + "version": "1E0DEBD1-601A-44EB-A40F-AC00EDBF11F2", + "access": "public", + "owner": "nathanf", + "allow_back": true, + "editable": false, + "end_uuid": "D45B5925-4FB5-4C55-BF64-47918E2C0513", + "questionUuidsVisited": [], + "sections": [ + { + "id": "EC16B5FE-8933-4786-BFEC-E7D8D24A774B", + "label": "", + "questions": [ + { + "id": "EBBC076B-20FD-48CB-A9FD-62B5C57C278A", + "heading": "q1 on first section", + "assets": [ + { + "rappUuid": "F1668F90-95D3-4A29-BD1C-A1956BD24783", + "filename": "Q1-Photo-001.jpg" + } + ], + "label": "Please select one of the following:", + "options": [ + { + "go_to": null, + "default": false, + "label": "Response 1", + "sub_question": null, + "value": "response_1" + }, + { + "go_to": null, + "default": false, + "label": "Response 2", + "sub_question": null, + "value": "response_2" + }, + { + "go_to": null, + "default": false, + "label": "Response 3", + "sub_question": null, + "value": "response_3" + }, + { + "go_to": null, + "default": false, + "label": "Response 4", + "sub_question": null, + "value": "response_4" + } + ], + "responseIndexes": [ + 1 + ], + "decline": null, + "type": "Single Select", + "value": "please_select_one_of_the_follo", + "mode": "list", + "instructions": "test", + "required": true + } + ] + }, + { + "id": "7DC933D8-EB1C-4041-9089-CAD3F57147D2", + "label": "section1", + "questions": [ + { + "id": "1B5C44E7-81DC-413F-92E8-8E56D2517B06", + "heading": "heading for yes and no question", + "assets": [ + { + "rappUuid": "2D679547-FDEF-40AE-8219-740F3895A6D7", + "filename": "Q2-Photo-001.jpg" + }, + { + "rappUuid": "498C45B7-62A3-4735-80EC-5F36A3C5660E", + "filename": "Q2-Photo-002.jpg" + } + ], + "label": "A yes or no question", + "options": [ + { + "go_to": null, + "default": false, + "label": "Yes", + "sub_question": null, + "value": "yes" + }, + { + "go_to": null, + "default": false, + "label": "No", + "sub_question": null, + "value": "no" + } + ], + "responseIndexes": [ + 1 + ], + "decline": null, + "type": "Yes \/ No", + "value": "question", + "mode": "list", + "instructions": "This is a a yes or no question", + "required": true + }, + { + "id": "73472BD4-41E5-4554-8F7D-382E0F230483", + "heading": "heading single select q2 question", + "assets": [ + { + "rappUuid": "9355B455-B678-4A42-9739-6E5643D2C1B8", + "filename": "Q3-Photo-001.jpg" + }, + { + "rappUuid": "DF7C3A81-B282-433A-836A-ED7BDDB1C02E", + "filename": "Q3-Photo-002.jpg" + } + ], + "label": "Please select one of the following:", + "options": [ + { + "go_to": null, + "default": false, + "label": "Response 1", + "sub_question": null, + "value": "response_1" + }, + { + "go_to": null, + "default": false, + "label": "Response 2", + "sub_question": null, + "value": "response_2" + }, + { + "go_to": null, + "default": false, + "label": "Response 3", + "sub_question": null, + "value": "response_3" + }, + { + "go_to": null, + "default": false, + "label": "Response 4", + "sub_question": null, + "value": "response_4" + } + ], + "responseIndexes": [ + 1 + ], + "decline": null, + "type": "Single Select", + "value": "please_select_one_of_the_follo", + "mode": "list", + "instructions": "instructions for q2", + "required": true + }, + { + "value": "please_select_one_or_more_of_t", + "responseIndexes": [ + 1 + ], + "assets": [], + "id": "2CC0FDF0-2279-476A-A547-360738AACF95", + "instructions": "q2 instructions", + "decline": null, + "label": "Please select one or more of the following:", + "type": "Multi Select", + "required": true, + "heading": "q3 haeading", + "options": [ + { + "go_to": null, + "default": false, + "label": "Response 1a", + "sub_question": null, + "value": "response_1a" + }, + { + "go_to": null, + "default": false, + "label": "Response 2b", + "sub_question": null, + "value": "response_2b" + }, + { + "go_to": null, + "default": false, + "label": "Response 3", + "sub_question": null, + "value": "response_3" + }, + { + "go_to": null, + "default": false, + "label": "Response 4b sdfsdf", + "sub_question": null, + "value": "response_4b_sdfsdf" + } + ] + }, + { + "assets": [], + "responseStrings": null, + "id": "A73A00D1-3BF5-46B9-B2B1-93CAC32BFC2A", + "instructions": null, + "decline": null, + "value": "", + "type": "Multi Text", + "required": true, + "heading": "a text field heading", + "questions": [ + { + "assets": [], + "responseStrings": [ + "Test" + ], + "id": "A6352257-EE96-4389-8689-F0CA40E40BAA", + "instructions": "test", + "decline": null, + "mode": "short", + "label": "Response 1", + "type": "Text Field", + "required": true, + "heading": null, + "value": "response_1" + } + ] + }, + { + "assets": [], + "responseStrings": [ + "May 8, 2025" + ], + "id": "0FED5898-D3C9-4466-8957-31EA3C2D4D3D", + "instructions": "date", + "decline": null, + "mode": "date", + "label": "Please enter a date", + "type": "DateTime", + "required": true, + "heading": "date time", + "value": "please_enter_an_date" + }, + { + "assets": [], + "responseStrings": [ + "4" + ], + "id": "A5100404-26BA-4B0D-AEBF-C2D0DC6C7913", + "instructions": null, + "decline": null, + "mode": "integer", + "label": "Please enter an integer", + "type": "Number", + "required": true, + "heading": "q6 number", + "value": "please_enter_an_integer" + }, + { + "id": "BAB27ADF-1EEF-4837-BF5F-957354B6E348", + "heading": "q7 a range", + "assets": [], + "label": "Please rate the following from 1 (the least likely) to 10 (the most likely):", + "options": [ + { + "go_to": null, + "default": false, + "label": "1", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "2", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "3", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "4", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "5", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "6", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "7", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "8", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "9", + "sub_question": null, + "value": null + }, + { + "go_to": null, + "default": false, + "label": "10", + "sub_question": null, + "value": null + } + ], + "responseIndexes": [ + 1 + ], + "decline": null, + "type": "Range", + "value": "please_rate_the_following_from", + "instructions": null, + "range": { + "min": 1, + "max": 10 + }, + "required": true + }, + { + "id": "A26E27C8-D5D2-4C06-ACDC-853CC8D1A233", + "heading": "a matrix", + "label": "Please select one or more of the following:", + "assets": [], + "decline": null, + "responseMatrixIndexes": [ + [ + 0 + ], + [ + 1 + ] + ], + "type": "Matrix", + "value": "", + "columns": [ + { + "label": "Column 1", + "value": "column_1" + }, + { + "label": "Column 2", + "value": "column_2" + } + ], + "mode": "single", + "rows": [ + { + "label": "Row 1", + "value": "row_1" + }, + { + "label": "Row 2", + "value": "row_1" + } + ], + "instructions": null, + "required": false + }, + { + "assets": [], + "responseStrings": [ + "30.319140688537654", + "-97.72014149862869", + "5405 Avenue F\nAustin, TX 78751\nUS" + ], + "id": "BF57DE13-333E-41C7-A2C2-6754BAC97409", + "instructions": null, + "decline": null, + "label": "Please indicate your location", + "type": "Location", + "heading": "q9: a location heading", + "required": true, + "value": "please_indicate_your_location" + }, + { + "assets": [ + { + "rappUuid": "F4AB2BEE-CFF3-4AC5-B28B-054730E15777", + "filename": "Q11-Photo-001.jpg" + } + ], + "responseStrings": null, + "id": "8981DD40-A560-42CD-96F8-8F1982357F7B", + "instructions": null, + "decline": null, + "value": "", + "type": "Text", + "heading": "p2 heading", + "required": false + }, + { + "id": "E2B43583-EEA9-43A8-B247-F4004F989868", + "heading": "q11", + "assets": [], + "label": "Enter your question sdfdsfds", + "options": [ + { + "go_to": null, + "default": false, + "label": "Yes", + "sub_question": null, + "value": "yes" + }, + { + "go_to": null, + "default": false, + "label": "No", + "sub_question": null, + "value": "no" + } + ], + "responseIndexes": [ + 0 + ], + "decline": null, + "type": "Yes \/ No", + "value": "question", + "mode": "list", + "instructions": null, + "required": true + } + ] + } + ], + "asset_embedding": true, + "subversion": "1", + "is_invalid": false, + "display_mode": "multi", + "end_text": "End", + "name": "questionnaire_nmf_example1", + "errors": [] +} From 74e2aedf2f9a93d0006750be3accd2bbca85d9ec Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 29 Jun 2023 13:20:53 -0500 Subject: [PATCH 03/17] Improve importing of assets Ensure file name is correct and a preview image is created. Also, store info on assets' geolocation --- geoapi/services/features.py | 31 ++++++++++++++++--- geoapi/tasks/external_data.py | 1 + .../tests/api_tests/test_feature_service.py | 3 +- geoapi/utils/agave.py | 1 + 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 4ae6e660..03cabed1 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -248,11 +248,12 @@ def from_rapp_questionnaire(projectId: int, fileObj: IO, additional_files: List[ as the questionnaire .rq file. :param projectId: int - :param fileObj: file + :param fileObj: questionnaire file :param additional_files: list of file objs :param original_path: str path of original file location :return: Feature """ + logger.info(f"Processing f{original_path}") data = json.loads(fileObj.read()) lng = data.get('geolocation')[0].get('longitude') @@ -272,12 +273,34 @@ def from_rapp_questionnaire(projectId: int, fileObj: IO, additional_files: List[ with open(asset_path, 'w') as tmp: tmp.write(json.dumps(data)) + additional_files_properties = [] + # write all asset files (i.e jpgs) if additional_files is not None: - for file_obj in additional_files: - image_asset_path = os.path.join(questionnaire_path, pathlib.Path(file_obj.name).name) + logger.info(f"Processing {len(additional_files)} assets for {original_path}") + for asset_file_obj in additional_files: + base_filename = os.path.basename(asset_file_obj.filename) + image_asset_path = os.path.join(questionnaire_path, base_filename) + + # save original jpg (i.e. Q1-Photo-001.jpg) with open(image_asset_path, 'wb') as image_asset: - image_asset.write(file_obj.read()) + image_asset.write(asset_file_obj.read()) + + # create preview image (i.e. Q1-Photo-001.preview.jpg) + processed_asset_image = ImageService.processImage(asset_file_obj) + path = pathlib.Path(image_asset_path) + processed_asset_image.resized.save(path.with_suffix('.preview' + path.suffix), "JPEG") + + # gather coordinates information for this asset + logger.debug(f"{asset_file_obj.filename} has the geospatial coordinates of {processed_asset_image.coordinates}") + additional_files_properties.append({"filename": base_filename, + "coordinates": processed_asset_image.coordinates}) + asset_file_obj.close() + + + if additional_files_properties: + # add info about assets to properties (i.e. coordinates of asset) for quick retrieval + feat.properties = {"_hazmapper": {"questionnaire": {"assets": additional_files_properties}}} fa = FeatureAsset( uuid=asset_uuid, diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index c2373dda..2b19287d 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -130,6 +130,7 @@ def get_additional_files(current_file, system_id: str, path: str, client, availa f'agave: {system_id} :: {additional_file_path} ---- error: {error}') continue + logger.debug(f'Finished getting {file_suffix}-related file: ({system_id}/{additional_file_path}') result_file.filename = Path(additional_file_path).name additional_files_result.append(result_file) return additional_files_result diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index 6d08e2ea..f349f3ce 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -226,6 +226,7 @@ def test_create_questionnaire_feature_with_assets(projects_fixture, questionnair assert db_session.query(Feature).count() == 1 assert db_session.query(FeatureAsset).count() == 1 assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 1 - assert len(os.listdir(get_asset_path(feature.assets[0].path))) == 2 + assert len(os.listdir(get_asset_path(feature.assets[0].path))) == 3 assert os.path.isfile(get_asset_path(feature.assets[0].path, "questionnaire.rq")) + assert os.path.isfile(get_asset_path(feature.assets[0].path, "image.preview.jpg")) assert os.path.isfile(get_asset_path(feature.assets[0].path, "image.jpg")) diff --git a/geoapi/utils/agave.py b/geoapi/utils/agave.py index 6ec28873..a7ba004e 100644 --- a/geoapi/utils/agave.py +++ b/geoapi/utils/agave.py @@ -214,6 +214,7 @@ def getFile(self, systemId: str, path: str) -> IO: allowed_attempts = 5 while allowed_attempts > 0: try: + logger.debug(f"Getting file {systemId}/{path}") return self._get_file(systemId, path) except RetryableTapisFileError: allowed_attempts = allowed_attempts - 1 From 9e1c60dca28b5017756e738125142155085aacb0 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 29 Jun 2023 15:12:44 -0500 Subject: [PATCH 04/17] Sort the metadata of assets based on filename --- geoapi/services/features.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 03cabed1..aa439eae 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -4,6 +4,7 @@ import json import tempfile import configparser +import re from typing import List, IO, Dict from geoapi.services.videos import VideoService @@ -299,6 +300,8 @@ def from_rapp_questionnaire(projectId: int, fileObj: IO, additional_files: List[ if additional_files_properties: + # Sort the list of dictionaries based on 'QX' value and then 'PhotoX' value + additional_files_properties.sort(key=lambda x: tuple(map(int, re.findall(r'\d+', x['filename'])))) # add info about assets to properties (i.e. coordinates of asset) for quick retrieval feat.properties = {"_hazmapper": {"questionnaire": {"assets": additional_files_properties}}} From 5868c959bb59911bb7426707851b0f73b88bac53 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 29 Jun 2023 15:13:42 -0500 Subject: [PATCH 05/17] Change logging level for image orientations --- geoapi/services/images.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/geoapi/services/images.py b/geoapi/services/images.py index 0e079b7d..417908b9 100644 --- a/geoapi/services/images.py +++ b/geoapi/services/images.py @@ -79,14 +79,9 @@ def _fix_orientation(fileObj: IO) -> PILImage: # from https://github.com/ianare/exif-py#usage-example im = Image.open(fileObj) tags = exifread.process_file(fileObj, details=False) - if "Image Orientation" in tags.keys(): - logger.info("yes Image Orientation") - else: - logger.info("no Image Orientation") - if "Image Orientation" in tags.keys(): orientation = tags["Image Orientation"] - logger.info("Orientation: %s (%s)", orientation, orientation.values) + logger.debug("image orientation: %s (%s)", orientation, orientation.values) val = orientation.values if 2 in val: val += [4, 3] @@ -95,16 +90,16 @@ def _fix_orientation(fileObj: IO) -> PILImage: if 7 in val: val += [4, 8] if 3 in val: - logger.info("Rotating by 180 degrees.") + logger.debug("Rotating by 180 degrees.") im = im.transpose(Image.ROTATE_180) if 4 in val: - logger.info("Mirroring horizontally.") + logger.debug("Mirroring horizontally.") im = im.transpose(Image.FLIP_TOP_BOTTOM) if 6 in val: - logger.info("Rotating by 270 degrees.") + logger.debug("Rotating by 270 degrees.") im = im.transpose(Image.ROTATE_270) if 8 in val: - logger.info("Rotating by 90 degrees.") + logger.debug("Rotating by 90 degrees.") im = im.transpose(Image.ROTATE_90) return im From 27c8de8ad3749a11453a591a64a724a3d5242821 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 29 Jun 2023 15:17:14 -0500 Subject: [PATCH 06/17] Fix flake8 errors --- geoapi/services/features.py | 1 - geoapi/tasks/external_data.py | 3 --- geoapi/tests/conftest.py | 3 ++- geoapi/tests/external_data_tests/test_external_data.py | 1 + 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index aa439eae..a3b6c8ef 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -298,7 +298,6 @@ def from_rapp_questionnaire(projectId: int, fileObj: IO, additional_files: List[ "coordinates": processed_asset_image.coordinates}) asset_file_obj.close() - if additional_files_properties: # Sort the list of dictionaries based on 'QX' value and then 'PhotoX' value additional_files_properties.sort(key=lambda x: tuple(map(int, re.findall(r'\d+', x['filename'])))) diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index 2b19287d..0c0b8708 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -66,8 +66,6 @@ def get_file(client, system_id, path, required): return system_id, path, required, result_file, error - - def get_additional_files(current_file, system_id: str, path: str, client, available_files=None): """ Get any additional files needed for processing the current file being imported @@ -113,7 +111,6 @@ def get_additional_files(current_file, system_id: str, path: str, client, availa else: return None - # Try to get all additional files. additional_files_result = [] with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index f7c93da5..2d24320c 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -486,10 +486,11 @@ def questionnaire_file_fixture(): f.filename = filename yield f + @pytest.fixture(scope="function") def questionnaire_file_with_assets_fixture(): home = os.path.dirname(__file__) filename = 'fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq' - with open(os.path.join(home, filename ), 'rb') as f: + with open(os.path.join(home, filename), 'rb') as f: f.filename = filename yield f diff --git a/geoapi/tests/external_data_tests/test_external_data.py b/geoapi/tests/external_data_tests/test_external_data.py index 560e296e..6a459547 100644 --- a/geoapi/tests/external_data_tests/test_external_data.py +++ b/geoapi/tests/external_data_tests/test_external_data.py @@ -493,6 +493,7 @@ def test_get_additional_files_rapid_questionnaire(questionnaire_file_with_assets agave_utils_with_geojson_file) assert len(files) == 1 + def test_get_additional_files_rapid_questionnaire_no_assets(questionnaire_file_fixture, agave_utils_with_geojson_file): files = get_additional_files(questionnaire_file_fixture, "testSystem", From 6e7c9cbce70b1f14d6ef6028d8350008bd958dbd Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 29 Jun 2023 15:24:52 -0500 Subject: [PATCH 07/17] Remove unused fixture --- .../questionnaire_with_assets2.rq | 481 ------------------ 1 file changed, 481 deletions(-) delete mode 100644 geoapi/tests/fixtures/questionnaire_with_assets2.rqa/questionnaire_with_assets2.rq diff --git a/geoapi/tests/fixtures/questionnaire_with_assets2.rqa/questionnaire_with_assets2.rq b/geoapi/tests/fixtures/questionnaire_with_assets2.rqa/questionnaire_with_assets2.rq deleted file mode 100644 index baec7257..00000000 --- a/geoapi/tests/fixtures/questionnaire_with_assets2.rqa/questionnaire_with_assets2.rq +++ /dev/null @@ -1,481 +0,0 @@ -{ - "id": 2324, - "description": "Example made by Nathan", - "geolocation": [ - { - "timestamp": 1683578336.2153249, - "latitude": 30.319060724035527, - "course": -1, - "longitude": -97.720111181677041, - "altitude": 208.61720085144043 - } - ], - "uuid": "1E0DEBD1-601A-44EB-A40F-AC00EDBF11F2", - "version": "1E0DEBD1-601A-44EB-A40F-AC00EDBF11F2", - "access": "public", - "owner": "nathanf", - "allow_back": true, - "editable": false, - "end_uuid": "D45B5925-4FB5-4C55-BF64-47918E2C0513", - "questionUuidsVisited": [], - "sections": [ - { - "id": "EC16B5FE-8933-4786-BFEC-E7D8D24A774B", - "label": "", - "questions": [ - { - "id": "EBBC076B-20FD-48CB-A9FD-62B5C57C278A", - "heading": "q1 on first section", - "assets": [ - { - "rappUuid": "F1668F90-95D3-4A29-BD1C-A1956BD24783", - "filename": "Q1-Photo-001.jpg" - } - ], - "label": "Please select one of the following:", - "options": [ - { - "go_to": null, - "default": false, - "label": "Response 1", - "sub_question": null, - "value": "response_1" - }, - { - "go_to": null, - "default": false, - "label": "Response 2", - "sub_question": null, - "value": "response_2" - }, - { - "go_to": null, - "default": false, - "label": "Response 3", - "sub_question": null, - "value": "response_3" - }, - { - "go_to": null, - "default": false, - "label": "Response 4", - "sub_question": null, - "value": "response_4" - } - ], - "responseIndexes": [ - 1 - ], - "decline": null, - "type": "Single Select", - "value": "please_select_one_of_the_follo", - "mode": "list", - "instructions": "test", - "required": true - } - ] - }, - { - "id": "7DC933D8-EB1C-4041-9089-CAD3F57147D2", - "label": "section1", - "questions": [ - { - "id": "1B5C44E7-81DC-413F-92E8-8E56D2517B06", - "heading": "heading for yes and no question", - "assets": [ - { - "rappUuid": "2D679547-FDEF-40AE-8219-740F3895A6D7", - "filename": "Q2-Photo-001.jpg" - }, - { - "rappUuid": "498C45B7-62A3-4735-80EC-5F36A3C5660E", - "filename": "Q2-Photo-002.jpg" - } - ], - "label": "A yes or no question", - "options": [ - { - "go_to": null, - "default": false, - "label": "Yes", - "sub_question": null, - "value": "yes" - }, - { - "go_to": null, - "default": false, - "label": "No", - "sub_question": null, - "value": "no" - } - ], - "responseIndexes": [ - 1 - ], - "decline": null, - "type": "Yes \/ No", - "value": "question", - "mode": "list", - "instructions": "This is a a yes or no question", - "required": true - }, - { - "id": "73472BD4-41E5-4554-8F7D-382E0F230483", - "heading": "heading single select q2 question", - "assets": [ - { - "rappUuid": "9355B455-B678-4A42-9739-6E5643D2C1B8", - "filename": "Q3-Photo-001.jpg" - }, - { - "rappUuid": "DF7C3A81-B282-433A-836A-ED7BDDB1C02E", - "filename": "Q3-Photo-002.jpg" - } - ], - "label": "Please select one of the following:", - "options": [ - { - "go_to": null, - "default": false, - "label": "Response 1", - "sub_question": null, - "value": "response_1" - }, - { - "go_to": null, - "default": false, - "label": "Response 2", - "sub_question": null, - "value": "response_2" - }, - { - "go_to": null, - "default": false, - "label": "Response 3", - "sub_question": null, - "value": "response_3" - }, - { - "go_to": null, - "default": false, - "label": "Response 4", - "sub_question": null, - "value": "response_4" - } - ], - "responseIndexes": [ - 1 - ], - "decline": null, - "type": "Single Select", - "value": "please_select_one_of_the_follo", - "mode": "list", - "instructions": "instructions for q2", - "required": true - }, - { - "value": "please_select_one_or_more_of_t", - "responseIndexes": [ - 1 - ], - "assets": [], - "id": "2CC0FDF0-2279-476A-A547-360738AACF95", - "instructions": "q2 instructions", - "decline": null, - "label": "Please select one or more of the following:", - "type": "Multi Select", - "required": true, - "heading": "q3 haeading", - "options": [ - { - "go_to": null, - "default": false, - "label": "Response 1a", - "sub_question": null, - "value": "response_1a" - }, - { - "go_to": null, - "default": false, - "label": "Response 2b", - "sub_question": null, - "value": "response_2b" - }, - { - "go_to": null, - "default": false, - "label": "Response 3", - "sub_question": null, - "value": "response_3" - }, - { - "go_to": null, - "default": false, - "label": "Response 4b sdfsdf", - "sub_question": null, - "value": "response_4b_sdfsdf" - } - ] - }, - { - "assets": [], - "responseStrings": null, - "id": "A73A00D1-3BF5-46B9-B2B1-93CAC32BFC2A", - "instructions": null, - "decline": null, - "value": "", - "type": "Multi Text", - "required": true, - "heading": "a text field heading", - "questions": [ - { - "assets": [], - "responseStrings": [ - "Test" - ], - "id": "A6352257-EE96-4389-8689-F0CA40E40BAA", - "instructions": "test", - "decline": null, - "mode": "short", - "label": "Response 1", - "type": "Text Field", - "required": true, - "heading": null, - "value": "response_1" - } - ] - }, - { - "assets": [], - "responseStrings": [ - "May 8, 2025" - ], - "id": "0FED5898-D3C9-4466-8957-31EA3C2D4D3D", - "instructions": "date", - "decline": null, - "mode": "date", - "label": "Please enter a date", - "type": "DateTime", - "required": true, - "heading": "date time", - "value": "please_enter_an_date" - }, - { - "assets": [], - "responseStrings": [ - "4" - ], - "id": "A5100404-26BA-4B0D-AEBF-C2D0DC6C7913", - "instructions": null, - "decline": null, - "mode": "integer", - "label": "Please enter an integer", - "type": "Number", - "required": true, - "heading": "q6 number", - "value": "please_enter_an_integer" - }, - { - "id": "BAB27ADF-1EEF-4837-BF5F-957354B6E348", - "heading": "q7 a range", - "assets": [], - "label": "Please rate the following from 1 (the least likely) to 10 (the most likely):", - "options": [ - { - "go_to": null, - "default": false, - "label": "1", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "2", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "3", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "4", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "5", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "6", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "7", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "8", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "9", - "sub_question": null, - "value": null - }, - { - "go_to": null, - "default": false, - "label": "10", - "sub_question": null, - "value": null - } - ], - "responseIndexes": [ - 1 - ], - "decline": null, - "type": "Range", - "value": "please_rate_the_following_from", - "instructions": null, - "range": { - "min": 1, - "max": 10 - }, - "required": true - }, - { - "id": "A26E27C8-D5D2-4C06-ACDC-853CC8D1A233", - "heading": "a matrix", - "label": "Please select one or more of the following:", - "assets": [], - "decline": null, - "responseMatrixIndexes": [ - [ - 0 - ], - [ - 1 - ] - ], - "type": "Matrix", - "value": "", - "columns": [ - { - "label": "Column 1", - "value": "column_1" - }, - { - "label": "Column 2", - "value": "column_2" - } - ], - "mode": "single", - "rows": [ - { - "label": "Row 1", - "value": "row_1" - }, - { - "label": "Row 2", - "value": "row_1" - } - ], - "instructions": null, - "required": false - }, - { - "assets": [], - "responseStrings": [ - "30.319140688537654", - "-97.72014149862869", - "5405 Avenue F\nAustin, TX 78751\nUS" - ], - "id": "BF57DE13-333E-41C7-A2C2-6754BAC97409", - "instructions": null, - "decline": null, - "label": "Please indicate your location", - "type": "Location", - "heading": "q9: a location heading", - "required": true, - "value": "please_indicate_your_location" - }, - { - "assets": [ - { - "rappUuid": "F4AB2BEE-CFF3-4AC5-B28B-054730E15777", - "filename": "Q11-Photo-001.jpg" - } - ], - "responseStrings": null, - "id": "8981DD40-A560-42CD-96F8-8F1982357F7B", - "instructions": null, - "decline": null, - "value": "", - "type": "Text", - "heading": "p2 heading", - "required": false - }, - { - "id": "E2B43583-EEA9-43A8-B247-F4004F989868", - "heading": "q11", - "assets": [], - "label": "Enter your question sdfdsfds", - "options": [ - { - "go_to": null, - "default": false, - "label": "Yes", - "sub_question": null, - "value": "yes" - }, - { - "go_to": null, - "default": false, - "label": "No", - "sub_question": null, - "value": "no" - } - ], - "responseIndexes": [ - 0 - ], - "decline": null, - "type": "Yes \/ No", - "value": "question", - "mode": "list", - "instructions": null, - "required": true - } - ] - } - ], - "asset_embedding": true, - "subversion": "1", - "is_invalid": false, - "display_mode": "multi", - "end_text": "End", - "name": "questionnaire_nmf_example1", - "errors": [] -} From 69d6a6ce1561b22445f43f65014322242a78dc2f Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Tue, 4 Jul 2023 09:01:47 -0500 Subject: [PATCH 08/17] Allow preflight requests --- conf/nginx.kube.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/conf/nginx.kube.conf b/conf/nginx.kube.conf index 79abe44f..a631e38b 100644 --- a/conf/nginx.kube.conf +++ b/conf/nginx.kube.conf @@ -12,6 +12,15 @@ http { max_ranges 0; expires 30d; add_header "Access-Control-Allow-Origin" *; + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; + add_header "Access-Control-Allow-Headers" "*"; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Length' 0; + return 204; + } alias /assets/; } } From dfde96597a98e218e052e27540c155a83fedfcfa Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Wed, 5 Jul 2023 10:09:53 -0500 Subject: [PATCH 09/17] Allow preflight requests to / --- conf/nginx.kube.conf | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/conf/nginx.kube.conf b/conf/nginx.kube.conf index a631e38b..cce3f9be 100644 --- a/conf/nginx.kube.conf +++ b/conf/nginx.kube.conf @@ -8,6 +8,20 @@ http { include /etc/nginx/mime.types; client_max_body_size 10g; + location / { + add_header "Access-Control-Allow-Origin" *; + + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; + add_header "Access-Control-Allow-Headers" "*"; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Length' 0; + return 204; + } + } + location /assets { max_ranges 0; expires 30d; From 192a5f78afbfaa26791c514f803cae2f0d99f9b0 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Wed, 5 Jul 2023 10:20:25 -0500 Subject: [PATCH 10/17] Add preflighted requests to /assets --- kube/geoapi.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kube/geoapi.yaml b/kube/geoapi.yaml index 41570f9b..d5f0e99b 100644 --- a/kube/geoapi.yaml +++ b/kube/geoapi.yaml @@ -48,6 +48,16 @@ data: max_ranges 0; expires 30d; add_header "Access-Control-Allow-Origin" *; + + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; + add_header "Access-Control-Allow-Headers" "*"; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Length' 0; + return 204; + } alias /assets/; } } From 6e2b2f086fbfeb69d89e1b052681a13b9bec022e Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Thu, 28 Sep 2023 10:31:35 -0500 Subject: [PATCH 11/17] Fix merge issues --- geoapi/services/features.py | 3 ++- geoapi/tasks/external_data.py | 3 ++- geoapi/tests/api_tests/test_feature_service.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index 71dc9688..f5251485 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -239,7 +239,8 @@ def fromShapefile(database_session, projectId: int, fileObj: IO, metadata: Dict, return features @staticmethod - def from_rapp_questionnaire(database_session, projectId: int, fileObj: IO, additional_files: List[IO], original_path: str = None) -> Feature: + def from_rapp_questionnaire(database_session, projectId: int, fileObj: IO, + additional_files: List[IO], original_path: str = None) -> Feature: """ Import RAPP questionnaire diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index 2a91433c..9f639b33 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -6,6 +6,7 @@ import time import datetime from celery import uuid as celery_uuid +import json from geoapi.celery_app import app from geoapi.exceptions import InvalidCoordinateReferenceSystem, MissingServiceAccount @@ -358,7 +359,7 @@ def import_from_files_from_path(session, tenant_id: str, userId: int, systemId: logger.info("importing:{} for user:{}".format(item_system_path, user.username)) tmp_file = client.getFile(systemId, item.path) tmp_file.filename = Path(item.path).name - additional_files = get_additional_files(tmpFile, systemId, item.path, client, available_files=filenames_in_directory) + additional_files = get_additional_files(tmp_file, systemId, item.path, client, available_files=filenames_in_directory) FeaturesService.fromFileObj(session, projectId, tmp_file, {}, original_path=item_system_path, additional_files=additional_files) NotificationsService.create(session, user, "success", "Imported {f}".format(f=item_system_path)) diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index 4b4d416f..c09f231f 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -225,7 +225,8 @@ def test_create_questionnaire_feature(projects_fixture, questionnaire_file_fixtu def test_create_questionnaire_feature_with_assets(projects_fixture, questionnaire_file_with_assets_fixture, image_file_fixture): assets = [image_file_fixture] - feature = FeaturesService.from_rapp_questionnaire(db_session, projects_fixture.id, questionnaire_file_with_assets_fixture, additional_files=assets) + feature = FeaturesService.from_rapp_questionnaire(db_session, projects_fixture.id, + questionnaire_file_with_assets_fixture, additional_files=assets) assert feature.project_id == projects_fixture.id assert len(feature.assets) == 1 assert db_session.query(Feature).count() == 1 From df2217e733bb900ab340a0e320220675bcfad3f0 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 13 Oct 2023 16:15:09 -0500 Subject: [PATCH 12/17] Rename fixture --- geoapi/tests/conftest.py | 2 +- .../{questionnaire.rq => questionnaire_without_assets.rq} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename geoapi/tests/fixtures/{questionnaire.rq => questionnaire_without_assets.rq} (100%) diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index f793d03a..152069e5 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -476,7 +476,7 @@ def tile_server_ini_file_fixture(): @pytest.fixture(scope="function") def questionnaire_file_fixture(): home = os.path.dirname(__file__) - filename = 'fixtures/questionnaire.rq' + filename = 'fixtures/questionnaire_without_assets.rq' with open(os.path.join(home, filename), 'rb') as f: f.filename = filename yield f diff --git a/geoapi/tests/fixtures/questionnaire.rq b/geoapi/tests/fixtures/questionnaire_without_assets.rq similarity index 100% rename from geoapi/tests/fixtures/questionnaire.rq rename to geoapi/tests/fixtures/questionnaire_without_assets.rq From 587f212d6f5e87521a37ac416fe31be68c4ada2e Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 13 Oct 2023 16:18:18 -0500 Subject: [PATCH 13/17] Rename fixture and test --- geoapi/tests/api_tests/test_feature_service.py | 6 ++++-- geoapi/tests/conftest.py | 2 +- geoapi/tests/external_data_tests/test_external_data.py | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index c09f231f..86651534 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -212,8 +212,10 @@ def test_create_tile_server_from_file(projects_fixture, tile_server_ini_file_fix assert tile_server.attribution == "OpenStreetMap contributorshttps://www.openstreetmap.org/copyright" -def test_create_questionnaire_feature(projects_fixture, questionnaire_file_fixture): - feature = FeaturesService.from_rapp_questionnaire(db_session, projects_fixture.id, questionnaire_file_fixture, additional_files=None) +def test_create_questionnaire_feature(projects_fixture, questionnaire_file_without_assets_fixture): + feature = FeaturesService.from_rapp_questionnaire(db_session, projects_fixture.id, + questionnaire_file_without_assets_fixture, + additional_files=None) assert feature.project_id == projects_fixture.id assert len(feature.assets) == 1 assert db_session.query(Feature).count() == 1 diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index 152069e5..19c994ba 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -474,7 +474,7 @@ def tile_server_ini_file_fixture(): @pytest.fixture(scope="function") -def questionnaire_file_fixture(): +def questionnaire_file_without_assets_fixture(): home = os.path.dirname(__file__) filename = 'fixtures/questionnaire_without_assets.rq' with open(os.path.join(home, filename), 'rb') as f: diff --git a/geoapi/tests/external_data_tests/test_external_data.py b/geoapi/tests/external_data_tests/test_external_data.py index 5fa56841..f2884580 100644 --- a/geoapi/tests/external_data_tests/test_external_data.py +++ b/geoapi/tests/external_data_tests/test_external_data.py @@ -501,7 +501,7 @@ def test_get_additional_files_shapefiles_but_missing_prj(shapefile_fixture, agav available_files=available_files_missing_prj) -def test_get_additional_files_rapid_questionnaire(questionnaire_file_with_assets_fixture, agave_utils_with_geojson_file): +def test_get_additional_files_rapid_questionnaire_with_assets(questionnaire_file_with_assets_fixture, agave_utils_with_geojson_file): files = get_additional_files(questionnaire_file_with_assets_fixture, "testSystem", questionnaire_file_with_assets_fixture.filename, @@ -509,9 +509,9 @@ def test_get_additional_files_rapid_questionnaire(questionnaire_file_with_assets assert len(files) == 1 -def test_get_additional_files_rapid_questionnaire_no_assets(questionnaire_file_fixture, agave_utils_with_geojson_file): - files = get_additional_files(questionnaire_file_fixture, +def test_get_additional_files_rapid_questionnaire_no_assets(questionnaire_file_without_assets_fixture, agave_utils_with_geojson_file): + files = get_additional_files(questionnaire_file_without_assets_fixture, "testSystem", - questionnaire_file_fixture.filename, + questionnaire_file_without_assets_fixture.filename, agave_utils_with_geojson_file) assert files == [] From c52741573f6503f253cf66aaee7750127f4047fe Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 13 Oct 2023 16:18:58 -0500 Subject: [PATCH 14/17] Improve test name --- geoapi/tests/external_data_tests/test_external_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoapi/tests/external_data_tests/test_external_data.py b/geoapi/tests/external_data_tests/test_external_data.py index f2884580..c82c0bb2 100644 --- a/geoapi/tests/external_data_tests/test_external_data.py +++ b/geoapi/tests/external_data_tests/test_external_data.py @@ -491,7 +491,7 @@ def test_get_additional_files_shapefiles_with_available_files(shapefile_fixture, assert len(files) == 3 -def test_get_additional_files_shapefiles_but_missing_prj(shapefile_fixture, agave_utils_with_geojson_file): +def test_get_additional_files_shapefiles_missing_prj(shapefile_fixture, agave_utils_with_geojson_file): available_files_missing_prj = ["/testPath/file.shx", "/testPath/file.dbf"] with pytest.raises(Exception): get_additional_files(shapefile_fixture, From 117abfaf6a6e9990407a70d820579b0c5eb7bd08 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 13 Oct 2023 16:19:49 -0500 Subject: [PATCH 15/17] Improve test name --- geoapi/tests/external_data_tests/test_external_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geoapi/tests/external_data_tests/test_external_data.py b/geoapi/tests/external_data_tests/test_external_data.py index c82c0bb2..015e6cc7 100644 --- a/geoapi/tests/external_data_tests/test_external_data.py +++ b/geoapi/tests/external_data_tests/test_external_data.py @@ -453,7 +453,7 @@ def test_get_additional_files_none(shapefile_fixture, agave_utils_with_geojson_f assert not get_additional_files(shapefile_fixture, "testSystem", "/testPath/file.jpg", agave_utils_with_geojson_file) -def test_get_additional_shapefiles_files(shapefile_fixture, agave_utils_with_geojson_file): +def test_get_additional_files_shapefiles(shapefile_fixture, agave_utils_with_geojson_file): files = get_additional_files(shapefile_fixture, "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file) assert len(files) == 14 From 1772da69c5fe820efb55e0436420a40bd80d7ede Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 13 Oct 2023 16:40:46 -0500 Subject: [PATCH 16/17] Improve rq related comments --- geoapi/services/features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geoapi/services/features.py b/geoapi/services/features.py index f5251485..3f95b531 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -249,7 +249,7 @@ def from_rapp_questionnaire(database_session, projectId: int, fileObj: IO, as the questionnaire .rq file. :param projectId: int - :param fileObj: questionnaire file + :param fileObj: questionnaire rq file :param additional_files: list of file objs :param original_path: str path of original file location :return: Feature @@ -270,7 +270,7 @@ def from_rapp_questionnaire(database_session, projectId: int, fileObj: IO, pathlib.Path(questionnaire_path).mkdir(parents=True, exist_ok=True) asset_path = os.path.join(questionnaire_path, 'questionnaire.rq') - # write questionnaire file (rq) + # write questionnaire rq file with open(asset_path, 'w') as tmp: tmp.write(json.dumps(data)) From 65eeef84e6a409bbad1d1eda9a241e33f79b77f7 Mon Sep 17 00:00:00 2001 From: Nathan Franklin Date: Fri, 13 Oct 2023 16:45:03 -0500 Subject: [PATCH 17/17] Improve use of quotes --- geoapi/tasks/external_data.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index bba039d7..b7d602b0 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -85,18 +85,18 @@ def get_additional_files(current_file, system_id: str, path: str, client, availa current_file_path = Path(path) file_suffix = current_file_path.suffix.lower().lstrip('.') if file_suffix == "shp": - logger.info(f'Determining which shapefile-related files need to be downloaded for file {current_file.filename}') + logger.info(f"Determining which shapefile-related files need to be downloaded for file {current_file.filename}") for extension, required in SHAPEFILE_FILE_ADDITIONAL_FILES.items(): additional_file_path = current_file_path.with_suffix(extension) if available_files and str(additional_file_path) not in available_files: if required: logger.error(f"Could not import required shapefile-related file: agave: {system_id}/{additional_file_path}") - raise Exception(f'Required file ({system_id}/{additional_file_path}) missing') + raise Exception(f"Required file ({system_id}/{additional_file_path}) missing") else: continue additional_files_to_get.append(AdditionalFile(path=additional_file_path, required=required)) elif file_suffix == "rq": - logger.info(f'Parsing rq file {current_file.filename} to see what assets need to be downloaded ') + logger.info(f"Parsing rq file {current_file.filename} to see what assets need to be downloaded ") data = json.load(current_file) for section in data["sections"]: for question in section["questions"]: @@ -104,7 +104,7 @@ def get_additional_files(current_file, system_id: str, path: str, client, availa # determine full path for this asset and add to list additional_file_path = current_file_path.with_name(asset["filename"]) additional_files_to_get.append(AdditionalFile(path=additional_file_path, required=True)) - logger.info(f'{len(additional_files_to_get)} assets were found for rq file {current_file.filename}') + logger.info(f"{len(additional_files_to_get)} assets were found for rq file {current_file.filename}") # Seek back to start of file current_file.seek(0) @@ -119,15 +119,15 @@ def get_additional_files(current_file, system_id: str, path: str, client, availa for future in concurrent.futures.as_completed(getting_files_futures): _, additional_file_path, required, result_file, error = future.result() if not result_file and required: - logger.error(f'Could not import a required {file_suffix}-related file: ' - f'agave: {system_id} :: {additional_file_path} ---- error: {error}') - raise Exception(f'Required file ({system_id}/{additional_file_path}) missing') + logger.error(f"Could not import a required {file_suffix}-related file: " + f"agave: {system_id} :: {additional_file_path} ---- error: {error}") + raise Exception(f"Required file ({system_id}/{additional_file_path}) missing") if not result_file: - logger.error(f'Unable to get non-required {file_suffix}-related file: ' - f'agave: {system_id} :: {additional_file_path} ---- error: {error}') + logger.error(f"Unable to get non-required {file_suffix}-related file: " + f"agave: {system_id} :: {additional_file_path} ---- error: {error}") continue - logger.debug(f'Finished getting {file_suffix}-related file: ({system_id}/{additional_file_path}') + logger.debug(f"Finished getting {file_suffix}-related file: ({system_id}/{additional_file_path}") result_file.filename = Path(additional_file_path).name additional_files_result.append(result_file) return additional_files_result @@ -217,12 +217,12 @@ def import_point_clouds_from_agave(userId: int, files, pointCloudId: int): except InvalidCoordinateReferenceSystem: logger.error("Could not import point cloud file due to missing" " coordinate reference system: {}:{}".format(system_id, path)) - failed_message = 'Error importing {}: missing coordinate reference system'.format(path) + failed_message = "Error importing {}: missing coordinate reference system".format(path) except Exception as e: logger.error("Could not import point cloud file for user:{} from tapis: {}/{} : {}".format(user.username, system_id, path, e)) - failed_message = 'Unknown error importing {}:{}'.format(system_id, path) + failed_message = "Unknown error importing {}:{}".format(system_id, path) if failed_message: for file_path in new_asset_files: