diff --git a/geoapi/services/features.py b/geoapi/services/features.py index d45cc77f..3f95b531 100644 --- a/geoapi/services/features.py +++ b/geoapi/services/features.py @@ -4,6 +4,7 @@ import json import tempfile import configparser +import re from typing import List, IO, Dict from geoapi.services.videos import VideoService @@ -60,12 +61,10 @@ class FeaturesService: ) ALLOWED_GEOSPATIAL_EXTENSIONS = IMAGE_FILE_EXTENSIONS + GPX_FILE_EXTENSIONS + GEOJSON_FILE_EXTENSIONS\ - + SHAPEFILE_FILE_EXTENSIONS - # RAPP_FILE_EXTENSIONS to be added in https://jira.tacc.utexas.edu/browse/DES-2462 + + SHAPEFILE_FILE_EXTENSIONS + RAPP_FILE_EXTENSIONS ALLOWED_EXTENSIONS = IMAGE_FILE_EXTENSIONS + VIDEO_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS + GPX_FILE_EXTENSIONS\ - + GEOJSON_FILE_EXTENSIONS + SHAPEFILE_FILE_EXTENSIONS + INI_FILE_EXTENSIONS - # RAPP_FILE_EXTENSIONS to be added in https://jira.tacc.utexas.edu/browse/DES-2462 + + GEOJSON_FILE_EXTENSIONS + SHAPEFILE_FILE_EXTENSIONS + INI_FILE_EXTENSIONS + RAPP_FILE_EXTENSIONS @staticmethod def get(database_session, featureId: int) -> Feature: @@ -240,15 +239,22 @@ def fromShapefile(database_session, projectId: int, fileObj: IO, metadata: Dict, return features @staticmethod - def fromRAPP(database_session, projectId: int, fileObj: IO, metadata: Dict, original_path: str = None) -> Feature: + def from_rapp_questionnaire(database_session, projectId: int, fileObj: IO, + additional_files: List[IO], original_path: str = None) -> Feature: """ + Import RAPP questionnaire + + RAPP questionnaire is imported along with any asset images that it + refers to. The asset images are assumed to reside in the same directory + as the questionnaire .rq file. :param projectId: int - :param fileObj: file descriptor - :param metadata: Dict of pairs + :param fileObj: questionnaire rq file + :param additional_files: list of file objs :param original_path: str path of original file location :return: Feature """ + logger.info(f"Processing f{original_path}") data = json.loads(fileObj.read()) lng = data.get('geolocation')[0].get('longitude') @@ -264,9 +270,40 @@ def fromRAPP(database_session, projectId: int, fileObj: IO, metadata: Dict, orig pathlib.Path(questionnaire_path).mkdir(parents=True, exist_ok=True) asset_path = os.path.join(questionnaire_path, 'questionnaire.rq') + # write questionnaire rq file with open(asset_path, 'w') as tmp: tmp.write(json.dumps(data)) + additional_files_properties = [] + + # write all asset files (i.e jpgs) + if additional_files is not None: + logger.info(f"Processing {len(additional_files)} assets for {original_path}") + for asset_file_obj in additional_files: + base_filename = os.path.basename(asset_file_obj.filename) + image_asset_path = os.path.join(questionnaire_path, base_filename) + + # save original jpg (i.e. Q1-Photo-001.jpg) + with open(image_asset_path, 'wb') as image_asset: + image_asset.write(asset_file_obj.read()) + + # create preview image (i.e. Q1-Photo-001.preview.jpg) + processed_asset_image = ImageService.processImage(asset_file_obj) + path = pathlib.Path(image_asset_path) + processed_asset_image.resized.save(path.with_suffix('.preview' + path.suffix), "JPEG") + + # gather coordinates information for this asset + logger.debug(f"{asset_file_obj.filename} has the geospatial coordinates of {processed_asset_image.coordinates}") + additional_files_properties.append({"filename": base_filename, + "coordinates": processed_asset_image.coordinates}) + asset_file_obj.close() + + if additional_files_properties: + # Sort the list of dictionaries based on 'QX' value and then 'PhotoX' value + additional_files_properties.sort(key=lambda x: tuple(map(int, re.findall(r'\d+', x['filename'])))) + # add info about assets to properties (i.e. coordinates of asset) for quick retrieval + feat.properties = {"_hazmapper": {"questionnaire": {"assets": additional_files_properties}}} + fa = FeatureAsset( uuid=asset_uuid, asset_type="questionnaire", @@ -344,8 +381,8 @@ def fromFileObj(database_session, projectId: int, fileObj: IO, return FeaturesService.fromShapefile(database_session, projectId, fileObj, {}, additional_files, original_path) elif ext in FeaturesService.INI_FILE_EXTENSIONS: return FeaturesService.fromINI(database_session, projectId, fileObj, {}, original_path) - elif False and ext in FeaturesService.RAPP_FILE_EXTENSIONS: # Activate for https://jira.tacc.utexas.edu/browse/DES-2462 - return FeaturesService.fromRAPP(database_session, projectId, fileObj, {}, original_path) + elif ext in FeaturesService.RAPP_FILE_EXTENSIONS: + return FeaturesService.from_rapp_questionnaire(database_session, projectId, fileObj, additional_files, original_path) else: raise ApiException("Filetype not supported for direct upload. Create a feature and attach as an asset?") diff --git a/geoapi/services/images.py b/geoapi/services/images.py index 0e079b7d..417908b9 100644 --- a/geoapi/services/images.py +++ b/geoapi/services/images.py @@ -79,14 +79,9 @@ def _fix_orientation(fileObj: IO) -> PILImage: # from https://github.com/ianare/exif-py#usage-example im = Image.open(fileObj) tags = exifread.process_file(fileObj, details=False) - if "Image Orientation" in tags.keys(): - logger.info("yes Image Orientation") - else: - logger.info("no Image Orientation") - if "Image Orientation" in tags.keys(): orientation = tags["Image Orientation"] - logger.info("Orientation: %s (%s)", orientation, orientation.values) + logger.debug("image orientation: %s (%s)", orientation, orientation.values) val = orientation.values if 2 in val: val += [4, 3] @@ -95,16 +90,16 @@ def _fix_orientation(fileObj: IO) -> PILImage: if 7 in val: val += [4, 8] if 3 in val: - logger.info("Rotating by 180 degrees.") + logger.debug("Rotating by 180 degrees.") im = im.transpose(Image.ROTATE_180) if 4 in val: - logger.info("Mirroring horizontally.") + logger.debug("Mirroring horizontally.") im = im.transpose(Image.FLIP_TOP_BOTTOM) if 6 in val: - logger.info("Rotating by 270 degrees.") + logger.debug("Rotating by 270 degrees.") im = im.transpose(Image.ROTATE_270) if 8 in val: - logger.info("Rotating by 90 degrees.") + logger.debug("Rotating by 90 degrees.") im = im.transpose(Image.ROTATE_90) return im diff --git a/geoapi/tasks/external_data.py b/geoapi/tasks/external_data.py index 7ba7d4d0..b7d602b0 100644 --- a/geoapi/tasks/external_data.py +++ b/geoapi/tasks/external_data.py @@ -6,6 +6,7 @@ import time import datetime from celery import uuid as celery_uuid +import json from geoapi.celery_app import app from geoapi.exceptions import InvalidCoordinateReferenceSystem, MissingServiceAccount @@ -21,6 +22,7 @@ from geoapi.db import create_task_session from geoapi.services.notifications import NotificationsService from geoapi.services.users import UserService +from dataclasses import dataclass class ImportState(Enum): @@ -29,6 +31,13 @@ class ImportState(Enum): RETRYABLE_FAILURE = 3 +@dataclass +class AdditionalFile: + """Represents an additional file with its path and and if its required (i.e. not optional).""" + path: str + required: bool + + def _parse_rapid_geolocation(loc): coords = loc[0] lat = coords["latitude"] @@ -57,47 +66,71 @@ def get_file(client, system_id, path, required): return system_id, path, required, result_file, error -def get_additional_files(systemId: str, path: str, client, available_files=None): +def get_additional_files(current_file, system_id: str, path: str, client, available_files=None): """ - Get any additional files needed for processing - :param systemId: str - :param path: str - :param client + Get any additional files needed for processing the current file being imported + + Note `available_files` is optional. if provided, then it can be used to fail early if it is known + that a required file is missing + + :param str current_file: active file that is being imported + :param str system_id: system of active file + :param path: path of active file + :param client: :param available_files: list of files that exist (optional) :return: list of additional files """ - path = Path(path) - if path.suffix.lower().lstrip('.') == "shp": - paths_to_get = [] + additional_files_to_get = [] + + current_file_path = Path(path) + file_suffix = current_file_path.suffix.lower().lstrip('.') + if file_suffix == "shp": + logger.info(f"Determining which shapefile-related files need to be downloaded for file {current_file.filename}") for extension, required in SHAPEFILE_FILE_ADDITIONAL_FILES.items(): - additional_file_path = path.with_suffix(extension) + additional_file_path = current_file_path.with_suffix(extension) if available_files and str(additional_file_path) not in available_files: if required: - logger.error("Could not import required shapefile-related file: " - "agave: {} :: {}".format(systemId, additional_file_path)) - raise Exception("Required file ({}) missing".format(additional_file_path)) + logger.error(f"Could not import required shapefile-related file: agave: {system_id}/{additional_file_path}") + raise Exception(f"Required file ({system_id}/{additional_file_path}) missing") else: continue - paths_to_get.append(additional_file_path) - - additional_files = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - getting_files_futures = [executor.submit(get_file, client, systemId, additional_file_path, required) - for additional_file_path in paths_to_get] - for future in concurrent.futures.as_completed(getting_files_futures): - _, additional_file_path, required, result_file, error = future.result() - if not result_file and required: - logger.error("Could not import a required shapefile-related file: " - "agave: {} :: {} ---- error: {}".format(systemId, additional_file_path, error)) - if not result_file: - logger.debug("Unable to get non-required shapefile-related file: " - "agave: {} :: {}".format(systemId, additional_file_path)) - continue - result_file.filename = Path(additional_file_path).name - additional_files.append(result_file) + additional_files_to_get.append(AdditionalFile(path=additional_file_path, required=required)) + elif file_suffix == "rq": + logger.info(f"Parsing rq file {current_file.filename} to see what assets need to be downloaded ") + data = json.load(current_file) + for section in data["sections"]: + for question in section["questions"]: + for asset in question.get("assets", []): + # determine full path for this asset and add to list + additional_file_path = current_file_path.with_name(asset["filename"]) + additional_files_to_get.append(AdditionalFile(path=additional_file_path, required=True)) + logger.info(f"{len(additional_files_to_get)} assets were found for rq file {current_file.filename}") + + # Seek back to start of file + current_file.seek(0) else: - additional_files = None - return additional_files + return None + + # Try to get all additional files. + additional_files_result = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + getting_files_futures = [executor.submit(get_file, client, system_id, additional_file.path, additional_file.required) + for additional_file in additional_files_to_get] + for future in concurrent.futures.as_completed(getting_files_futures): + _, additional_file_path, required, result_file, error = future.result() + if not result_file and required: + logger.error(f"Could not import a required {file_suffix}-related file: " + f"agave: {system_id} :: {additional_file_path} ---- error: {error}") + raise Exception(f"Required file ({system_id}/{additional_file_path}) missing") + if not result_file: + logger.error(f"Unable to get non-required {file_suffix}-related file: " + f"agave: {system_id} :: {additional_file_path} ---- error: {error}") + + continue + logger.debug(f"Finished getting {file_suffix}-related file: ({system_id}/{additional_file_path}") + result_file.filename = Path(additional_file_path).name + additional_files_result.append(result_file) + return additional_files_result @app.task(rate_limit="10/s") @@ -111,10 +144,9 @@ def import_file_from_agave(userId: int, systemId: str, path: str, projectId: int try: user = session.query(User).get(userId) client = AgaveUtils(user.jwt) - temp_file = client.getFile(systemId, path) temp_file.filename = Path(path).name - additional_files = get_additional_files(systemId, path, client) + additional_files = get_additional_files(temp_file, systemId, path, client) FeaturesService.fromFileObj(session, projectId, temp_file, {}, original_path=path, additional_files=additional_files) NotificationsService.create(session, user, "success", "Imported {f}".format(f=path)) @@ -185,12 +217,12 @@ def import_point_clouds_from_agave(userId: int, files, pointCloudId: int): except InvalidCoordinateReferenceSystem: logger.error("Could not import point cloud file due to missing" " coordinate reference system: {}:{}".format(system_id, path)) - failed_message = 'Error importing {}: missing coordinate reference system'.format(path) + failed_message = "Error importing {}: missing coordinate reference system".format(path) except Exception as e: logger.error("Could not import point cloud file for user:{} from tapis: {}/{} : {}".format(user.username, system_id, path, e)) - failed_message = 'Unknown error importing {}:{}'.format(system_id, path) + failed_message = "Unknown error importing {}:{}".format(system_id, path) if failed_message: for file_path in new_asset_files: @@ -331,7 +363,7 @@ def import_from_files_from_path(session, tenant_id: str, userId: int, systemId: logger.info("importing:{} for user:{}".format(item_system_path, user.username)) tmp_file = client.getFile(systemId, item.path) tmp_file.filename = Path(item.path).name - additional_files = get_additional_files(systemId, item.path, client, filenames_in_directory) + additional_files = get_additional_files(tmp_file, systemId, item.path, client, available_files=filenames_in_directory) FeaturesService.fromFileObj(session, projectId, tmp_file, {}, original_path=item_system_path, additional_files=additional_files) NotificationsService.create(session, user, "success", "Imported {f}".format(f=item_system_path)) diff --git a/geoapi/tests/api_tests/test_feature_service.py b/geoapi/tests/api_tests/test_feature_service.py index 340c0ffe..86651534 100644 --- a/geoapi/tests/api_tests/test_feature_service.py +++ b/geoapi/tests/api_tests/test_feature_service.py @@ -212,11 +212,29 @@ def test_create_tile_server_from_file(projects_fixture, tile_server_ini_file_fix assert tile_server.attribution == "OpenStreetMap contributorshttps://www.openstreetmap.org/copyright" -def test_create_questionnaire_feature(projects_fixture, questionnaire_file_fixture): - feature = FeaturesService.fromRAPP(db_session, projects_fixture.id, questionnaire_file_fixture, metadata={}) +def test_create_questionnaire_feature(projects_fixture, questionnaire_file_without_assets_fixture): + feature = FeaturesService.from_rapp_questionnaire(db_session, projects_fixture.id, + questionnaire_file_without_assets_fixture, + additional_files=None) assert feature.project_id == projects_fixture.id assert len(feature.assets) == 1 assert db_session.query(Feature).count() == 1 assert db_session.query(FeatureAsset).count() == 1 assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 1 - assert os.path.isfile(os.path.join(get_project_asset_dir(projects_fixture.id), str(feature.assets[0].uuid) + "/questionnaire.rq")) + assert len(os.listdir(get_asset_path(feature.assets[0].path))) == 1 + assert os.path.isfile(get_asset_path(feature.assets[0].path, "questionnaire.rq")) + + +def test_create_questionnaire_feature_with_assets(projects_fixture, questionnaire_file_with_assets_fixture, image_file_fixture): + assets = [image_file_fixture] + feature = FeaturesService.from_rapp_questionnaire(db_session, projects_fixture.id, + questionnaire_file_with_assets_fixture, additional_files=assets) + assert feature.project_id == projects_fixture.id + assert len(feature.assets) == 1 + assert db_session.query(Feature).count() == 1 + assert db_session.query(FeatureAsset).count() == 1 + assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 1 + assert len(os.listdir(get_asset_path(feature.assets[0].path))) == 3 + assert os.path.isfile(get_asset_path(feature.assets[0].path, "questionnaire.rq")) + assert os.path.isfile(get_asset_path(feature.assets[0].path, "image.preview.jpg")) + assert os.path.isfile(get_asset_path(feature.assets[0].path, "image.jpg")) diff --git a/geoapi/tests/conftest.py b/geoapi/tests/conftest.py index 5acc390a..19c994ba 100644 --- a/geoapi/tests/conftest.py +++ b/geoapi/tests/conftest.py @@ -162,6 +162,7 @@ def gpx_file_fixture(): def image_file_fixture(): home = os.path.dirname(__file__) with open(os.path.join(home, 'fixtures/image.jpg'), 'rb') as f: + f.filename = 'image.jpg' yield f @@ -473,7 +474,18 @@ def tile_server_ini_file_fixture(): @pytest.fixture(scope="function") -def questionnaire_file_fixture(): +def questionnaire_file_without_assets_fixture(): home = os.path.dirname(__file__) - with open(os.path.join(home, 'fixtures/questionnaire.rq'), 'rb') as f: + filename = 'fixtures/questionnaire_without_assets.rq' + with open(os.path.join(home, filename), 'rb') as f: + f.filename = filename + yield f + + +@pytest.fixture(scope="function") +def questionnaire_file_with_assets_fixture(): + home = os.path.dirname(__file__) + filename = 'fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq' + with open(os.path.join(home, filename), 'rb') as f: + f.filename = filename yield f diff --git a/geoapi/tests/external_data_tests/test_external_data.py b/geoapi/tests/external_data_tests/test_external_data.py index dcd02e5b..015e6cc7 100644 --- a/geoapi/tests/external_data_tests/test_external_data.py +++ b/geoapi/tests/external_data_tests/test_external_data.py @@ -449,16 +449,16 @@ def test_is_member_of_rapp_project_folder(): assert not is_member_of_rapp_project_folder("/something/test.jpg") -def test_get_additional_files_none(agave_utils_with_geojson_file): - assert not get_additional_files("testSystem", "/testPath/file.jpg", agave_utils_with_geojson_file) +def test_get_additional_files_none(shapefile_fixture, agave_utils_with_geojson_file): + assert not get_additional_files(shapefile_fixture, "testSystem", "/testPath/file.jpg", agave_utils_with_geojson_file) -def test_get_additional_files(agave_utils_with_geojson_file): - files = get_additional_files("testSystem", "/testPath/file.shp", agave_utils_with_geojson_file) +def test_get_additional_files_shapefiles(shapefile_fixture, agave_utils_with_geojson_file): + files = get_additional_files(shapefile_fixture, "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file) assert len(files) == 14 -def test_get_additional_files_with_available_files(agave_utils_with_geojson_file): +def test_get_additional_files_shapefiles_with_available_files(shapefile_fixture, agave_utils_with_geojson_file): available_files = ["/testPath/file.shx", "/testPath/file.dbf", "/testPath/file.sbn", @@ -473,7 +473,8 @@ def test_get_additional_files_with_available_files(agave_utils_with_geojson_file "/testPath/file.prj", "/testPath/file.xml", "/testPath/file.cpg"] - files = get_additional_files("testSystem", + files = get_additional_files(shapefile_fixture, + "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file, available_files=available_files) @@ -482,17 +483,35 @@ def test_get_additional_files_with_available_files(agave_utils_with_geojson_file available_files = ["/testPath/file.shx", "/testPath/file.dbf", "/testPath/file.prj"] - files = get_additional_files("testSystem", + files = get_additional_files(shapefile_fixture, + "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file, available_files=available_files) assert len(files) == 3 -def test_get_additional_files_but_missing_prj(agave_utils_with_geojson_file): +def test_get_additional_files_shapefiles_missing_prj(shapefile_fixture, agave_utils_with_geojson_file): available_files_missing_prj = ["/testPath/file.shx", "/testPath/file.dbf"] with pytest.raises(Exception): - get_additional_files("testSystem", + get_additional_files(shapefile_fixture, + "testSystem", "/testPath/file.shp", agave_utils_with_geojson_file, available_files=available_files_missing_prj) + + +def test_get_additional_files_rapid_questionnaire_with_assets(questionnaire_file_with_assets_fixture, agave_utils_with_geojson_file): + files = get_additional_files(questionnaire_file_with_assets_fixture, + "testSystem", + questionnaire_file_with_assets_fixture.filename, + agave_utils_with_geojson_file) + assert len(files) == 1 + + +def test_get_additional_files_rapid_questionnaire_no_assets(questionnaire_file_without_assets_fixture, agave_utils_with_geojson_file): + files = get_additional_files(questionnaire_file_without_assets_fixture, + "testSystem", + questionnaire_file_without_assets_fixture.filename, + agave_utils_with_geojson_file) + assert files == [] diff --git a/geoapi/tests/fixtures/questionnaire.rq b/geoapi/tests/fixtures/questionnaire.rq deleted file mode 100644 index 76137a0f..00000000 --- a/geoapi/tests/fixtures/questionnaire.rq +++ /dev/null @@ -1 +0,0 @@ -{"id":2171,"description":"Yellowstone Test","geolocation":[{"longitude":-122.30502990145563,"timestamp":1658877846.8723259,"latitude":47.652537730540047,"course":-1,"heading":24.99053955078125,"altitude":38.239620208740234}],"uuid":"8288DDDC-E50B-4A72-980E-84D7BD066BA9","version":"8288DDDC-E50B-4A72-980E-84D7BD066BA9","access":"private","owner":"elliot_n","end_uuid":"B20EBBAE-E3E2-42AB-88EA-7F9068BD4502","editable":false,"allow_back":true,"questionUuidsVisited":[],"sections":[{"id":"AA901679-2D1F-47BD-A67B-ABA5FDB0B23C","label":"Researcher Questions","questions":[{"id":"B5B0D1D8-7555-4414-A9FE-C3389AFD000F","heading":"Q1","label":"Are you wearing proper PPE","options":[{"go_to":null,"default":false,"label":"Yes","sub_question":null,"value":"yes"},{"go_to":null,"default":false,"label":"No","sub_question":null,"value":"no"}],"responseIndexes":[0],"decline":null,"type":"Yes \/ No","value":"question","mode":"list","assetUuids":[],"instructions":"Select one","required":true},{"id":"06A57174-5336-4BC4-BF87-BD5E9C6AF9B4","heading":"Is your subject wearing proper PPE?","label":"Enter your question text","options":[{"go_to":null,"default":false,"label":"Yes","sub_question":null,"value":"yes"},{"go_to":null,"default":false,"label":"No","sub_question":null,"value":"no"}],"responseIndexes":[0],"decline":null,"type":"Yes \/ No","value":"enter_your_question_text","mode":"list","assetUuids":[],"instructions":"Select one:","required":true},{"id":"2DAC188A-BBD0-42FF-B0B9-0C88498B2D36","heading":"Q3","label":"Please select one of the following:","options":[{"go_to":null,"default":false,"label":"Uninjured, not displaced","sub_question":null,"value":"uninjured_not_displaced"},{"go_to":null,"default":false,"label":"Minor Injuries, not displaced","sub_question":null,"value":"minor_injuries_not_displaced"},{"go_to":null,"default":false,"label":"Major injuries, not displaced","sub_question":null,"value":"major_injuries_not_displaced"},{"go_to":null,"default":false,"label":"Uninjured, displaced","sub_question":null,"value":"uninjured_displaced"},{"go_to":null,"default":false,"label":"Minor Injuries, displaced","sub_question":null,"value":"minor_injuries_displaced"},{"go_to":null,"default":false,"label":"Major injuries, displaced","sub_question":null,"value":"major_injuries_displaced"}],"responseIndexes":[5],"decline":null,"type":"Single Select","value":"please_select_one_of_the_follo","mode":"list","assetUuids":[],"instructions":"What is the status of your subject?","required":true}]},{"id":"EFFDA218-C78B-4EED-B107-7EB1A96F9C93","label":"Resident Questions","questions":[{"value":"primary_residence","responseIndexes":[0],"id":"2C63F564-BC9F-4BDA-B511-68470AF464B1","instructions":"Where were you during the flooding event?","decline":null,"assetUuids":[],"label":"Please select one or more of the following:","type":"Multi Select","required":true,"heading":"Q1","options":[{"go_to":null,"default":false,"label":"Primary Residence","sub_question":null,"value":"primary_residence"},{"go_to":null,"default":false,"label":"Secondary Residence","sub_question":null,"value":"secondary_residence"},{"go_to":null,"default":false,"label":"In the area, but not at primary\/secondary residence","sub_question":null,"value":"in_the_area_but_not_at_primary"},{"go_to":null,"default":false,"label":"Out of town","sub_question":null,"value":"out_of_town"},{"go_to":null,"default":false,"label":"Other","sub_question":null,"value":"other"}]}]}],"subversion":"1","is_invalid":false,"display_mode":"multi","end_text":"End","name":"EN_Test","errors":[]} diff --git a/geoapi/tests/fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq b/geoapi/tests/fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq new file mode 100644 index 00000000..85d16cac --- /dev/null +++ b/geoapi/tests/fixtures/questionnaire_with_assets.rqa/questionnaire_with_assets.rq @@ -0,0 +1,56 @@ +{ + "editable": false, + "uuid": "DE99A39A-B352-4FF6-BA6B-C325BB25CDFF", + "access": "private", + "subversion": "1", + "allow_back": true, + "errors": [], + "geolocation": [ + { + "altitude": 44.7580108642578, + "latitude": 12.3456789, + "course": -1, + "timestamp": 1680540248.33747, + "longitude": -123.456789 + } + ], + "version": "DE99A39A-B352-4FF6-BA6B-C325BB25CDFF", + "end_text": "End", + "description": "Questionnaire with only one question", + "sections": [ + { + "id": "D1893426-9206-4678-9589-B81B9E7042AB", + "label": "", + "questions": [ + { + "decline": null, + "id": "7389C442-05C0-46DA-AB1E-C9AF29A29A0C", + "required": true, + "responseStrings": [ + "5" + ], + "type": "Number", + "mode": "integer", + "heading": null, + "label": "Please enter an integer", + "assets": [ + { + "rappUuid": "9378FA7A-8BDD-4947-A3EC-2D66B938C59F", + "filename": "Q1-Photo-001.jpg" + } + ], + "instructions": null, + "value": "please_enter_an_integer" + } + ] + } + ], + "owner": "adioso", + "questionUuidsVisited": [], + "asset_embedding": true, + "is_invalid": false, + "id": 1918, + "display_mode": "single", + "end_uuid": "C551C60C-AA8D-4BBD-BCE4-87A1A11120E7", + "name": "Single Question" +} \ No newline at end of file diff --git a/geoapi/tests/fixtures/questionnaire_without_assets.rq b/geoapi/tests/fixtures/questionnaire_without_assets.rq new file mode 100644 index 00000000..48172d85 --- /dev/null +++ b/geoapi/tests/fixtures/questionnaire_without_assets.rq @@ -0,0 +1,214 @@ +{ + "id": 2171, + "description": "Yellowstone Test", + "geolocation": [ + { + "longitude": -122.30502990145563, + "timestamp": 1658877846.8723259, + "latitude": 47.652537730540047, + "course": -1, + "heading": 24.99053955078125, + "altitude": 38.239620208740234 + } + ], + "uuid": "8288DDDC-E50B-4A72-980E-84D7BD066BA9", + "version": "8288DDDC-E50B-4A72-980E-84D7BD066BA9", + "access": "private", + "owner": "elliot_n", + "end_uuid": "B20EBBAE-E3E2-42AB-88EA-7F9068BD4502", + "editable": false, + "allow_back": true, + "questionUuidsVisited": [], + "sections": [ + { + "id": "AA901679-2D1F-47BD-A67B-ABA5FDB0B23C", + "label": "Researcher Questions", + "questions": [ + { + "id": "B5B0D1D8-7555-4414-A9FE-C3389AFD000F", + "heading": "Q1", + "label": "Are you wearing proper PPE", + "options": [ + { + "go_to": null, + "default": false, + "label": "Yes", + "sub_question": null, + "value": "yes" + }, + { + "go_to": null, + "default": false, + "label": "No", + "sub_question": null, + "value": "no" + } + ], + "responseIndexes": [ + 0 + ], + "decline": null, + "type": "Yes \/ No", + "value": "question", + "mode": "list", + "assetUuids": [], + "instructions": "Select one", + "required": true + }, + { + "id": "06A57174-5336-4BC4-BF87-BD5E9C6AF9B4", + "heading": "Is your subject wearing proper PPE?", + "label": "Enter your question text", + "options": [ + { + "go_to": null, + "default": false, + "label": "Yes", + "sub_question": null, + "value": "yes" + }, + { + "go_to": null, + "default": false, + "label": "No", + "sub_question": null, + "value": "no" + } + ], + "responseIndexes": [ + 0 + ], + "decline": null, + "type": "Yes \/ No", + "value": "enter_your_question_text", + "mode": "list", + "assetUuids": [], + "instructions": "Select one:", + "required": true + }, + { + "id": "2DAC188A-BBD0-42FF-B0B9-0C88498B2D36", + "heading": "Q3", + "label": "Please select one of the following:", + "options": [ + { + "go_to": null, + "default": false, + "label": "Uninjured, not displaced", + "sub_question": null, + "value": "uninjured_not_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Minor Injuries, not displaced", + "sub_question": null, + "value": "minor_injuries_not_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Major injuries, not displaced", + "sub_question": null, + "value": "major_injuries_not_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Uninjured, displaced", + "sub_question": null, + "value": "uninjured_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Minor Injuries, displaced", + "sub_question": null, + "value": "minor_injuries_displaced" + }, + { + "go_to": null, + "default": false, + "label": "Major injuries, displaced", + "sub_question": null, + "value": "major_injuries_displaced" + } + ], + "responseIndexes": [ + 5 + ], + "decline": null, + "type": "Single Select", + "value": "please_select_one_of_the_follo", + "mode": "list", + "assetUuids": [], + "instructions": "What is the status of your subject?", + "required": true + } + ] + }, + { + "id": "EFFDA218-C78B-4EED-B107-7EB1A96F9C93", + "label": "Resident Questions", + "questions": [ + { + "value": "primary_residence", + "responseIndexes": [ + 0 + ], + "id": "2C63F564-BC9F-4BDA-B511-68470AF464B1", + "instructions": "Where were you during the flooding event?", + "decline": null, + "assetUuids": [], + "label": "Please select one or more of the following:", + "type": "Multi Select", + "required": true, + "heading": "Q1", + "options": [ + { + "go_to": null, + "default": false, + "label": "Primary Residence", + "sub_question": null, + "value": "primary_residence" + }, + { + "go_to": null, + "default": false, + "label": "Secondary Residence", + "sub_question": null, + "value": "secondary_residence" + }, + { + "go_to": null, + "default": false, + "label": "In the area, but not at primary\/secondary residence", + "sub_question": null, + "value": "in_the_area_but_not_at_primary" + }, + { + "go_to": null, + "default": false, + "label": "Out of town", + "sub_question": null, + "value": "out_of_town" + }, + { + "go_to": null, + "default": false, + "label": "Other", + "sub_question": null, + "value": "other" + } + ] + } + ] + } + ], + "subversion": "1", + "is_invalid": false, + "display_mode": "multi", + "end_text": "End", + "name": "EN_Test", + "errors": [] +} diff --git a/geoapi/utils/agave.py b/geoapi/utils/agave.py index 6ec28873..a7ba004e 100644 --- a/geoapi/utils/agave.py +++ b/geoapi/utils/agave.py @@ -214,6 +214,7 @@ def getFile(self, systemId: str, path: str) -> IO: allowed_attempts = 5 while allowed_attempts > 0: try: + logger.debug(f"Getting file {systemId}/{path}") return self._get_file(systemId, path) except RetryableTapisFileError: allowed_attempts = allowed_attempts - 1 diff --git a/kube/geoapi.yaml b/kube/geoapi.yaml index fd4725c5..4a802b35 100644 --- a/kube/geoapi.yaml +++ b/kube/geoapi.yaml @@ -48,6 +48,16 @@ data: max_ranges 0; expires 30d; add_header "Access-Control-Allow-Origin" *; + + # Preflighted requests + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE"; + add_header "Access-Control-Allow-Headers" "*"; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Length' 0; + return 204; + } alias /assets/; } }