Hotfix/mapillary (#212)

* Use 4bfebf8 hash for mapillary tools * Fix typing * Log additional errors * Fix missing param * Pining mapillary_tooks to 0.9.5 Pinning to hash is causing problems in some environmemnts * Fix regression of missing database session param * Fix regression of missing database session param Missed one spot
TACC-Cloud · Aug 5, 2024 · 3071438 · 3071438
1 parent 6763af1
commit 3071438
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 37 deletions.
diff --git a/devops/poetry.lock b/devops/poetry.lock
diff --git a/devops/pyproject.toml b/devops/pyproject.toml
@@ -35,7 +35,7 @@ ExifRead = "^2.3.2"
 flake8 = "^5.0.4"
 urllib3 = "<2"
 pytest-mock = "^3.12.0"
-mapillary-tools = {git = "https://github.com/mapillary/mapillary_tools.git"}
+mapillary-tools = "0.9.5"
 
 [tool.poetry.dev-dependencies]
 

diff --git a/geoapi/services/notifications.py b/geoapi/services/notifications.py
@@ -67,7 +67,7 @@ def getProgressStatus(database_session, status: str) -> List[ProgressNotificatio
 
     @staticmethod
     def createProgress(database_session, user: User, status: AnyStr, message: AnyStr, task_uuid: UUID,
-                       logs: List = None) -> ProgressNotification:
+                       logs: Dict = None) -> ProgressNotification:
         note = ProgressNotification(
             user_id=user.id,
             tenant_id=user.tenant_id,

diff --git a/geoapi/tasks/streetview.py b/geoapi/tasks/streetview.py
@@ -41,7 +41,7 @@ def publish(database_session, user: User, params: Dict):
     streetview_service = StreetviewService.getByService(database_session, user, service)
 
     if not streetview_service.token or not streetview_service.service_user:
-        logger.error('Not authenticated to {} for user: {}'.format(params['service'], user.username))
+        logger.exception('Not authenticated to {} for user:{}'.format(params['service'], user.username))
         raise StreetviewAuthException('Not authenticated to {}!'.format(service))
 
     # TODO: Find better solution for limiting uploads.
@@ -153,9 +153,10 @@ def _to_mapillary(database_session, user: User, streetview_instance: StreetviewI
                                             "Uploading to Mapillary")
 
         MapillaryUtils.authenticate(user.id, token, service_user)
-        MapillaryUtils.upload(user.id, task_uuid, service_user, organization_key)
+        MapillaryUtils.upload(database_session, user.id, task_uuid, service_user, organization_key)
     except Exception as e:
-        raise Exception("Errors during mapillary upload task {} for user {}: Error: {}".format(task_uuid, user.username, e))
+        msg = "Errors during mapillary upload task {} for user {}: Error: {}".format(task_uuid, user.username, e)
+        raise Exception(msg)
 
 
 # NOTE: At the time of writing, Mapillary's api does not return the sequence
@@ -214,6 +215,8 @@ def from_tapis_to_streetview(user_id: int,
         try:
             check_existing_upload(session, user, streetview_service, task_uuid, system_id, path)
         except StreetviewExistsException as e:
+            logger.exception(f"Error checking existing upload for user:{user.username} "
+                             f"system_id:{system_id}, path:{path}, {e})")
             NotificationsService.create(session, user, 'warning', str(e))
             return
 
@@ -241,8 +244,9 @@ def from_tapis_to_streetview(user_id: int,
         # TODO: Handle
         except Exception as e:
             error_message = "Error during getting files from tapis system:{} path:{} \
-            for streetview upload task: {} for user: {}. Error Message: {}" \
+            for streetview upload task: {} for user:{}. Error Message: {}" \
                   .format(system_id, path, task_uuid, user.username, e)
+            logger.exception(error_message)
             clean_session(session,
                           streetview_instance,
                           user,
@@ -256,9 +260,10 @@ def from_tapis_to_streetview(user_id: int,
             try:
                 _to_mapillary(session, user, streetview_instance, task_uuid, organization_key)
             except Exception as e:
-                error_message = "Error during uploading to mapillary for streetview task: {} \
-                  for user: {}. Error message: {}" \
+                error_message = "Error during uploading to mapillary for streetview task:{} \
+                  for user:{}. Error message: {}" \
                   .format(task_uuid, user.username, e)
+                logger.exception(error_message)
                 clean_session(session,
                               streetview_instance,
                               user,
@@ -271,10 +276,11 @@ def from_tapis_to_streetview(user_id: int,
             try:
                 _mapillary_finalize(session, user, streetview_instance, task_uuid, organization_key)
             except Exception as e:
-                error_message = "Error during finalization of mapillary upload for streetview task: {} \
-                for user: {}. Error message: {}" \
-                      .format(task_uuid, user.username, e)
-                clean_session(streetview_instance,
+                error_message = ("Error during finalization of mapillary upload for streetview task:{} "
+                                 "for user:{}. Error message: {}").format(task_uuid, user.username, e)
+                logger.exception(error_message)
+                clean_session(session,
+                              streetview_instance,
                               user,
                               task_uuid,
                               'error',

diff --git a/geoapi/utils/streetview.py b/geoapi/utils/streetview.py
@@ -113,7 +113,7 @@ def parse_upload_output():
         pass
 
     @staticmethod
-    def upload(userId: int, task_uuid: UUID, service_user: str, organization_key: str):
+    def upload(database_session, userId: int, task_uuid: UUID, service_user: str, organization_key: str):
         command = [
             '/opt/conda/bin/mapillary_tools',
             'process_and_upload',
@@ -134,7 +134,10 @@ def upload(userId: int, task_uuid: UUID, service_user: str, organization_key: st
                                         'MAPILLARY_CONFIG_PATH': MapillaryUtils.get_auth_file(userId)
                                     },
                                     text=True)
-            NotificationsService.updateProgress(task_uuid, "created", "Uploading to Mapillary")
+            NotificationsService.updateProgress(database_session,
+                                                task_uuid,
+                                                "created",
+                                                "Uploading to Mapillary")
 
             for line in iter(prog.stdout.readline, b''):
                 if line == '':
@@ -143,12 +146,14 @@ def upload(userId: int, task_uuid: UUID, service_user: str, organization_key: st
                 catch_upload_status = re.compile(r'(.*): (\d+(?=%))')
                 upload_status_match = re.search(catch_upload_status, str(line.rstrip()))
                 if upload_status_match:
-                    NotificationsService.updateProgress(task_uuid,
+                    NotificationsService.updateProgress(database_session,
+                                                        task_uuid,
                                                         "in_progress",
                                                         upload_status_match.group(1),
                                                         int(float(upload_status_match.group(2))))
                 else:
-                    NotificationsService.updateProgress(task_uuid,
+                    NotificationsService.updateProgress(database_session,
+                                                        task_uuid,
                                                         "created",
                                                         "Processing upload...")