From 2d05a15640a005254b2ee023bbe387434a918f64 Mon Sep 17 00:00:00 2001 From: Anuj Gupta <84966248+Anuj-Gupta4@users.noreply.github.com> Date: Mon, 18 Nov 2024 22:53:32 +0545 Subject: [PATCH] refactor(logging): enhance error logging with stack information across multiple modules (#1887) * refactor(logging): enhance error logging with stack information across multiple modules * small-fix: improve exception format --- src/backend/app/auth/auth_routes.py | 2 +- src/backend/app/auth/osm.py | 5 +++-- src/backend/app/auth/roles.py | 6 ++++-- src/backend/app/central/central_crud.py | 18 +++++++++--------- src/backend/app/central/central_routes.py | 2 +- src/backend/app/projects/project_crud.py | 8 +++++--- src/backend/app/s3.py | 14 +++++++++----- 7 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/backend/app/auth/auth_routes.py b/src/backend/app/auth/auth_routes.py index 122afcc2a6..0482e2b4e4 100644 --- a/src/backend/app/auth/auth_routes.py +++ b/src/backend/app/auth/auth_routes.py @@ -221,7 +221,7 @@ async def get_or_create_user( except Exception as e: await db.rollback() - log.error(f"Exception occurred: {e}") + log.exception(f"Exception occurred: {e}", stack_info=True) if 'duplicate key value violates unique constraint "users_username_key"' in str( e ): diff --git a/src/backend/app/auth/osm.py b/src/backend/app/auth/osm.py index 8122a1093e..6f7d5a7866 100644 --- a/src/backend/app/auth/osm.py +++ b/src/backend/app/auth/osm.py @@ -72,8 +72,9 @@ async def login_required( try: token_data = verify_token(access_token) except ValueError as e: - log.error(e) - log.error("Failed to deserialise access token") + log.exception( + f"Failed to deserialise access token. Error: {e}", stack_info=True + ) raise HTTPException( status_code=HTTPStatus.UNAUTHORIZED, detail="Access token not valid", diff --git a/src/backend/app/auth/roles.py b/src/backend/app/auth/roles.py index 74099db1eb..c4b4e7a951 100644 --- a/src/backend/app/auth/roles.py +++ b/src/backend/app/auth/roles.py @@ -46,8 +46,10 @@ async def get_uid(user_data: AuthUser | DbUser) -> int: return user_id except Exception as e: - log.error(e) - log.error(f"Failed to get user id from auth object: {user_data}") + log.exception( + f"Failed to get user id from auth object: {user_data}. Error: {e}", + stack_info=True, + ) raise HTTPException( status_code=HTTPStatus.UNAUTHORIZED, detail="Auth failed. No user id present", diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 7d28631457..2ff1f07fc9 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -69,7 +69,7 @@ def get_odk_project(odk_central: Optional[central_schemas.ODKCentralDecrypted] = ), ) from e except Exception as e: - log.exception(e) + log.exception(f"Error: {e}", stack_info=True) raise HTTPException( HTTPStatus.INTERNAL_SERVER_ERROR, detail=f"Error creating project on ODK Central: {e}", @@ -88,7 +88,7 @@ def get_odk_form(odk_central: central_schemas.ODKCentralDecrypted): log.debug(f"Connecting to ODKCentral: url={url} user={user}") form = OdkForm(url, user, pw) except Exception as e: - log.error(e) + log.exception(f"Error: {e}", stack_info=True) raise HTTPException( HTTPStatus.INTERNAL_SERVER_ERROR, detail=f"Error creating project on ODK Central: {e}", @@ -114,7 +114,7 @@ def get_odk_app_user(odk_central: Optional[central_schemas.ODKCentralDecrypted] log.debug(f"Connecting to ODKCentral: url={url} user={user}") form = OdkAppUser(url, user, pw) except Exception as e: - log.error(e) + log.exception(f"Error: {e}", stack_info=True) raise HTTPException( HTTPStatus.INTERNAL_SERVER_ERROR, detail=f"Error creating project on ODK Central: {e}", @@ -156,7 +156,7 @@ def create_odk_project( log.info(f"Project {name} available on the ODK Central server.") return result except Exception as e: - log.error(e) + log.exception(f"Error: {e}", stack_info=True) raise HTTPException( HTTPStatus.INTERNAL_SERVER_ERROR, detail=f"Error creating project on ODK Central: {e}", @@ -206,7 +206,7 @@ def create_odk_xform( try: xform = get_odk_form(odk_credentials) except Exception as e: - log.error(e) + log.exception(f"Error: {e}", stack_info=True) raise HTTPException( HTTPStatus.INTERNAL_SERVER_ERROR, detail={"message": "Connection failed to odk central"}, @@ -278,7 +278,7 @@ async def get_form_list(db: Connection) -> list: try: return await DbXLSForm.all(db) except Exception as e: - log.error(e) + log.exception(f"Error: {e}", stack_info=True) raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=str(e), @@ -302,7 +302,7 @@ async def read_and_test_xform(input_data: BytesIO) -> None: # NOTE pyxform.xls2xform.convert returns a ConvertResult object return BytesIO(xform_convert(input_data).xform.encode("utf-8")) except Exception as e: - log.error(e) + log.exception(f"Error: {e}", stack_info=True) msg = f"XLSForm is invalid: {str(e)}" raise HTTPException( status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=msg @@ -688,8 +688,8 @@ async def get_entities_data( url_params=f"$select=__id{',' if fields else ''} {fields}", ) except Exception as e: + log.exception(f"Error: {e}", stack_info=True) msg = f"Getting entity data failed for ODK project ({odk_id})" - log.error(msg) raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=msg, @@ -928,7 +928,7 @@ async def get_appuser_token( return f"{odk_url}/v1/key/{appuser_token}/projects/{project_odk_id}" except Exception as e: - log.error(f"An error occurred: {str(e)}") + log.exception(f"An error occurred: {str(e)}", stack_info=True) raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="An error occurred while creating the app user token.", diff --git a/src/backend/app/central/central_routes.py b/src/backend/app/central/central_routes.py index 63e98066ea..e352161659 100644 --- a/src/backend/app/central/central_routes.py +++ b/src/backend/app/central/central_routes.py @@ -106,7 +106,7 @@ async def refresh_appuser_token( ) except Exception as e: - log.error(e) + log.exception(f"Error: {e}", stack_info=True) msg = f"failed to refresh the appuser token for project {project_id}" log.error(msg) raise HTTPException( diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 3938d3fc4e..98bbe6fffd 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -226,8 +226,10 @@ async def read_and_insert_xlsforms(db: Connection, directory: str) -> None: log.info(f"XLSForm for '{category}' inserted/updated in the database") except Exception as e: - log.error( - f"Failed to insert or update {category} in the database. Error: {e}" + log.exception( + f"Failed to insert or update {category} in the database. " + f"Error: {e}", + stack_info=True, ) # Determine the forms that need to be deleted (those in the DB but @@ -777,7 +779,7 @@ def generate_project_basemap( except Exception as e: log.debug(str(format_exc())) - log.exception(str(e)) + log.exception(f"Error: {e}", stack_info=True) log.error(f"Tiles generation process failed for project id {project_id}") if new_basemap: diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index 7cf6441049..a280f385f1 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -195,8 +195,11 @@ def copy_obj_bucket_to_bucket( ) except Exception as e: - log.exception(e) - log.error(f"Failed to copy object {source_path} to new bucket: {dest_bucket}") + log.exception( + f"Failed to copy object {source_path} to new bucket {dest_bucket}. " + f"Error: {e}", + stack_info=True, + ) return False return True @@ -237,9 +240,10 @@ async def delete_all_objs_under_prefix(bucket_name: str, s3_path: str) -> bool: return False except Exception as e: - log.exception(e) - log.error( - f"Failed to delete bucket ({bucket_name}) files under path: {s3_path}" + log.exception( + f"Failed to delete bucket ({bucket_name}) files under path: {s3_path}. " + f"Error: {e}", + stack_info=True, ) return False