Logging, Exception Management Assignment

fast_api_als/database/db_helper.py

@@ -355,3 +355,9 @@ def verify_response(response_code):
 
 session = get_boto3_session()
 db_helper_session = DBHelper(session)
+
+
+def log_res_data(obj : DBHelper):
+    res = obj.table.some_operation()
+    http_code = res['ResponseMetadata']['HTTPStatusCode']
+    logging.info("Response status from table operation:"+str(http_code))

fast_api_als/main.py

@@ -1,5 +1,5 @@
 import time
-
+import logging
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fast_api_als.routers import users, submit_lead, lead_conversion, reinforcement, oem, three_pl, quicksight
@@ -25,6 +25,8 @@
     allow_headers=["*"],
 )
 
+# to ensure logs don't print to stdout and data is not lost after clearing output
+logging.basicConfig(filename = "fast_api_als.log", level = logging.INFO)
 
 @app.get("/")
 def root():
@@ -35,4 +37,4 @@ def root():
 def ping():
     start = time.process_time()
     time_taken = (time.process_time() - start) * 1000
-    return {f"Pong with response time {time_taken} ms"}
+    return {f"Ping with response time {time_taken} ms"}

fast_api_als/routers/lead_conversion.py

@@ -3,7 +3,7 @@
 import logging
 import time
 
-from fastapi import Request
+from fastapi import Request, HTTPException
 from starlette import status
 
 from fast_api_als.database.db_helper import db_helper_session
@@ -26,6 +26,15 @@ def get_quicksight_data(lead_uuid, item):
             Returns:
                 S3 data
     """
+    try:
+        assert(lead_uuid != None)
+    except AssertionError:
+        logging.error("lead_uuid should not be null")
+        raise AssertionError
+
+    if "make" not in item.keys() or "model" not in item.keys():
+        logging.error("model and make fields missing in item")
+        raise KeyError
     data = {
         "lead_hash": lead_uuid,
         "epoch_timestamp": int(time.time()),
@@ -47,15 +56,17 @@ async def submit(file: Request, token: str = Depends(get_token)):
 
     if 'lead_uuid' not in body or 'converted' not in body:
         # throw proper HTTPException
-        pass
+        logging.error("lead_uuid and converted both keys not present in response body")
+        raise HTTPException(status_code = 406, detail = "lead_uuid and converted both keys not present in response")
 
     lead_uuid = body['lead_uuid']
     converted = body['converted']
 
     oem, role = get_user_role(token)
     if role != "OEM":
         # throw proper HTTPException
-        pass
+        logging.error("Role not OEM")
+        raise HTTPException(status_code = 406, detail = "Role required to be OEM")
 
     is_updated, item = db_helper_session.update_lead_conversion(lead_uuid, oem, converted)
     if is_updated:
@@ -66,5 +77,5 @@ async def submit(file: Request, token: str = Depends(get_token)):
             "message": "Lead Conversion Status Update"
         }
     else:
-        # throw proper HTTPException
-        pass
+        logging.error("Lead conversion not updated")
+        raise HTTPException(status_code = 406, detail = "Lead conversion not updated")

fast_api_als/routers/submit_lead.py

@@ -4,7 +4,7 @@
 
 from datetime import datetime
 from fastapi import APIRouter
-from fastapi import Request, Depends
+from fastapi import Request, Depends, HTTPException
 from fastapi.security.api_key import APIKey
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
@@ -38,7 +38,8 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
 
     if not db_helper_session.verify_api_key(apikey):
         # throw proper fastpi.HTTPException
-        pass
+        logging.error("Invalid API key")
+        raise HTTPException(status_code = 403, detail = "API key invalid")
 
     body = await file.body()
     body = str(body, 'utf-8')
@@ -47,6 +48,7 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
 
     # check if xml was not parsable, if not return
     if not obj:
+        logging.error("XML could not be parsed")
         provider = db_helper_session.get_api_key_author(apikey)
         obj = {
             'provider': {
@@ -60,14 +62,16 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
             "code": "1_INVALID_XML",
             "message": "Error occured while parsing XML"
         }
-
+    logging.info("Calculating lead hash")
     lead_hash = calculate_lead_hash(obj)
+    logging.info("Lead has calculated:"+str(lead_hash))
 
     # check if adf xml is valid
     validation_check, validation_code, validation_message = check_validation(obj)
 
     #if not valid return
     if not validation_check:
+        logging.error("Validation failed")
         item, path = create_quicksight_data(obj['adf']['prospect'], lead_hash, 'REJECTED', validation_code, {})
         s3_helper_client.put_file(item, path)
         return {
@@ -95,11 +99,13 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
         for future in as_completed(futures):
             result = future.result()
             if result.get('Duplicate_Api_Call', {}).get('status', False):
+                logging.warning("Duplicate API call")
                 return {
                     "status": f"Already {result['Duplicate_Api_Call']['response']}",
                     "message": "Duplicate Api Call"
                 }
             if result.get('Duplicate_Lead', False):
+                logging.error("Duplicate lead")
                 return {
                     "status": "REJECTED",
                     "code": "12_DUPLICATE",
@@ -108,12 +114,14 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
             if "fetch_oem_data" in result:
                 fetched_oem_data = result['fetch_oem_data']
     if fetched_oem_data == {}:
+        logging.error("Empty response")
         return {
             "status": "REJECTED",
             "code": "20_OEM_DATA_NOT_FOUND",
             "message": "OEM data not found"
         }
     if 'threshold' not in fetched_oem_data:
+        logging.error("Threshold field not present")
         return {
             "status": "REJECTED",
             "code": "20_OEM_DATA_NOT_FOUND",
@@ -123,6 +131,7 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
 
     # if dealer is not available then find nearest dealer
     if not dealer_available:
+        logging.info("Current dealer not available, finding nearest dealer")
         lat, lon = get_customer_coordinate(obj['adf']['prospect']['customer']['contact']['address']['postalcode'])
         nearest_vendor = db_helper_session.fetch_nearest_dealer(oem=make,
                                                                 lat=lat,
@@ -145,13 +154,15 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
         response_body["status"] = "ACCEPTED"
         response_body["code"] = "0_ACCEPTED"
     else:
+        logging.info("Low score")
         response_body["status"] = "REJECTED"
         response_body["code"] = "16_LOW_SCORE"
 
     # verify the customer
     if response_body['status'] == 'ACCEPTED':
         contact_verified = await new_verify_phone_and_email(email, phone)
         if not contact_verified:
+            logging.info("Contact not verified")
             response_body['status'] = 'REJECTED'
             response_body['code'] = '17_FAILED_CONTACT_VALIDATION'
 
@@ -161,6 +172,7 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
     # insert the lead into ddb with oem & customer details
     # delegate inserts to sqs queue
     if response_body['status'] == 'ACCEPTED':
+        logging.info("Accepted current response")
         make_model_filter = db_helper_session.get_make_model_filter_status(make)
         message = {
             'put_file': {
@@ -199,6 +211,7 @@ async def submit(file: Request, apikey: APIKey = Depends(get_api_key)):
         res = sqs_helper_session.send_message(message)
 
     else:
+        logging.info("Rejected current response")
         message = {
             'put_file': {
                 'item': item,

fast_api_als/routers/three_pl.py

@@ -14,13 +14,28 @@
 @router.post("/reset_authkey")
 async def reset_authkey(request: Request, token: str = Depends(get_token)):
     body = await request.body()
-    body = json.loads(body)
+    try:
+        body = json.loads(body)
+    except ValueError:
+        logging.error("JSON format invalid")
+        raise ValueError
     provider, role = get_user_role(token)
+    logging.debug("Provider, role = "+provider+", "+role)
     if role != "ADMIN" and (role != "3PL"):
-        pass
+        logging.error("Role required to be admin or 3PL")
+        raise HTTPException(status_code = 400, detail = "Role required to be admin or 3PL")
     if role == "ADMIN":
+        logging.info("Role is Admin, updating provider to 3PL")
         provider = body['3pl']
-    apikey = db_helper_session.set_auth_key(username=provider)
+
+    logging.info("Trying to update API key")
+    try:
+        apikey = db_helper_session.set_auth_key(username=provider)
+        logging.debug("API key "+apikey)
+    except Exception as e:
+        logging.error("API key not set:"+str(e.message))
+        raise e
+    logging.info("Successfully updated API key")
     return {
         "status_code": HTTP_200_OK,
         "x-api-key": apikey
@@ -30,14 +45,28 @@ async def reset_authkey(request: Request, token: str = Depends(get_token)):
 @router.post("/view_authkey")
 async def view_authkey(request: Request, token: str = Depends(get_token)):
     body = await request.body()
-    body = json.loads(body)
+    try:
+        body = json.loads(body)
+    except ValueError:
+        logging.error("JSON format invalid")
+        raise ValueError
     provider, role = get_user_role(token)
-
-    if role != "ADMIN" and role != "3PL":
-        pass
+    logging.debug("Provider, role = "+provider+", "+role)
+    if role != "ADMIN" and (role != "3PL"):
+        logging.error("Role required to be admin or 3PL")
+        raise HTTPException(status_code = 400, detail = "Role required to be admin or 3PL")
     if role == "ADMIN":
+        logging.info("Role is Admin, updating provider to 3PL")
         provider = body['3pl']
-    apikey = db_helper_session.get_auth_key(username=provider)
+
+    logging.info("Trying to get API key")
+    try:
+        apikey = db_helper_session.get_auth_key(username=provider)
+        logging.debug("API key "+apikey)
+    except Exception as e:
+        logging.error("API key not found:"+str(e.message))
+        raise e
+    logging.info("Successfully found API key")
     return {
         "status_code": HTTP_200_OK,
         "x-api-key": apikey

fast_api_als/services/enrich_lead.py

@@ -1,5 +1,6 @@
 import calendar
 import time
+import json
 import logging
 from dateutil import parser
 from fast_api_als.database.db_helper import db_helper_session
@@ -12,4 +13,10 @@
 
 
 def get_enriched_lead_json(adf_json: dict) -> dict:
+    if 'adf' not in adf_json.keys():
+        raise KeyError
+    try:
+        json.loads(adf_json)
+    except ValueError:
+        raise ValueError
     pass

fast_api_als/services/verify_phone_and_email.py

@@ -15,6 +15,7 @@
 
 async def call_validation_service(url: str, topic: str, value: str, data: dict) -> None:  # 2
     if value == '':
+        logging.info("Found empty value")
         return
     async with httpx.AsyncClient() as client:  # 3
         response = await client.get(url)

fast_api_als/utils/adf.py

@@ -27,10 +27,14 @@ def process_before_validating(input_json):
 def validate_iso8601(requestdate):
     try:
         if match_iso8601(requestdate) is not None:
+            logging.info(f"Match found for {requestdate}")
             return True
-    except:
-        pass
-    return False
+        else:
+            logging.error(f"No match found for {requestdate}")
+            return False
+    except Exception as e:
+        logging.error("ISO validation failed:"+str(e.message))
+        return False
 
 
 def is_nan(x):
@@ -39,8 +43,13 @@ def is_nan(x):
 
 def parse_xml(adf_xml):
     # use exception handling
-    obj = xmltodict.parse(adf_xml)
-    return obj
+    try:
+        obj = xmltodict.parse(adf_xml)
+        logging.info("XML parsed successfully")
+        return obj
+    except Exception as e:
+        logging.error("XML not parsed successfully:"+str(e.message))
+        return None
 
 
 def validate_adf_values(input_json):
@@ -60,14 +69,17 @@ def validate_adf_values(input_json):
 
     if not first_name or not last_name:
         return {"status": "REJECTED", "code": "6_MISSING_FIELD", "message": "name is incomplete"}
+    logging.info("First Name and Last Name field present")
 
     if not email and not phone:
         return {"status": "REJECTED", "code": "6_MISSING_FIELD", "message": "either phone or email is required"}
+    logging.info("Email and Phone field present")
 
     # zipcode validation
     res = zipcode_search.by_zipcode(zipcode)
     if not res:
         return {"status": "REJECTED", "code": "4_INVALID_ZIP", "message": "Invalid Postal Code"}
+    logging.info("Zip code validated")
 
     # check for TCPA Consent
     tcpa_consent = False
@@ -76,11 +88,11 @@ def validate_adf_values(input_json):
             tcpa_consent = True
     if not email and not tcpa_consent:
         return {"status": "REJECTED", "code": "7_NO_CONSENT", "message": "Contact Method missing TCPA consent"}
-
+    logging.info("TCPA Consent present")
+
     # request date in ISO8601 format
     if not validate_iso8601(input_json['requestdate']):
         return {"status": "REJECTED", "code": "3_INVALID_FIELD", "message": "Invalid DateTime"}
-
     return {"status": "OK"}
 
 
@@ -94,8 +106,10 @@ def check_validation(input_json):
         )
         response = validate_adf_values(input_json)
         if response['status'] == "REJECTED":
+            logging.error("Response status REJECTED")
             return False, response['code'], response['message']
+        logging.info("Response status OK")
         return True, "input validated", "validation_ok"
     except Exception as e:
-        logger.error(f"Validation failed: {e.message}")
+        logging.error(f"Validation failed: {e.message}")
         return False, "6_MISSING_FIELD", e.message