Skip to content

Commit

Permalink
convert to using python logging and setting google handler such that …
Browse files Browse the repository at this point in the history 
…we can properly set different logging levels...........
  • Loading branch information
@GondekNP
GondekNP committed Mar 5, 2024
1 parent dfbb6f3 commit 5266a22
Show file tree
Hide file tree
Showing 14 changed files with 187 additions and 104 deletions.
133 changes: 97 additions & 36 deletions exploratory/historic_calfire_analysis.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -53,7 +53,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -74,7 +74,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 3,
"metadata": {},
"outputs": [
{
Expand All @@ -83,7 +83,7 @@
"['rxburn22_1', 'firep22_1', 'Non_RXFire_Legacy13_2']"
]
},
"execution_count": 5,
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -94,7 +94,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -109,7 +109,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 5,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -287,7 +287,7 @@
"4 249980.461716 MULTIPOLYGON (((-120.54811 38.74108, -120.5481... "
]
},
"execution_count": 7,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -298,7 +298,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 6,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -476,7 +476,7 @@
"4 4.303726e+05 MULTIPOLYGON (((-120.79859 39.31701, -120.7983... "
]
},
"execution_count": 8,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -487,7 +487,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 7,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -683,7 +683,7 @@
"4 MULTIPOLYGON (((-121.30066 39.48714, -121.3004... "
]
},
"execution_count": 9,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -694,7 +694,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 8,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1081,7 +1081,7 @@
"[1031 rows x 20 columns]"
]
},
"execution_count": 10,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -1099,21 +1099,43 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 27,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/envs/burn-severity/lib/python3.10/site-packages/google/auth/_default.py:76: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n",
" warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n",
"/opt/conda/envs/burn-severity/lib/python3.10/site-packages/google/auth/_default.py:76: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n",
" warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n",
" warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/envs/burn-severity/lib/python3.10/site-packages/google/auth/_default.py:76: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n",
" warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n",
"/opt/conda/envs/burn-severity/lib/python3.10/site-packages/google/auth/_default.py:76: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n",
" warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n"
"Traceback (most recent call last):\n",
" File \"_pydevd_bundle/pydevd_cython.pyx\", line 577, in _pydevd_bundle.pydevd_cython.PyDBFrame._handle_exception\n",
" File \"_pydevd_bundle/pydevd_cython.pyx\", line 312, in _pydevd_bundle.pydevd_cython.PyDBFrame.do_wait_suspend\n",
" File \"/opt/conda/envs/burn-severity/lib/python3.10/site-packages/debugpy/_vendored/pydevd/pydevd.py\", line 2070, in do_wait_suspend\n",
" keep_suspended = self._do_wait_suspend(thread, frame, event, arg, suspend_type, from_this_thread, frames_tracker)\n",
" File \"/opt/conda/envs/burn-severity/lib/python3.10/site-packages/debugpy/_vendored/pydevd/pydevd.py\", line 2106, in _do_wait_suspend\n",
" time.sleep(0.01)\n",
"KeyboardInterrupt\n"
]
},
{
"ename": "AttributeError",
"evalue": "'Logger' object has no attribute 'log_text'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/workspace/exploratory/historic_calfire_analysis.ipynb Cell 12\u001b[0m line \u001b[0;36m1\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdev.devpod/workspace/exploratory/historic_calfire_analysis.ipynb#X14sdnNjb2RlLXJlbW90ZQ%3D%3D?line=7'>8</a>\u001b[0m logger \u001b[39m=\u001b[39m get_cloud_logger()\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdev.devpod/workspace/exploratory/historic_calfire_analysis.ipynb#X14sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39m# Get cloud static io client\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdev.devpod/workspace/exploratory/historic_calfire_analysis.ipynb#X14sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m \u001b[39m# s3_bucket_name = os.getenv(\"S3_BUCKET_NAME\")\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdev.devpod/workspace/exploratory/historic_calfire_analysis.ipynb#X14sdnNjb2RlLXJlbW90ZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39m# cloud_static_io_client = CloudStaticIOClient(s3_bucket_name, \"s3\")\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bdev.devpod/workspace/exploratory/historic_calfire_analysis.ipynb#X14sdnNjb2RlLXJlbW90ZQ%3D%3D?line=12'>13</a>\u001b[0m cloud_static_io_client \u001b[39m=\u001b[39m get_cloud_static_io_client(logger\u001b[39m=\u001b[39;49mlogger)\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdev.devpod/workspace/exploratory/historic_calfire_analysis.ipynb#X14sdnNjb2RlLXJlbW90ZQ%3D%3D?line=14'>15</a>\u001b[0m \u001b[39m# Define our Google Cloud Project ID and Queue ID\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdev.devpod/workspace/exploratory/historic_calfire_analysis.ipynb#X14sdnNjb2RlLXJlbW90ZQ%3D%3D?line=15'>16</a>\u001b[0m project \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mdse-nps\u001b[39m\u001b[39m\"\u001b[39m\n",
"File \u001b[0;32m/workspace/exploratory/../src/routers/dependencies.py:64\u001b[0m, in \u001b[0;36mget_cloud_static_io_client\u001b[0;34m(logger)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_cloud_static_io_client\u001b[39m(logger: Logger \u001b[39m=\u001b[39m Depends(get_cloud_logger)):\n\u001b[1;32m 55\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \u001b[39m Get an instance of CloudStaticIOClient.\u001b[39;00m\n\u001b[1;32m 57\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[39m CloudStaticIOClient: An instance of CloudStaticIOClient.\u001b[39;00m\n\u001b[1;32m 63\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 64\u001b[0m logger\u001b[39m.\u001b[39;49mlog_text(\u001b[39m\"\u001b[39m\u001b[39mCreating CloudStaticIOClient\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 65\u001b[0m s3_bucket_name \u001b[39m=\u001b[39m os\u001b[39m.\u001b[39mgetenv(\u001b[39m\"\u001b[39m\u001b[39mS3_BUCKET_NAME\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 66\u001b[0m \u001b[39mreturn\u001b[39;00m CloudStaticIOClient(s3_bucket_name, \u001b[39m\"\u001b[39m\u001b[39ms3\u001b[39m\u001b[39m\"\u001b[39m)\n",
"\u001b[0;31mAttributeError\u001b[0m: 'Logger' object has no attribute 'log_text'"
]
}
],
Expand All @@ -1138,12 +1160,14 @@
"location = \"us-central1\"\n",
"\n",
"# Define the URL endpoint of our cloud function\n",
"dev_url = \"https://tf-rest-burn-severity-dev-ohi6r6qs2a-uc.a.run.app\"\n"
"# dev_url = \"https://tf-rest-burn-severity-dev-ohi6r6qs2a-uc.a.run.app\"\n",
"dev_url = \"localhost:5050\"\n",
"batch_url = dev_url + \"/api/batch/analyze-and-fetch\""
]
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -1192,8 +1216,8 @@
" \"containment_date\": str(containment_date),\n",
" \"time_buffer_days\": 30,\n",
" \"derive_boundary\": False,\n",
" \"geojson_boundary\": row_geojson,\n",
" \"boundary_source\": \"Calfire\"\n",
" \"geojson_boundary\": json.dumps(row_geojson),\n",
" \"boundary_source\": \"calfire\"\n",
" }\n",
"\n",
" # batch_analyze_and_fetch(\n",
Expand All @@ -1209,15 +1233,41 @@
" # boundary_source=post_body[\"boundary_source\"]\n",
" # )\n",
"\n",
" # Convert the dictionary to a JSON string\n",
" ### FIRST, the boundary as given by calfire\n",
" payload = json.dumps(post_body).encode()\n",
"\n",
" # Construct the request body\n",
" parent = f\"projects/{project}/locations/{location}/queues/{queue}\"\n",
" task = {\n",
" \"http_request\": {\n",
" \"http_method\": \"POST\",\n",
" \"url\": batch_url,\n",
" \"body\": payload,\n",
" \"headers\": {\"Content-type\": \"application/json\"},\n",
" }\n",
" }\n",
"\n",
" # Add the timestamp to the tasks\n",
" timestamp = timestamp_pb2.Timestamp()\n",
" timestamp.FromDatetime(datetime.datetime.utcnow() + datetime.timedelta(seconds=10))\n",
" task[\"schedule_time\"] = timestamp\n",
"\n",
" # Use the client to build and send the task\n",
" response = client.create_task(request={\"parent\": parent, \"task\": task})\n",
"\n",
" print(\"Created task {}\".format(response.name))\n",
"\n",
" ### SECOND, the boundary as derived by the simple threshold\n",
" post_body[\"derive_boundary\"] = True\n",
" post_body[\"fire_event_name\"] = post_body[\"fire_event_name\"] + \"_derived\"\n",
" payload = json.dumps(post_body).encode()\n",
"\n",
" # Construct the request body\n",
" parent = f\"projects/{project}/locations/{location}/queues/{queue}\"\n",
" task = {\n",
" \"http_request\": {\n",
" \"http_method\": \"POST\",\n",
" \"url\": dev_url,\n",
" \"url\": batch_url,\n",
" \"body\": payload,\n",
" \"headers\": {\"Content-type\": \"application/json\"},\n",
" }\n",
Expand All @@ -1243,7 +1293,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 11,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1284,7 +1334,7 @@
"0 POLYGON ((-115.00000 32.00000, -115.00000 37.0..."
]
},
"execution_count": 13,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -1303,7 +1353,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 12,
"metadata": {},
"outputs": [
{
Expand All @@ -1312,7 +1362,7 @@
"(-120.0, 32.0, -115.0, 37.0)"
]
},
"execution_count": 14,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -1323,7 +1373,7 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -1336,7 +1386,7 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 14,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -2455,7 +2505,7 @@
"[35 rows x 21 columns]"
]
},
"execution_count": 16,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -2466,13 +2516,24 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 25,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Created task projects/dse-nps/locations/us-central1/queues/tf-rest-burn-severity-queue-dev/tasks/55879882422203795601\n",
"Created task projects/dse-nps/locations/us-central1/queues/tf-rest-burn-severity-queue-dev/tasks/3815361991987033631\n"
]
}
],
"source": [
"# Iterate over DataFrame rows and create a task for each\n",
"for index, row in nps_fires.iterrows():\n",
" create_task(row)"
"# for index, row in nps_fires.iterrows():\n",
"# create_task(row)\n",
"row = nps_fires.iloc[0]\n",
"create_task(row)"
]
}
],
Expand Down
14 changes: 7 additions & 7 deletions src/routers/analyze/spectral_burn_metrics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ def main(
cloud_static_io_client,
):

logger.log_text(f"Received analyze-fire-event request for {fire_event_name}")
logger.info(f"Received analyze-fire-event request for {fire_event_name}")
derived_boundary = None

try:
Expand All @@ -102,18 +102,18 @@ def main(
postfire_date_range=date_ranges["postfire"],
from_bbox=True,
)
logger.log_text(f"Obtained imagery for {fire_event_name}")
logger.info(f"Obtained imagery for {fire_event_name}")

# calculate burn metrics
geo_client.calc_burn_metrics()
logger.log_text(f"Calculated burn metrics for {fire_event_name}")
logger.info(f"Calculated burn metrics for {fire_event_name}")

if derive_boundary:
# Derive a boundary from the imagery
# TODO [#16]: Derived boundary hardcoded for rbr / .025 threshold
# Not sure yet but we will probably want to make this configurable
geo_client.derive_boundary("rbr", 0.025)
logger.log_text(f"Derived boundary for {fire_event_name}")
logger.info(f"Derived boundary for {fire_event_name}")

# Upload the derived boundary
with tempfile.NamedTemporaryFile(suffix=".geojson", delete=False) as tmp:
Expand All @@ -138,7 +138,7 @@ def main(
postfire_date_range=date_ranges["postfire"],
derive_boundary=derive_boundary,
)
logger.log_text(f"Cogs uploaded for {fire_event_name}")
logger.info(f"Cogs uploaded for {fire_event_name}")

return JSONResponse(
status_code=200,
Expand All @@ -150,7 +150,7 @@ def main(
)

except NoFireBoundaryDetectedError as e:
logger.log_text(f"No Fire Boundary Detected for fire event {fire_event_name}")
logger.info(f"No Fire Boundary Detected for fire event {fire_event_name}")
return JSONResponse(
status_code=204,
content={
Expand All @@ -160,5 +160,5 @@ def main(

except Exception as e:
sentry_sdk.capture_exception(e)
logger.log_text(f"Error: {e}")
logger.error(f"Error: {e}")
raise HTTPException(status_code=400, detail=str(e))
Loading

0 comments on commit 5266a22

Please sign in to comment.