feat(cameras): add a route to update the last known image of a camera (…

…#371)

* feat(cameras): add route to update last image

* test(cameras): update backend tests

* feat(client): add update_last_image method

* docs(uml): add new camera column

* style(mypy): remove warnings

* docs(contributing): update UML

* docs(contributing): update contirbuting

* docs(contributing): remove iframe

* fix(models): fix table references

* test(confest): fix DB initialization

* test(cameras): fix test cases

* style(tests): fix lint

CONTRIBUTING.md

@@ -13,12 +13,17 @@ The back-end core feature is to interact with the metadata tables. For the servi
 
 - Users: stores the hashed credentials and access level for users.
 - Cameras: stores the camera metadata.
+- Organizations: scope the access to the API.
 
 #### Core worklow tables
 
 - Detection: association of a picture and a camera.
 
-![UML diagram](https://github.com/user-attachments/assets/d0160a58-b494-4b81-bef0-b1a9f483be3e)
+#### Client-related tables
+
+- Webhook: stores the webhook URLs.
+
+_The UML is versioned at [`scripts/dbdiagram.txt`](https://github.com/pyronear/pyro-api/blob/main/scripts/dbdiagram.txt) and the UML diagram is available on [DBDiagram](https://dbdiagram.io/d/Pyronear-UML-665a15d0b65d933879357b58)._
 
 ### What is the full detection workflow through the API
 

client/pyroclient/client.py

@@ -23,6 +23,7 @@
     # CAMERAS
     #################
     "cameras-heartbeat": "/cameras/heartbeat",
+    "cameras-image": "/cameras/image",
     "cameras-fetch": "/cameras",
     #################
     # DETECTIONS
@@ -103,6 +104,22 @@ def headers(self) -> Dict[str, str]:
         return {"Authorization": f"Bearer {self.token}"}
 
     # CAMERAS
+    def fetch_cameras(self) -> Response:
+        """List the cameras accessible to the authenticated user
+
+        >>> from pyroclient import client
+        >>> api_client = Client("MY_USER_TOKEN")
+        >>> response = api_client.fetch_cameras()
+
+        Returns:
+            HTTP response
+        """
+        return requests.get(
+            self.routes["cameras-fetch"],
+            headers=self.headers,
+            timeout=self.timeout,
+        )
+
     def heartbeat(self) -> Response:
         """Update the last ping of the camera
 
@@ -115,6 +132,24 @@ def heartbeat(self) -> Response:
         """
         return requests.patch(self.routes["cameras-heartbeat"], headers=self.headers, timeout=self.timeout)
 
+    def update_last_image(self, media: bytes) -> Response:
+        """Update the last image of the camera
+
+        >>> from pyroclient import Client
+        >>> api_client = Client("MY_CAM_TOKEN")
+        >>> with open("path/to/my/file.ext", "rb") as f: data = f.read()
+        >>> response = api_client.update_last_image(data)
+
+        Returns:
+            HTTP response containing the update device info
+        """
+        return requests.patch(
+            self.routes["cameras-image"],
+            headers=self.headers,
+            files={"file": ("logo.png", media, "image/png")},
+            timeout=self.timeout,
+        )
+
     # DETECTIONS
     def create_detection(
         self,
@@ -171,22 +206,6 @@ def label_detection(self, detection_id: int, is_wildfire: bool) -> Response:
             timeout=self.timeout,
         )
 
-    def fetch_cameras(self) -> Response:
-        """List the cameras accessible to the authenticated user
-
-        >>> from pyroclient import client
-        >>> api_client = Client("MY_USER_TOKEN")
-        >>> response = api_client.fetch_cameras()
-
-        Returns:
-            HTTP response
-        """
-        return requests.get(
-            self.routes["cameras-fetch"],
-            headers=self.headers,
-            timeout=self.timeout,
-        )
-
     def get_detection_url(self, detection_id: int) -> Response:
         """Retrieve the URL of the media linked to a detection
 

client/tests/test_client.py

@@ -26,7 +26,13 @@ def test_client_constructor(token, host, timeout, expected_error):
 @pytest.fixture(scope="session")
 def test_cam_workflow(cam_token, mock_img):
     cam_client = Client(cam_token, "http://localhost:5050", timeout=10)
-    assert cam_client.heartbeat().status_code == 200
+    response = cam_client.heartbeat()
+    assert response.status_code == 200
+    # Check that last_image gets changed
+    assert response.json()["last_image"] is None
+    response = cam_client.update_last_image(mock_img)
+    assert response.status_code == 200, response.__dict__
+    assert isinstance(response.json()["last_image"], str)
     # Check that adding bboxes works
     with pytest.raises(ValueError, match="bboxes must be a non-empty list of tuples"):
         cam_client.create_detection(mock_img, 123.2, None)

pyproject.toml

@@ -158,5 +158,7 @@ module = [
     "botocore.*",
     "databases",
     "posthog",
+    "prometheus_fastapi_instrumentator",
+    "pydantic_settings",
 ]
 ignore_missing_imports = true

scripts/dbdiagram.txt

@@ -8,8 +8,8 @@ Table "User" as U {
   "id" int [not null]
   "organization_id" int [ref: > O.id, not null]
   "role" userrole [not null]
-  "login" str [not null]
-  "hashed_password" str [not null]
+  "login" text [not null]
+  "hashed_password" text [not null]
   "created_at" timestamp [not null]
   Indexes {
     (id, login) [pk]
@@ -19,14 +19,15 @@ Table "User" as U {
 Table "Camera" as C {
   "id" int [not null]
   "organization_id" int [ref: > O.id, not null]
-  "name" str [not null]
+  "name" text [not null]
   "angle_of_view" float [not null]
   "elevation" float [not null]
   "lat" float [not null]
   "lon" float [not null]
   "is_trustable" bool [not null]
   "created_at" timestamp [not null]
   "last_active_at" timestamp
+  "last_image" text
   Indexes {
     (id) [pk]
   }
@@ -36,8 +37,8 @@ Table "Detection" as D {
   "id" int [not null]
   "camera_id" int [ref: > C.id, not null]
   "azimuth" float [not null]
-  "bucket_key" str [not null]
-  "bboxes" str [not null]
+  "bucket_key" text [not null]
+  "bboxes" text [not null]
   "is_wildfire" bool
   "created_at" timestamp [not null]
   "updated_at" timestamp [not null]
@@ -48,7 +49,7 @@ Table "Detection" as D {
 
 Table "Organization" as O {
   "id" int [not null]
-  "name" str [not null]
+  "name" text [not null]
   Indexes {
     (id) [pk]
   }
@@ -57,7 +58,7 @@ Table "Organization" as O {
 
 Table "Webhook" as W {
   "id" int [not null]
-  "url" str [not null]
+  "url" text [not null]
   Indexes {
     (id) [pk]
   }

src/app/api/api_v1/endpoints/cameras.py

@@ -6,15 +6,16 @@
 from datetime import datetime
 from typing import List, cast
 
-from fastapi import APIRouter, Depends, HTTPException, Path, Security, status
+from fastapi import APIRouter, Depends, File, HTTPException, Path, Security, UploadFile, status
 
 from app.api.dependencies import get_camera_crud, get_jwt
 from app.core.config import settings
 from app.core.security import create_access_token
 from app.crud import CameraCRUD
 from app.models import Camera, Role, UserRole
-from app.schemas.cameras import CameraCreate, LastActive
+from app.schemas.cameras import CameraCreate, LastActive, LastImage
 from app.schemas.login import Token, TokenPayload
+from app.services.storage import s3_service, upload_file
 from app.services.telemetry import telemetry_client
 
 router = APIRouter()
@@ -62,10 +63,26 @@ async def heartbeat(
     cameras: CameraCRUD = Depends(get_camera_crud),
     token_payload: TokenPayload = Security(get_jwt, scopes=[Role.CAMERA]),
 ) -> Camera:
-    # telemetry_client.capture(f"camera|{token_payload.sub}", event="cameras-heartbeat", properties={"camera_id": camera_id})
+    # telemetry_client.capture(f"camera|{token_payload.sub}", event="cameras-heartbeat")
     return await cameras.update(token_payload.sub, LastActive(last_active_at=datetime.utcnow()))
 
 
+@router.patch("/image", status_code=status.HTTP_200_OK, summary="Update last image of a camera")
+async def update_image(
+    file: UploadFile = File(..., alias="file"),
+    cameras: CameraCRUD = Depends(get_camera_crud),
+    token_payload: TokenPayload = Security(get_jwt, scopes=[Role.CAMERA]),
+) -> Camera:
+    # telemetry_client.capture(f"camera|{token_payload.sub}", event="cameras-image")
+    bucket_key = await upload_file(file, token_payload.organization_id, token_payload.sub)
+    # If the upload succeeds, delete the previous image
+    cam = cast(Camera, await cameras.get(token_payload.sub, strict=True))
+    if isinstance(cam.last_image, str):
+        s3_service.get_bucket(s3_service.resolve_bucket_name(token_payload.organization_id)).delete_file(cam.last_image)
+    # Update the DB entry
+    return await cameras.update(token_payload.sub, LastImage(last_image=bucket_key, last_active_at=datetime.utcnow()))
+
+
 @router.post("/{camera_id}/token", status_code=status.HTTP_200_OK, summary="Request an access token for the camera")
 async def create_camera_token(
     camera_id: int = Path(..., gt=0),

src/app/api/api_v1/endpoints/detections.py

@@ -3,13 +3,9 @@
 # This program is licensed under the Apache License 2.0.
 # See LICENSE or go to <https://www.apache.org/licenses/LICENSE-2.0> for full license details.
 
-import hashlib
-import logging
 from datetime import datetime
-from mimetypes import guess_extension
 from typing import List, cast
 
-import magic
 from fastapi import (
     APIRouter,
     BackgroundTasks,
@@ -37,7 +33,7 @@
     DetectionWithUrl,
 )
 from app.schemas.login import TokenPayload
-from app.services.storage import s3_service
+from app.services.storage import s3_service, upload_file
 from app.services.telemetry import telemetry_client
 
 router = APIRouter()
@@ -69,37 +65,7 @@ async def create_detection(
         )
 
     # Upload media
-    # Concatenate the first 8 chars (to avoid system interactions issues) of SHA256 hash with file extension
-    sha_hash = hashlib.sha256(file.file.read()).hexdigest()
-    await file.seek(0)
-    # Use MD5 to verify upload
-    md5_hash = hashlib.md5(file.file.read()).hexdigest()  # noqa S324
-    await file.seek(0)
-    # guess_extension will return none if this fails
-    extension = guess_extension(magic.from_buffer(file.file.read(), mime=True)) or ""
-    # Concatenate timestamp & hash
-    bucket_key = f"{token_payload.sub}-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}-{sha_hash[:8]}{extension}"
-    # Reset byte position of the file (cf. https://fastapi.tiangolo.com/tutorial/request-files/#uploadfile)
-    await file.seek(0)
-    bucket_name = s3_service.resolve_bucket_name(token_payload.organization_id)
-    bucket = s3_service.get_bucket(bucket_name)
-    # Upload the file
-    if not bucket.upload_file(bucket_key, file.file):  # type: ignore[arg-type]
-        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed upload")
-    logging.info(f"File uploaded to bucket {bucket_name} with key {bucket_key}.")
-
-    # Data integrity check
-    file_meta = bucket.get_file_metadata(bucket_key)
-    # Corrupted file
-    if md5_hash != file_meta["ETag"].replace('"', ""):
-        # Delete the corrupted upload
-        bucket.delete_file(bucket_key)
-        # Raise the exception
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail="Data was corrupted during upload",
-        )
-    # Format the string
+    bucket_key = await upload_file(file, token_payload.organization_id, token_payload.sub)
     det = await detections.create(
         DetectionCreate(camera_id=token_payload.sub, bucket_key=bucket_key, azimuth=azimuth, bboxes=bboxes)
     )

src/app/models.py

@@ -28,8 +28,9 @@ class Role(str, Enum):
 
 
 class User(SQLModel, table=True):
+    __tablename__ = "users"
     id: int = Field(None, primary_key=True)
-    organization_id: int = Field(..., foreign_key="organization.id", nullable=False)
+    organization_id: int = Field(..., foreign_key="organizations.id", nullable=False)
     role: UserRole = Field(UserRole.USER, nullable=False)
     # Allow sign-up/in via login + password
     login: str = Field(..., index=True, unique=True, min_length=2, max_length=50, nullable=False)
@@ -38,21 +39,24 @@ class User(SQLModel, table=True):
 
 
 class Camera(SQLModel, table=True):
+    __tablename__ = "cameras"
     id: int = Field(None, primary_key=True)
-    organization_id: int = Field(..., foreign_key="organization.id", nullable=False)
+    organization_id: int = Field(..., foreign_key="organizations.id", nullable=False)
     name: str = Field(..., min_length=5, max_length=100, nullable=False, unique=True)
     angle_of_view: float = Field(..., gt=0, le=360, nullable=False)
     elevation: float = Field(..., gt=0, lt=10000, nullable=False)
     lat: float = Field(..., gt=-90, lt=90)
     lon: float = Field(..., gt=-180, lt=180)
     is_trustable: bool = True
     last_active_at: Union[datetime, None] = None
+    last_image: Union[str, None] = None
     created_at: datetime = Field(default_factory=datetime.utcnow, nullable=False)
 
 
 class Detection(SQLModel, table=True):
+    __tablename__ = "detections"
     id: int = Field(None, primary_key=True)
-    camera_id: int = Field(..., foreign_key="camera.id", nullable=False)
+    camera_id: int = Field(..., foreign_key="cameras.id", nullable=False)
     azimuth: float = Field(..., gt=0, lt=360)
     bucket_key: str
     bboxes: str = Field(..., min_length=2, max_length=settings.MAX_BBOX_STR_LENGTH, nullable=False)
@@ -62,10 +66,12 @@ class Detection(SQLModel, table=True):
 
 
 class Organization(SQLModel, table=True):
+    __tablename__ = "organizations"
     id: int = Field(None, primary_key=True)
     name: str = Field(..., min_length=5, max_length=100, nullable=False, unique=True)
 
 
 class Webhook(SQLModel, table=True):
+    __tablename__ = "webhooks"
     id: int = Field(None, primary_key=True)
     url: str = Field(..., nullable=False, unique=True)

src/app/schemas/cameras.py

@@ -17,6 +17,10 @@ class LastActive(BaseModel):
     last_active_at: datetime = Field(default_factory=datetime.utcnow)
 
 
+class LastImage(LastActive):
+    last_image: str
+
+
 class CameraCreate(BaseModel):
     organization_id: int = Field(..., gt=0)
     name: str = Field(