Merge pull request #39 from ral-facilities/add-image-thumbnails-#29

Generate and store image thumbnails when creating #29

README.md

@@ -152,4 +152,5 @@ Listed below are the environment variables supported by the application.
 | `OBJECT_STORAGE__SECRET_ACCESS_KEY`            | The secret access key to use to authenticate with the S3 object storage.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | Yes       |                                                  |
 | `OBJECT_STORAGE__BUCKET_NAME`                  | The name of the S3 bucket to use for object storage.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Yes       |                                                  |
 | `OBJECT_STORAGE__PRESIGNED_URL_EXPIRY_SECONDS` | The expiry time of presigned URLs.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | Yes       |                                                  |
-| `OBJECT_STORAGE__ATTACHMENT_MAX_SIZE_BYTES`    | The maximum file size of an attachment given in bytes.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | Yes       |                                                  |
+| `ATTACHMENT__MAX_SIZE_BYTES`                   | The maximum file size of an attachment given in bytes.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | Yes       |                                                  |
+| `IMAGE__THUMBNAIL_MAX_SIZE_BYTES`              | The maximum width/height of generated image thumbnails. The actual width and height should maintain the original aspect ratio but neither the width nor height will exceed this value.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | Yes       |                                                  |

object_storage_api/.env.example

@@ -16,4 +16,5 @@ OBJECT_STORAGE__ACCESS_KEY=root
 OBJECT_STORAGE__SECRET_ACCESS_KEY=example_password
 OBJECT_STORAGE__BUCKET_NAME=object-storage
 OBJECT_STORAGE__PRESIGNED_URL_EXPIRY_SECONDS=1800
-OBJECT_STORAGE__ATTACHMENT_MAX_SIZE_BYTES=100000000
+ATTACHMENT__MAX_SIZE_BYTES=100000000
+IMAGE__THUMBNAIL_MAX_SIZE_PIXELS=300

object_storage_api/core/config.py

@@ -46,11 +46,26 @@ class ObjectStorageConfig(BaseModel):
     secret_access_key: SecretStr
     bucket_name: SecretStr
     presigned_url_expiry_seconds: int
-    attachment_max_size_bytes: int
 
     model_config = ConfigDict(hide_input_in_errors=True)
 
 
+class AttachmentConfig(BaseModel):
+    """
+    Configuration model for attachments.
+    """
+
+    max_size_bytes: int
+
+
+class ImageConfig(BaseModel):
+    """
+    Configuration model for images.
+    """
+
+    thumbnail_max_size_pixels: int
+
+
 class Config(BaseSettings):
     """
     Overall configuration model for the application.
@@ -63,6 +78,8 @@ class Config(BaseSettings):
     api: APIConfig
     database: DatabaseConfig
     object_storage: ObjectStorageConfig
+    attachment: AttachmentConfig
+    image: ImageConfig
 
     model_config = SettingsConfigDict(
         env_file=Path(__file__).parent.parent / ".env",

object_storage_api/core/exceptions.py

@@ -44,3 +44,12 @@ class InvalidObjectIdError(DatabaseError):
 
     status_code = 422
     response_detail = "Invalid ID given"
+
+
+class InvalidImageFileError(BaseAPIException):
+    """
+    The provided image file is not valid.
+    """
+
+    status_code = 422
+    response_detail = "File given is not a valid image"

object_storage_api/core/image.py

@@ -0,0 +1,54 @@
+"""
+Module for processing images.
+"""
+
+import base64
+import logging
+from io import BytesIO
+
+from fastapi import UploadFile
+from PIL import Image, UnidentifiedImageError
+
+from object_storage_api.core.config import config
+from object_storage_api.core.exceptions import InvalidImageFileError
+
+logger = logging.getLogger()
+
+image_config = config.image
+
+
+def generate_thumbnail_base64_str(uploaded_image_file: UploadFile) -> str:
+    """
+    Generates a thumbnail from an uploaded image file.
+
+    :param uploaded_image_file: Uploaded image file.
+    :return: Base64 encoded string of the thumbnail
+    :raises: InvalidImageFileError if the given image file cannot be processed due to being invalid in some way.
+    """
+
+    logger.debug("Generating thumbnail for uploaded image file")
+
+    # Image may fail to open if the file is either not an image or is invalid in some other way
+    try:
+        pillow_image = Image.open(uploaded_image_file.file)
+    except UnidentifiedImageError as exc:
+        raise InvalidImageFileError(
+            f"The uploaded file '{uploaded_image_file.filename}' could not be opened by Pillow"
+        ) from exc
+
+    pillow_image.thumbnail(
+        (image_config.thumbnail_max_size_pixels, image_config.thumbnail_max_size_pixels),
+        # https://pillow.readthedocs.io/en/stable/handbook/concepts.html#filters-comparison-table
+        resample=Image.Resampling.BICUBIC,
+    )
+
+    # Save into memory buffer using the WebP image format (There are other options available at
+    # https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#webp)
+    memory_image_buffer = BytesIO()
+    pillow_image.save(memory_image_buffer, "webp")
+
+    # Move buffer back to start ready for reading (it will be at the end after generating the thumbnail)
+    uploaded_image_file.file.seek(0)
+
+    # Encode the thumbnail data into a UTF-8 encoded bytestring
+    return base64.b64encode(memory_image_buffer.getvalue()).decode("utf-8")

object_storage_api/models/image.py

@@ -18,6 +18,7 @@ class ImageBase(BaseModel):
     file_name: str
     # Key of the image file in object storage
     object_key: str
+    thumbnail_base64: str
     title: Optional[str] = None
     description: Optional[str] = None
 

object_storage_api/schemas/image.py

@@ -26,3 +26,4 @@ class ImageSchema(CreatedModifiedSchemaMixin, ImagePostMetadataSchema):
 
     id: str = Field(description="ID of the image")
     file_name: str = Field(description="File name of the image")
+    thumbnail_base64: str = Field(description="Thumbnail of the image as a base64 encoded byte string")

object_storage_api/services/image.py

@@ -10,6 +10,7 @@
 from fastapi import Depends, UploadFile
 
 from object_storage_api.core.exceptions import InvalidObjectIdError
+from object_storage_api.core.image import generate_thumbnail_base64_str
 from object_storage_api.models.image import ImageIn
 from object_storage_api.repositories.image import ImageRepo
 from object_storage_api.schemas.image import ImagePostMetadataSchema, ImageSchema
@@ -51,11 +52,19 @@ def create(self, image_metadata: ImagePostMetadataSchema, upload_file: UploadFil
         # before generating the presigned URL which would then require transactions
         image_id = str(ObjectId())
 
+        # Generate the thumbnail
+        thumbnail_base64 = generate_thumbnail_base64_str(upload_file)
+
+        # Upload the full size image to object storage
         object_key = self._image_store.upload(image_id, image_metadata, upload_file)
 
         try:
             image_in = ImageIn(
-                **image_metadata.model_dump(), id=image_id, object_key=object_key, file_name=upload_file.filename
+                **image_metadata.model_dump(),
+                id=image_id,
+                file_name=upload_file.filename,
+                object_key=object_key,
+                thumbnail_base64=thumbnail_base64,
             )
         except InvalidObjectIdError as exc:
             # Provide more specific detail

object_storage_api/stores/attachment.py

@@ -4,6 +4,7 @@
 
 import logging
 
+from object_storage_api.core.config import config
 from object_storage_api.core.object_store import object_storage_config, s3_client
 from object_storage_api.schemas.attachment import AttachmentPostSchema, AttachmentPostUploadInfoSchema
 
@@ -39,7 +40,7 @@ def create_presigned_post(
                 "Content-Type": "multipart/form-data"
             },
             Conditions=[
-                ["content-length-range", 0, object_storage_config.attachment_max_size_bytes],
+                ["content-length-range", 0, config.attachment.max_size_bytes],
                 ["eq", "$Content-Type", "multipart/form-data"],
             ],
             ExpiresIn=object_storage_config.presigned_url_expiry_seconds,

pyproject.toml

@@ -10,7 +10,8 @@ dependencies = [
     "fastapi[all]",
     "PyJWT",
     "pymongo",
-    "boto3"
+    "boto3",
+    "Pillow"
 ]
 
 [project.urls]

requirements.txt

@@ -22,6 +22,7 @@ markdown-it-py==3.0.0
 MarkupSafe==3.0.0
 mdurl==0.1.2
 orjson==3.10.7
+pillow==10.4.0
 pycparser==2.22
 pydantic==2.9.2
 pydantic-extra-types==2.9.0

test/e2e/test_image.py

@@ -29,17 +29,18 @@ def setup(self, test_client):
 
         self.test_client = test_client
 
-    def post_image(self, image_post_metadata_data: dict) -> Optional[str]:
+    def post_image(self, image_post_metadata_data: dict, file_name: str) -> Optional[str]:
         """
         Posts an image with the given metadata and a test image file and returns the id of the created image if
         successful.
 
         :param image_post_metadata_data: Dictionary containing the image metadata data as would be required for an
                                          `ImagePostMetadataSchema`.
+        :param file_name: File name of the image to upload (relative to the 'test/files' directory).
         :return: ID of the created image (or `None` if not successful).
         """
 
-        with open("test/e2e/files/image.jpg", mode="rb") as file:
+        with open(f"test/files/{file_name}", mode="rb") as file:
             self._post_response_image = self.test_client.post(
                 "/images", data={**image_post_metadata_data}, files={"upload_file": file}
             )
@@ -74,17 +75,23 @@ class TestCreate(CreateDSL):
     def test_create_with_only_required_values_provided(self):
         """Test creating an image with only required values provided."""
 
-        self.post_image(IMAGE_POST_METADATA_DATA_REQUIRED_VALUES_ONLY)
+        self.post_image(IMAGE_POST_METADATA_DATA_REQUIRED_VALUES_ONLY, "image.jpg")
         self.check_post_image_success(IMAGE_GET_DATA_REQUIRED_VALUES_ONLY)
 
     def test_create_with_all_values_provided(self):
         """Test creating an image with all values provided."""
 
-        self.post_image(IMAGE_POST_METADATA_DATA_ALL_VALUES)
+        self.post_image(IMAGE_POST_METADATA_DATA_ALL_VALUES, "image.jpg")
         self.check_post_image_success(IMAGE_GET_DATA_ALL_VALUES)
 
     def test_create_with_invalid_entity_id(self):
         """Test creating an image with an invalid `entity_id`."""
 
-        self.post_image({**IMAGE_POST_METADATA_DATA_REQUIRED_VALUES_ONLY, "entity_id": "invalid-id"})
+        self.post_image({**IMAGE_POST_METADATA_DATA_REQUIRED_VALUES_ONLY, "entity_id": "invalid-id"}, "image.jpg")
         self.check_post_image_failed_with_detail(422, "Invalid `entity_id` given")
+
+    def test_create_with_invalid_image_file(self):
+        """Test creating an image with an invalid image file."""
+
+        self.post_image(IMAGE_POST_METADATA_DATA_REQUIRED_VALUES_ONLY, "invalid_image.jpg")
+        self.check_post_image_failed_with_detail(422, "File given is not a valid image")

test/mock_data.py

@@ -86,6 +86,7 @@
     **CREATED_MODIFIED_GET_DATA_EXPECTED,
     "id": ANY,
     "file_name": "image.jpg",
+    "thumbnail_base64": "UklGRjQAAABXRUJQVlA4ICgAAADQAQCdASoCAAEAAUAmJYwCdAEO/gOOAAD+qlQWHDxhNJOjVlqIb8AA",
     "title": None,
     "description": None,
 }
@@ -101,11 +102,13 @@
     "id": str(ObjectId()),
     "file_name": "image.jpg",
     "object_key": "images/65df5ee771892ddcc08bd28f/65e0a624d64aaae884abaaee",
+    "thumbnail_base64": "UklGRjQAAABXRUJQVlA4ICgAAADQAQCdASoCAAEAAUAmJYwCdAEO/gOOAAD+qlQWHDxhNJOjVlqIb8AA",
 }
 
 IMAGE_GET_DATA_ALL_VALUES = {
     **IMAGE_POST_METADATA_DATA_ALL_VALUES,
     **CREATED_MODIFIED_GET_DATA_EXPECTED,
     "id": ANY,
     "file_name": "image.jpg",
+    "thumbnail_base64": "UklGRjQAAABXRUJQVlA4ICgAAADQAQCdASoCAAEAAUAmJYwCdAEO/gOOAAD+qlQWHDxhNJOjVlqIb8AA",
 }

test/pytest.ini

@@ -19,4 +19,5 @@ env =
     OBJECT_STORAGE__SECRET_ACCESS_KEY=example_password
     OBJECT_STORAGE__BUCKET_NAME=test-object-storage
     OBJECT_STORAGE__PRESIGNED_URL_EXPIRY_SECONDS=1800
-    OBJECT_STORAGE__ATTACHMENT_MAX_SIZE_BYTES=100
+    ATTACHMENT__MAX_SIZE_BYTES=100
+    IMAGE__THUMBNAIL_MAX_SIZE_PIXELS=2

test/unit/core/test_image.py

@@ -0,0 +1,32 @@
+"""
+Unit tests for image processing functions.
+"""
+
+import pytest
+from fastapi import UploadFile
+
+from object_storage_api.core.exceptions import InvalidImageFileError
+from object_storage_api.core.image import generate_thumbnail_base64_str
+
+
+class TestGenerateThumbnailBase64Str:
+    """Tests for the `generate_thumbnail_base64_str` method."""
+
+    def test_with_valid_image(self):
+        """Tests `generate_thumbnail_base64_str` with a valid image file provided."""
+
+        with open("test/files/image.jpg", "rb") as file:
+            uploaded_image_file = UploadFile(file, filename="image.jpg")
+            result = generate_thumbnail_base64_str(uploaded_image_file)
+
+        assert result == "UklGRjQAAABXRUJQVlA4ICgAAADQAQCdASoCAAEAAUAmJYwCdAEO/gOOAAD+qlQWHDxhNJOjVlqIb8AA"
+
+    def test_with_invalid_image(self):
+        """Tests `generate_thumbnail_base64_str` with an invalid image file provided."""
+
+        with open("test/files/invalid_image.jpg", "rb") as file:
+            uploaded_image_file = UploadFile(file, filename="image.jpg")
+            with pytest.raises(InvalidImageFileError) as exc:
+                generate_thumbnail_base64_str(uploaded_image_file)
+
+            assert str(exc.value) == f"The uploaded file '{uploaded_image_file.filename}' could not be opened by Pillow"

test/unit/services/test_image.py

@@ -23,6 +23,7 @@ class ImageServiceDSL:
     image_service: ImageService
 
     mock_object_id: MagicMock
+    mock_generate_thumbnail_base64_str: MagicMock
 
     @pytest.fixture(autouse=True)
     def setup(
@@ -42,7 +43,11 @@ def setup(
 
         with patch("object_storage_api.services.image.ObjectId") as object_id_mock:
             self.mock_object_id = object_id_mock
-            yield
+            with patch(
+                "object_storage_api.services.image.generate_thumbnail_base64_str"
+            ) as generate_thumbnail_base64_str_mock:
+                self.mock_generate_thumbnail_base64_str = generate_thumbnail_base64_str_mock
+                yield
 
 
 class CreateDSL(ImageServiceDSL):
@@ -70,6 +75,10 @@ def mock_create(self, image_post_metadata_data: dict) -> None:
         self._expected_image_id = ObjectId()
         self.mock_object_id.return_value = self._expected_image_id
 
+        # Thumbnail
+        expected_thumbnail_base64 = "some_thumbnail"
+        self.mock_generate_thumbnail_base64_str.return_value = expected_thumbnail_base64
+
         # Store
         expected_object_key = "some/object/key"
         self.mock_image_store.upload.return_value = expected_object_key
@@ -81,6 +90,7 @@ def mock_create(self, image_post_metadata_data: dict) -> None:
                 id=str(self._expected_image_id),
                 object_key=expected_object_key,
                 file_name=self._upload_file.filename,
+                thumbnail_base64=expected_thumbnail_base64,
             )
 
             # Repo (The contents of the returned output model does not matter here as long as its valid)
@@ -109,6 +119,7 @@ def call_create_expecting_error(self, error_type: type[BaseException]) -> None:
     def check_create_success(self) -> None:
         """Checks that a prior call to `call_create` worked as expected."""
 
+        self.mock_generate_thumbnail_base64_str.assert_called_once_with(self._upload_file)
         self.mock_image_store.upload.assert_called_once_with(
             str(self._expected_image_id), self._image_post_metadata, self._upload_file
         )
@@ -124,6 +135,7 @@ def check_create_failed_with_exception(self, message: str) -> None:
         :param message: Message of the raised exception.
         """
 
+        self.mock_generate_thumbnail_base64_str.assert_called_once_with(self._upload_file)
         self.mock_image_store.upload.assert_called_once_with(
             str(self._expected_image_id), self._image_post_metadata, self._upload_file
         )