Merge pull request #9 from pablerass/feature/solve-tracing-multiproce…

…ssing

Make tracing work with multiprocessing

cartuli/__main__.py

@@ -6,7 +6,7 @@
 import re
 import sys
 
-from carpeta import Tracer, ImageHandler, trace_output
+from carpeta import ProcessTracer, ImageHandler, trace_output
 from pathlib import Path
 
 from .definition import Definition
@@ -33,7 +33,7 @@ def main(args=None):
     """Execute main package command line functionality."""
     args = parse_args()
 
-    tracer = Tracer()
+    tracer = ProcessTracer()
 
     # Logging
     if args.verbose < 3:
@@ -50,7 +50,7 @@ def main(args=None):
     if args.trace_output:
         processing_logger = logging.getLogger('cartuli.processing')
         processing_logger.setLevel(logging.DEBUG)
-        processing_handler = ImageHandler(tracer)
+        processing_handler = ImageHandler(tracer.remote_tracer)
         processing_handler.setLevel(logging.DEBUG)
         processing_logger.addHandler(processing_handler)
 
@@ -79,6 +79,8 @@ def main(args=None):
     if tracer:
         trace_output(tracer, args.trace_output)
 
+    tracer.wait_and_stop()
+
     return 0
 
 

cartuli/card.py

@@ -1,6 +1,7 @@
 """Card module."""
 from pathlib import Path
 
+from carpeta import register_id_extractor
 from PIL import Image
 
 from .measure import Size
@@ -41,7 +42,7 @@ def image(self) -> Image.Image:
 
     @property
     def image_path(self) -> Path | None:
-        return self.__image
+        return self.__image_path
 
     @property
     def size(self) -> Size:
@@ -85,6 +86,10 @@ def __str__(self) -> str:
         return super().__str__()
 
 
+# TUNE: Not sure if this is the best place to place this...
+register_id_extractor(CardImage, lambda x: x.name)
+
+
 class Card:
     """One or two sided card representation."""
 

cartuli/definition.py

@@ -18,6 +18,8 @@
 from .sheet import Sheet
 
 
+_CONCURRENT_PROCESSES = cpu_count() - 1
+
 CardsFilter = Callable[[Path], bool]
 
 
@@ -83,7 +85,7 @@ def _load_images(self, images_definition: dict, size: Size, deck_name: str, side
         image_filter = images_definition.get('filter', '')
         image_files = sorted(glob(images_definition['images']))
         logger.debug(f"Found {len(image_files)} {side} images for '{deck_name}' deck")
-        with Pool(processes=cpu_count() - 1) as pool:
+        with Pool(processes=_CONCURRENT_PROCESSES) as pool:
             images = pool.map(
                 self.filters[image_filter].apply,
                 (CardImage(
@@ -111,6 +113,7 @@ def _load_deck(self, definition: dict, name: str) -> Deck:
                 if len(front_images) != len(back_images):
                     raise DefinitionError(f"The number of front ({len(front_images)}) and "
                                           f"back ({len(back_images)}) images must be the same")
+                # TODO Allow all back images to be filtered without errors
                 cards = [Card(front_image, back_image) for front_image, back_image in zip(front_images, back_images)]
             else:
                 cards = [Card(image) for image in front_images]
@@ -123,15 +126,18 @@ def _load_deck(self, definition: dict, name: str) -> Deck:
         default_back = None
         if 'default_back' in definition:
             default_back_file = definition['default_back']['image']
-            default_back_filter = definition['default_back'].get('filter', '')
-            default_back = self.filters[default_back_filter].apply(
-                CardImage(
-                    default_back_file,
-                    size=size,
-                    bleed=from_str(definition['default_back'].get('bleed', str(CardImage.DEFAULT_BLEED))),
-                    name=Path(default_back_file).stem
+            if self.__cards_filter(default_back_file):
+                default_back_filter = definition['default_back'].get('filter', '')
+                default_back = self.filters[default_back_filter].apply(
+                    CardImage(
+                        default_back_file,
+                        size=size,
+                        bleed=from_str(definition['default_back'].get('bleed', str(CardImage.DEFAULT_BLEED))),
+                        name=Path(default_back_file).stem
+                    )
                 )
-            )
+            else:
+                logger.debug(f"Default back image '{default_back_file}' filtered for '{name}' deck")
 
         return Deck(cards, name=name, default_back=default_back, size=size)
 

cartuli/filters.py

@@ -1,6 +1,7 @@
 import logging
 
 from abc import ABC, abstractmethod
+from carpeta import Traceable, extract_id
 from dataclasses import dataclass
 
 from .card import CardImage
@@ -47,7 +48,7 @@ def apply(self, card_image: CardImage) -> CardImage:
 
         return CardImage(
             inpaint(
-                card_image.image,
+                Traceable(card_image.image, extract_id(card_image)),
                 inpaint_size=card_image.resolution * self.inpaint_size,
                 image_crop=card_image.resolution * self.image_crop,
                 corner_radius=card_image.resolution * self.corner_radius,
@@ -68,7 +69,10 @@ def apply(self, card_image: CardImage) -> CardImage:
         logger.debug(f'Applying to {card_image}')
 
         return CardImage(
-            straighten(card_image.image, self.outliers_iqr_scale),
+            straighten(
+                Traceable(card_image.image, extract_id(card_image)),
+                self.outliers_iqr_scale
+            ),
             size=card_image.size,
             bleed=card_image.bleed,
             name=card_image.name
@@ -84,7 +88,10 @@ def apply(self, card_image: CardImage) -> CardImage:
         logger.debug(f'Applying to {card_image}')
 
         return CardImage(
-            crop(card_image.image, size=card_image.resolution * self.size),
+            crop(
+                Traceable(card_image.image, extract_id(card_image)),
+                size=card_image.resolution * self.size
+            ).value,    # Traceable values are returned as traceable in crop, this solution is crap
             size=card_image.size,
             bleed=card_image.bleed,
             name=card_image.name

cartuli/processing.py

@@ -2,6 +2,7 @@
 import logging
 import numpy as np
 
+from carpeta import extract_id
 from PIL import Image, ImageOps, ImageDraw
 
 from .measure import Size
@@ -22,11 +23,14 @@ def _to_size(value: Size | float | int) -> Size:
 # def scale(image: Image.Image, /, ...) -> Image.Image:
 
 
-def inpaint(image: Image.Image, /, inpaint_size: Size | float | int, image_crop: Size | float | int = 0,
-            corner_radius: Size | float | int = 0, inpaint_radius: float | int = 12) -> Image.Image:
+def inpaint(image: Image.Image, /, inpaint_size: Size | float | int,
+            image_crop: Size | float | int = 0, corner_radius: Size | float | int = 0,
+            inpaint_radius: float | int = 12) -> Image.Image:
     logger = logging.getLogger('cartuli.processing')
-    # TODO: Add args dict and/or start trace extra
-    logger.debug(f"Start image {image} inpaint", extra={'trace': image})
+
+    trace_id = extract_id(image)
+
+    logger.debug(f"Start image {image} inpaint", extra={'trace': image, 'trace_id': trace_id})
 
     inpaint_size = _to_size(inpaint_size)
     image_crop = _to_size(image_crop)
@@ -39,7 +43,7 @@ def inpaint(image: Image.Image, /, inpaint_size: Size | float | int, image_crop:
         .crop((expand_crop.width, expand_crop.height,
                image.size[0] + expand_size*2 - expand_crop.width,
                image.size[1] + expand_size*2 - expand_crop.height))
-    logger.debug(f"Expand {image} image", extra={'trace': expanded_image})
+    logger.debug(f"Expand {image} image", extra={'trace': expanded_image, 'trace_id': trace_id})
 
     mask_image = Image.new('L', (image.size[0] + inpaint_size.width*2,
                                  image.size[1] + inpaint_size.height*2), color='white')
@@ -50,13 +54,13 @@ def inpaint(image: Image.Image, /, inpaint_size: Size | float | int, image_crop:
          mask_image.size[1] - inpaint_size.height - image_crop.height),
         fill='black', width=0, radius=max(corner_radius))
     # TUNE: Find a way to round with different vertical and horizontal values
-    logger.debug(f"Mask {image} image for inpainting", extra={'trace': mask_image})
+    logger.debug(f"Mask {image} image for inpainting", extra={'trace': mask_image, 'trace_id': trace_id})
 
     inpaint_image_cv = cv.inpaint(
         cv.cvtColor(np.array(expanded_image), cv.COLOR_RGB2BGR),
         np.array(mask_image), int(inpaint_radius), cv.INPAINT_NS)
     inpainted_image = Image.fromarray(cv.cvtColor(inpaint_image_cv, cv.COLOR_BGR2RGB))
-    logger.debug(f"Inpaint {image} image", extra={'trace': inpainted_image})
+    logger.debug(f"Inpaint {image} image", extra={'trace': inpainted_image, 'trace_id': trace_id})
 
     return inpainted_image
 
@@ -94,8 +98,10 @@ def _discard_outliers(data: np.ndarray | list, iqr_scale: float = 1.5) -> np.nda
 
 def straighten(image: Image.Image, /, outliers_iqr_scale: float = 0.01) -> Image.Image:
     logger = logging.getLogger('cartuli.processing')
-    # TODO: Add args dict and/or start trace extra
-    logger.debug(f"Start {image} image straighten", extra={'trace': image})
+
+    trace_id = extract_id(image)
+
+    logger.debug(f"Start {image} image straighten", extra={'trace': image, 'trace_id': trace_id})
 
     # Apply Canny edge detection an detect linkes using Hought Line Transform
     gray_image = cv.cvtColor(np.array(image), cv.COLOR_RGB2GRAY)
@@ -117,25 +123,28 @@ def straighten(image: Image.Image, /, outliers_iqr_scale: float = 0.01) -> Image
         if line_angles[line] not in angles:
             color = "red"
         image_lines_draw.line((line[0:2], line[2:4]), fill=color, width=2)
-    logger.debug(f"Calculate {image} image lines", extra={'trace': image_lines})
+    logger.debug(f"Calculate {image} image lines", extra={'trace': image_lines, 'trace_id': trace_id})
 
     # Calculate the average angle of the detected lines and rotate image
     rotation_angle = -np.mean(angles)
     rotated_image = image.rotate(rotation_angle, expand=False)
-    logger.debug(f"Rotate {image} image", extra={'trace': rotated_image})
+    logger.debug(f"Rotate {image} image", extra={'trace': rotated_image, 'trace_id': trace_id})
 
     # TUNE: Maybe new content generated after rotation should be inpainted
 
     return rotated_image
 
 
-def crop(image: Image.Image, /, size: Size | float | int = 5) -> Image.Image:
+def crop(image: Image.Image, /,
+         size: Size | float | int = 5) -> Image.Image:
     logger = logging.getLogger('cartuli.processing')
 
-    logger.debug(f"Start {image} image crop", extra={'trace': image})
+    trace_id = extract_id(image)
+
+    logger.debug(f"Start {image} image crop", extra={'trace': image, 'trace_id': trace_id})
     crop_size = _to_size(size)
     crop_box = (crop_size.width, crop_size.height, image.width - crop_size.width, image.height - crop_size.height)
     crop_image = image.crop(crop_box)
-    logger.debug(f"Crop {image}", extra={'trace': crop_image})
+    logger.debug(f"Crop {image}", extra={'trace': crop_image, 'trace_id': trace_id})
 
     return crop_image

requirements.txt

@@ -2,4 +2,4 @@ reportlab==4.*
 opencv-python==4.*
 pillow==10.*
 pyyaml==6.*
-carpeta==0.1.0a0
+carpeta==0.1.0a2