From afb343974d1f5f035fcf04a8c40c4f92f709ef23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Mu=C3=B1oz?= Date: Wed, 13 Dec 2023 23:46:34 +0100 Subject: [PATCH] Make tracing work with multiprocessing --- cartuli/__main__.py | 8 +++++--- cartuli/card.py | 7 ++++++- cartuli/definition.py | 24 +++++++++++++++--------- cartuli/filters.py | 13 ++++++++++--- cartuli/processing.py | 37 +++++++++++++++++++++++-------------- requirements.txt | 2 +- 6 files changed, 60 insertions(+), 31 deletions(-) diff --git a/cartuli/__main__.py b/cartuli/__main__.py index 69e8c75..f783fca 100644 --- a/cartuli/__main__.py +++ b/cartuli/__main__.py @@ -6,7 +6,7 @@ import re import sys -from carpeta import Tracer, ImageHandler, trace_output +from carpeta import ProcessTracer, ImageHandler, trace_output from pathlib import Path from .definition import Definition @@ -33,7 +33,7 @@ def main(args=None): """Execute main package command line functionality.""" args = parse_args() - tracer = Tracer() + tracer = ProcessTracer() # Logging if args.verbose < 3: @@ -50,7 +50,7 @@ def main(args=None): if args.trace_output: processing_logger = logging.getLogger('cartuli.processing') processing_logger.setLevel(logging.DEBUG) - processing_handler = ImageHandler(tracer) + processing_handler = ImageHandler(tracer.remote_tracer) processing_handler.setLevel(logging.DEBUG) processing_logger.addHandler(processing_handler) @@ -79,6 +79,8 @@ def main(args=None): if tracer: trace_output(tracer, args.trace_output) + tracer.wait_and_stop() + return 0 diff --git a/cartuli/card.py b/cartuli/card.py index 1b234bd..e7126bc 100644 --- a/cartuli/card.py +++ b/cartuli/card.py @@ -1,6 +1,7 @@ """Card module.""" from pathlib import Path +from carpeta import register_id_extractor from PIL import Image from .measure import Size @@ -41,7 +42,7 @@ def image(self) -> Image.Image: @property def image_path(self) -> Path | None: - return self.__image + return self.__image_path @property def size(self) -> Size: @@ -85,6 +86,10 @@ def __str__(self) -> str: return super().__str__() +# TUNE: Not sure if this is the best place to place this... +register_id_extractor(CardImage, lambda x: x.name) + + class Card: """One or two sided card representation.""" diff --git a/cartuli/definition.py b/cartuli/definition.py index 6cbbdac..c36405b 100644 --- a/cartuli/definition.py +++ b/cartuli/definition.py @@ -18,6 +18,8 @@ from .sheet import Sheet +_CONCURRENT_PROCESSES = cpu_count() - 1 + CardsFilter = Callable[[Path], bool] @@ -83,7 +85,7 @@ def _load_images(self, images_definition: dict, size: Size, deck_name: str, side image_filter = images_definition.get('filter', '') image_files = sorted(glob(images_definition['images'])) logger.debug(f"Found {len(image_files)} {side} images for '{deck_name}' deck") - with Pool(processes=cpu_count() - 1) as pool: + with Pool(processes=_CONCURRENT_PROCESSES) as pool: images = pool.map( self.filters[image_filter].apply, (CardImage( @@ -111,6 +113,7 @@ def _load_deck(self, definition: dict, name: str) -> Deck: if len(front_images) != len(back_images): raise DefinitionError(f"The number of front ({len(front_images)}) and " f"back ({len(back_images)}) images must be the same") + # TODO Allow all back images to be filtered without errors cards = [Card(front_image, back_image) for front_image, back_image in zip(front_images, back_images)] else: cards = [Card(image) for image in front_images] @@ -123,15 +126,18 @@ def _load_deck(self, definition: dict, name: str) -> Deck: default_back = None if 'default_back' in definition: default_back_file = definition['default_back']['image'] - default_back_filter = definition['default_back'].get('filter', '') - default_back = self.filters[default_back_filter].apply( - CardImage( - default_back_file, - size=size, - bleed=from_str(definition['default_back'].get('bleed', str(CardImage.DEFAULT_BLEED))), - name=Path(default_back_file).stem + if self.__cards_filter(default_back_file): + default_back_filter = definition['default_back'].get('filter', '') + default_back = self.filters[default_back_filter].apply( + CardImage( + default_back_file, + size=size, + bleed=from_str(definition['default_back'].get('bleed', str(CardImage.DEFAULT_BLEED))), + name=Path(default_back_file).stem + ) ) - ) + else: + logger.debug(f"Default back image '{default_back_file}' filtered for '{name}' deck") return Deck(cards, name=name, default_back=default_back, size=size) diff --git a/cartuli/filters.py b/cartuli/filters.py index 59e3bbd..3731018 100644 --- a/cartuli/filters.py +++ b/cartuli/filters.py @@ -1,6 +1,7 @@ import logging from abc import ABC, abstractmethod +from carpeta import Traceable, extract_id from dataclasses import dataclass from .card import CardImage @@ -47,7 +48,7 @@ def apply(self, card_image: CardImage) -> CardImage: return CardImage( inpaint( - card_image.image, + Traceable(card_image.image, extract_id(card_image)), inpaint_size=card_image.resolution * self.inpaint_size, image_crop=card_image.resolution * self.image_crop, corner_radius=card_image.resolution * self.corner_radius, @@ -68,7 +69,10 @@ def apply(self, card_image: CardImage) -> CardImage: logger.debug(f'Applying to {card_image}') return CardImage( - straighten(card_image.image, self.outliers_iqr_scale), + straighten( + Traceable(card_image.image, extract_id(card_image)), + self.outliers_iqr_scale + ), size=card_image.size, bleed=card_image.bleed, name=card_image.name @@ -84,7 +88,10 @@ def apply(self, card_image: CardImage) -> CardImage: logger.debug(f'Applying to {card_image}') return CardImage( - crop(card_image.image, size=card_image.resolution * self.size), + crop( + Traceable(card_image.image, extract_id(card_image)), + size=card_image.resolution * self.size + ).value, # Traceable values are returned as traceable in crop, this solution is crap size=card_image.size, bleed=card_image.bleed, name=card_image.name diff --git a/cartuli/processing.py b/cartuli/processing.py index 45ec4e8..4bf86dd 100644 --- a/cartuli/processing.py +++ b/cartuli/processing.py @@ -2,6 +2,7 @@ import logging import numpy as np +from carpeta import extract_id from PIL import Image, ImageOps, ImageDraw from .measure import Size @@ -22,11 +23,14 @@ def _to_size(value: Size | float | int) -> Size: # def scale(image: Image.Image, /, ...) -> Image.Image: -def inpaint(image: Image.Image, /, inpaint_size: Size | float | int, image_crop: Size | float | int = 0, - corner_radius: Size | float | int = 0, inpaint_radius: float | int = 12) -> Image.Image: +def inpaint(image: Image.Image, /, inpaint_size: Size | float | int, + image_crop: Size | float | int = 0, corner_radius: Size | float | int = 0, + inpaint_radius: float | int = 12) -> Image.Image: logger = logging.getLogger('cartuli.processing') - # TODO: Add args dict and/or start trace extra - logger.debug(f"Start image {image} inpaint", extra={'trace': image}) + + trace_id = extract_id(image) + + logger.debug(f"Start image {image} inpaint", extra={'trace': image, 'trace_id': trace_id}) inpaint_size = _to_size(inpaint_size) image_crop = _to_size(image_crop) @@ -39,7 +43,7 @@ def inpaint(image: Image.Image, /, inpaint_size: Size | float | int, image_crop: .crop((expand_crop.width, expand_crop.height, image.size[0] + expand_size*2 - expand_crop.width, image.size[1] + expand_size*2 - expand_crop.height)) - logger.debug(f"Expand {image} image", extra={'trace': expanded_image}) + logger.debug(f"Expand {image} image", extra={'trace': expanded_image, 'trace_id': trace_id}) mask_image = Image.new('L', (image.size[0] + inpaint_size.width*2, image.size[1] + inpaint_size.height*2), color='white') @@ -50,13 +54,13 @@ def inpaint(image: Image.Image, /, inpaint_size: Size | float | int, image_crop: mask_image.size[1] - inpaint_size.height - image_crop.height), fill='black', width=0, radius=max(corner_radius)) # TUNE: Find a way to round with different vertical and horizontal values - logger.debug(f"Mask {image} image for inpainting", extra={'trace': mask_image}) + logger.debug(f"Mask {image} image for inpainting", extra={'trace': mask_image, 'trace_id': trace_id}) inpaint_image_cv = cv.inpaint( cv.cvtColor(np.array(expanded_image), cv.COLOR_RGB2BGR), np.array(mask_image), int(inpaint_radius), cv.INPAINT_NS) inpainted_image = Image.fromarray(cv.cvtColor(inpaint_image_cv, cv.COLOR_BGR2RGB)) - logger.debug(f"Inpaint {image} image", extra={'trace': inpainted_image}) + logger.debug(f"Inpaint {image} image", extra={'trace': inpainted_image, 'trace_id': trace_id}) return inpainted_image @@ -94,8 +98,10 @@ def _discard_outliers(data: np.ndarray | list, iqr_scale: float = 1.5) -> np.nda def straighten(image: Image.Image, /, outliers_iqr_scale: float = 0.01) -> Image.Image: logger = logging.getLogger('cartuli.processing') - # TODO: Add args dict and/or start trace extra - logger.debug(f"Start {image} image straighten", extra={'trace': image}) + + trace_id = extract_id(image) + + logger.debug(f"Start {image} image straighten", extra={'trace': image, 'trace_id': trace_id}) # Apply Canny edge detection an detect linkes using Hought Line Transform gray_image = cv.cvtColor(np.array(image), cv.COLOR_RGB2GRAY) @@ -117,25 +123,28 @@ def straighten(image: Image.Image, /, outliers_iqr_scale: float = 0.01) -> Image if line_angles[line] not in angles: color = "red" image_lines_draw.line((line[0:2], line[2:4]), fill=color, width=2) - logger.debug(f"Calculate {image} image lines", extra={'trace': image_lines}) + logger.debug(f"Calculate {image} image lines", extra={'trace': image_lines, 'trace_id': trace_id}) # Calculate the average angle of the detected lines and rotate image rotation_angle = -np.mean(angles) rotated_image = image.rotate(rotation_angle, expand=False) - logger.debug(f"Rotate {image} image", extra={'trace': rotated_image}) + logger.debug(f"Rotate {image} image", extra={'trace': rotated_image, 'trace_id': trace_id}) # TUNE: Maybe new content generated after rotation should be inpainted return rotated_image -def crop(image: Image.Image, /, size: Size | float | int = 5) -> Image.Image: +def crop(image: Image.Image, /, + size: Size | float | int = 5) -> Image.Image: logger = logging.getLogger('cartuli.processing') - logger.debug(f"Start {image} image crop", extra={'trace': image}) + trace_id = extract_id(image) + + logger.debug(f"Start {image} image crop", extra={'trace': image, 'trace_id': trace_id}) crop_size = _to_size(size) crop_box = (crop_size.width, crop_size.height, image.width - crop_size.width, image.height - crop_size.height) crop_image = image.crop(crop_box) - logger.debug(f"Crop {image}", extra={'trace': crop_image}) + logger.debug(f"Crop {image}", extra={'trace': crop_image, 'trace_id': trace_id}) return crop_image diff --git a/requirements.txt b/requirements.txt index 14b860f..989a54a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ reportlab==4.* opencv-python==4.* pillow==10.* pyyaml==6.* -carpeta==0.1.0a0 +carpeta==0.1.0a2