Merge pull request #38

v3.0.0

acacore/__version__.py

@@ -1 +1 @@
-__version__ = "2.0.2"
+__version__ = "3.0.0"

acacore/database/base.py

@@ -843,6 +843,7 @@ def __init__(
         :param uri: If set to True, database is interpreted as a URI with a file path and an optional query string,
             defaults to False.
         """
+        self.committed_changes: int = 0
         super().__init__(
             database,
             timeout,
@@ -879,6 +880,11 @@ def is_open(self) -> bool:
         except DatabaseError:
             return False
 
+    def commit(self):
+        """Commit any pending transaction to the database."""
+        super().commit()
+        self.committed_changes = self.total_changes
+
     @overload
     def create_table(self, name: str, columns: Type[M], indices: list[Index] | None = None) -> ModelTable[M]: ...
 

acacore/database/upgrade.py

@@ -1,4 +1,8 @@
+from json import dumps
+from json import loads
 from sqlite3 import DatabaseError
+from sqlite3 import Row
+from typing import Any
 from typing import Callable
 
 from packaging.version import Version
@@ -30,6 +34,8 @@ def get_upgrade_function(current_version: Version, latest_version: Version) -> C
         return upgrade_1to2
     elif current_version < Version("2.0.2"):
         return upgrade_2to2_0_2
+    elif current_version < Version("3.0.0"):
+        return upgrade_2_0_2to3
     elif current_version < latest_version:
         return upgrade_last
     else:
@@ -38,27 +44,94 @@ def get_upgrade_function(current_version: Version, latest_version: Version) -> C
 
 # noinspection SqlResolve
 def upgrade_1to2(db: FileDB) -> Version:
+    # Add "lock" column if not already present
     if not db.execute("select 1 from pragma_table_info('Files') where name = 'lock'").fetchone():
-        db.execute("alter table Files add column lock boolean default false")
-    db.execute("update Files set lock = false where lock is null")
+        db.execute("alter table Files add column lock boolean")
+        db.execute("update Files set lock = false")
+    # Rename "replace" action to "template"
     db.execute("update Files set action = 'template' where action = 'replace'")
-    db.execute("update Files set action_data = '{}' where action_data is null")
+    # Ensure action_data is always a readable JSON
+    db.execute("update Files set action_data = '{}' where action_data is null or action_data = ''")
 
+    # Reset _IdentificationWarnings view
     db.execute("drop view if exists _IdentificationWarnings")
     db.identification_warnings.create()
 
-    for file in db.files.select():
-        db.files.update(file)
+    cursor = db.execute("select * from files where action_data != '{}'")
+    cursor.row_factory = Row
+
+    for file in cursor:
+        action_data: dict[str, Any] = loads(file["action_data"])
+        # Rename "replace" action to "template"
+        action_data["template"] = action_data.get("replace")
+        # Remove None and empty lists (default values)
+        action_data = {k: v for k, v in action_data.items() if v}
+        db.execute("update Files set action_data = ? where uuid = ?", [dumps(action_data), file["uuid"]])
+
+    db.commit()
 
     return set_db_version(db, Version("2.0.0"))
 
 
 def upgrade_2to2_0_2(db: FileDB) -> Version:
     db.execute("drop view if exists _IdentificationWarnings")
     db.identification_warnings.create()
+    db.commit()
     return set_db_version(db, Version("2.0.2"))
 
 
+# noinspection SqlResolve
+def upgrade_2_0_2to3(db: FileDB) -> Version:
+    def convert_action_data(data: dict):
+        new_data: dict[str, Any] = {}
+
+        if rename := data.get("rename"):
+            new_data["rename"] = rename
+
+        if reidentify := data.get("reidentify"):
+            new_data["reidentify"] = {"reason": reidentify["reason"]}
+            if on_fail := reidentify.get("onfail"):
+                new_data["reidentify"]["on_fail"] = on_fail
+
+        if convert := data.get("convert"):
+            new_data["convert"] = {"tool": convert[0]["converter"], "outputs": convert[0]["outputs"]}
+
+        if extract := data.get("extract"):
+            new_data["extract"] = {"tool": extract["tool"]}
+            if extension := extract.get("extension"):
+                new_data["extract"]["extension"] = extension
+
+        if manual := data.get("manual"):
+            new_data["manual"] = manual
+
+        if (ignore := data.get("ignore")) and ignore.get("reason"):
+            new_data["ignore"] = {"template": "not-preservable", "reason": ignore.get("reason", "")}
+        elif template := data.get("template"):
+            new_data["ignore"] = {"template": template["template"]}
+            if template_text := template.get("template_text"):
+                new_data["ignore"]["reason"] = template_text
+
+        return new_data
+
+    # Add "parent" column if not already present
+    if not db.execute("select 1 from pragma_table_info('Files') where name = 'parent'").fetchone():
+        db.execute("alter table Files add column parent text")
+        db.execute("update Files set parent = null")
+
+    cursor = db.execute("select * from Files where action_data != '{}'")
+    cursor.row_factory = Row
+
+    for file in cursor:
+        db.execute(
+            "update Files set action_data = ? where uuid is ?",
+            [dumps(convert_action_data(loads(file["action_data"]))), file["uuid"]],
+        )
+
+    db.commit()
+
+    return set_db_version(db, Version("3.0.0"))
+
+
 def upgrade_last(db: FileDB) -> Version:
     db.init()
     return set_db_version(db, Version(__version__))
@@ -97,6 +170,8 @@ def upgrade(db: FileDB):
     """
     if not db.is_initialised(check_views=False, check_indices=False):
         raise DatabaseError("Database is not initialised")
+    if db.committed_changes != db.total_changes:
+        raise DatabaseError("Database has uncommited transactions")
 
     if is_latest(db):
         return
@@ -106,4 +181,6 @@ def upgrade(db: FileDB):
 
     while current_version < latest_version:
         update_function = get_upgrade_function(current_version, latest_version)
+        print(current_version, end=" ")
         current_version = update_function(db)
+        print(current_version)

acacore/models/file.py

@@ -17,6 +17,7 @@
 from acacore.utils.functions import get_eof
 from acacore.utils.functions import image_size
 from acacore.utils.functions import is_binary
+from acacore.utils.functions import is_valid_suffix
 
 from .reference_files import Action
 from .reference_files import ActionData
@@ -27,33 +28,31 @@
 from .reference_files import TActionType
 
 
-def _ignore_if(file: "File", ignore_ifs: list[IgnoreIfAction]) -> "File":
+def ignore_if(file: "File", ignore_rules: IgnoreIfAction) -> "File":
     action: TActionType | None = None
-    action_data: ActionData | None = None
-
-    for ignore_if in ignore_ifs:
-        if ignore_if.pixel_total or ignore_if.pixel_width or ignore_if.pixel_height:
-            width, height = image_size(file.get_absolute_path())
-            if (
-                width * height < (ignore_if.pixel_total or 0)
-                or width < (ignore_if.pixel_width or 0)
-                or height < (ignore_if.pixel_height or 0)
-            ):
-                action = "ignore"
-        elif ignore_if.binary_size and file.is_binary and file.size < ignore_if.binary_size:  # noqa: SIM114
+    ignore_action: IgnoreAction = IgnoreAction(template="not-preservable")
+
+    if ignore_rules.image_pixels_min or ignore_rules.image_width_min or ignore_rules.image_height_min:
+        width, height = image_size(file.get_absolute_path())
+        if ignore_rules.image_width_min and width < ignore_rules.image_width_min:
             action = "ignore"
-        elif ignore_if.size and file.size < ignore_if.size:
+            ignore_action.reason = f"Image width is too small ({width}px < {ignore_rules.image_width_min})"
+        elif ignore_rules.image_height_min and height < ignore_rules.image_height_min:
             action = "ignore"
+            ignore_action.reason = f"Image height is too small  ({height}px < {ignore_rules.image_height_min})"
+        elif ignore_rules.image_pixels_min and (width * height) < ignore_rules.image_pixels_min:
+            action = "ignore"
+            ignore_action.reason = (
+                f"Image resolution is too small  ({width * height}px < {ignore_rules.image_pixels_min})"
+            )
+    elif ignore_rules.size and file.size < ignore_rules.size:
+        action = "ignore"
+        ignore_action.reason = "File size is too small"
 
-        if action:
-            action_data = file.action_data or ActionData()
-            action_data.ignore = action_data.ignore or IgnoreAction()
-            action_data.ignore.reason = ignore_if.reason or action_data.ignore.reason
-            break
-
-    if action and action_data:
+    if action:
         file.action = action
-        file.action_data = action_data
+        file.action_data = file.action_data or ActionData()
+        file.action_data.ignore = ignore_action
 
     return file
 
@@ -87,6 +86,7 @@ class File(BaseModel):
     warning: list[str] | None = None
     action: TActionType | None = DBField(index=["idx_action"])
     action_data: ActionData = Field(default_factory=ActionData)
+    parent: UUID4 | None = None
     processed: bool = False
     lock: bool = False
     root: Path | None = DBField(None, ignore=True)
@@ -153,7 +153,7 @@ def from_file(
         if actions:
             action = file.get_action(actions, file_classes)
 
-        if custom_signatures and file.action == "reidentify":
+        if action and action.reidentify and custom_signatures:
             custom_match = file.identify_custom(custom_signatures)
             if custom_match:
                 file.puid = custom_match.puid
@@ -170,12 +170,14 @@ def from_file(
                 file.action = "manual"
                 file.action_data = ActionData(manual=ManualAction(reason="Re-identify failure", process=""))
                 file.puid = file.signature = file.warning = None
+        elif action and action.reidentify:
+            raise ValueError(f"Cannot run re-identify for PUID {file.puid} without custom signatures")
 
-        if file.action_data and file.action_data.ignore:
-            file = _ignore_if(file, file.action_data.ignore.ignore_if)
+        if action and action.ignore_if:
+            file = ignore_if(file, action.ignore_if)
 
-        if file.action != "ignore" and actions and "*" in actions:
-            file = _ignore_if(file, actions["*"].ignore.ignore_if if actions["*"].ignore else [])
+        if file.action != "ignore" and actions and "*" in actions and actions["*"].ignore_if:
+            file = ignore_if(file, actions["*"].ignore_if)
 
         if action and file.warning:
             file.warning = [w for w in file.warning if w.lower() not in [aw.lower() for aw in action.ignore_warnings]]
@@ -276,7 +278,7 @@ def get_action(
 
         action: Action | None = reduce(lambda acc, cur: acc or actions.get(cur), identifiers, None)
 
-        if action and action.alternatives and (new_puid := action.alternatives.get(self.suffix.lower(), None)):
+        if action and action.alternatives and (new_puid := action.alternatives.get(self.suffixes.lower(), None)):
             puid: str = self.puid
             self.puid = new_puid
             if new_action := self.get_action(actions, file_classes, set_match=set_match):
@@ -359,3 +361,22 @@ def suffix(self) -> str:
     @suffix.setter
     def suffix(self, new_suffix: str):
         self.relative_path = self.relative_path.with_suffix(new_suffix)
+
+    @property
+    def suffixes(self) -> str:
+        """
+        Get file suffixes. Excludes invalid ones.
+
+        :return: All the file extensions as a string.
+        """
+        suffixes: str = ""
+        for suffix in self.relative_path.suffixes[::-1]:
+            if is_valid_suffix(suffix):
+                suffixes += suffix
+            else:
+                break
+        return suffixes
+
+    @suffixes.setter
+    def suffixes(self, new_suffixes: str):
+        self.relative_path = self.relative_path.with_name(self.name.removesuffix(self.suffixes) + new_suffixes)