Merge pull request #62

v4.1.0

acacore/__version__.py

@@ -1 +1 @@
-__version__ = "4.0.0"
+__version__ = "4.1.0"

acacore/database/files_db.py

@@ -156,9 +156,11 @@ def __init__(
             EventPath,
             "log_paths",
             f"""
-            select coalesce(fo.relative_path, fm.relative_path) as file_relative_path, l.* from {self.log.name} l
-                left join {self.original_files.name} fo on l.file_type = 'original' and fo.uuid = l.file_uuid
-                left join {self.master_files.name} fm on l.file_type = 'master' and fm.uuid = l.file_uuid
+            select coalesce(fo.relative_path, fm.relative_path, fa.relative_path, fs.relative_path) as file_relative_path, l.* from {self.log.name} l
+                left join {self.original_files.name}  fo on l.file_type = 'original'  and fo.uuid = l.file_uuid
+                left join {self.master_files.name}    fm on l.file_type = 'master'    and fm.uuid = l.file_uuid
+                left join {self.access_files.name}    fa on l.file_type = 'access'    and fa.uuid = l.file_uuid
+                left join {self.statutory_files.name} fs on l.file_type = 'statutory' and fs.uuid = l.file_uuid
             """,
         )
 

acacore/database/upgrade.py

@@ -1,3 +1,4 @@
+from json import dumps
 from json import loads
 from sqlite3 import Connection
 from sqlite3 import DatabaseError
@@ -25,13 +26,113 @@ def get_db_version(conn: Connection) -> Version | None:
 
 
 def set_db_version(conn: Connection, version: Version) -> Version:
-    conn.execute("insert or replace into Metadata (KEY, VALUE) values (?, ?)", ("version", str(version)))
+    conn.execute("insert or replace into Metadata (KEY, VALUE) values (?, ?)", ("version", dumps(str(version))))
     conn.commit()
     return version
 
 
+# noinspection SqlResolve
+def upgrade_4to4_1(con: Connection) -> Version:
+    con.execute("""
+    create table files_master_tmp
+    (
+        uuid              text    not null,
+        checksum          text    not null,
+        relative_path     text    not null,
+        is_binary         boolean not null,
+        size              integer not null,
+        puid              text,
+        signature         text,
+        warning           text,
+        original_uuid     text,
+        convert_access    text,
+        convert_statutory text,
+        processed         integer not null,
+        primary key (relative_path)
+    )
+    """)
+
+    con.execute("insert or ignore into files_master_tmp select * from files_master")
+    con.execute("update files_master_tmp set processed = 4 where processed != 0")
+
+    con.executemany(
+        "update files_master_tmp set processed = processed + 1 where uuid = ? and processed != 0",
+        ([uuid] for [uuid] in con.execute("select original_uuid from files_access")),
+    )
+
+    con.executemany(
+        "update files_master_tmp set processed = processed + 2 where uuid = ? and processed != 0",
+        ([uuid] for [uuid] in con.execute("select original_uuid from files_statutory")),
+    )
+
+    con.execute("update files_master_tmp set processed = processed - 4 where processed != 0")
+
+    con.execute("drop view files_all")
+    con.execute("drop view log_paths")
+    con.execute("drop table files_master")
+    con.execute("alter table files_master_tmp rename to 'files_master'")
+    con.execute("""
+    create view files_all as
+    select uuid,
+       checksum,
+       relative_path,
+       is_binary,
+       size,
+       puid,
+       signature,
+       warning
+    from files_original
+    union
+    select uuid,
+           checksum,
+           relative_path,
+           is_binary,
+           size,
+           puid,
+           signature,
+           warning
+    from files_master
+    union
+    select uuid,
+           checksum,
+           relative_path,
+           is_binary,
+           size,
+           puid,
+           signature,
+           warning
+    from files_access
+    union
+    select uuid,
+           checksum,
+           relative_path,
+           is_binary,
+           size,
+           puid,
+           signature,
+           warning
+    from files_statutory
+    """)
+    con.execute("""
+    create view log_paths as
+    select coalesce(fo.relative_path, fm.relative_path, fa.relative_path, fs.relative_path) as file_relative_path, l.*
+    from log l
+        left join files_original  fo on l.file_type = 'original'  and fo.uuid = l.file_uuid
+        left join files_master    fm on l.file_type = 'master'    and fm.uuid = l.file_uuid
+        left join files_access    fa on l.file_type = 'access'    and fa.uuid = l.file_uuid
+        left join files_statutory fs on l.file_type = 'statutory' and fs.uuid = l.file_uuid
+    """)
+    con.commit()
+
+    con.execute("vacuum")
+
+    return set_db_version(con, Version("4.1.0"))
+
+
 def get_upgrade_function(current_version: Version, latest_version: Version) -> Callable[[Connection], Version]:
-    if current_version < latest_version:
+    if current_version < Version("4.1.0"):
+        return upgrade_4to4_1
+    elif current_version < latest_version:
         return lambda c: set_db_version(c, Version(__version__))
     else:
         return lambda _: latest_version

acacore/models/file.py

@@ -84,7 +84,7 @@ def get_identifier(file: "BaseFile", file_classes: list[TSiegfriedFileClass], ac
 
 class BaseFile(BaseModel):
     """
-    File model containing all information used by the rest of the archival suite of tools.
+    Base model for file identification.
 
     :ivar uuid: The UUID of the file.
     :ivar checksum: The checksum of the file.
@@ -294,7 +294,7 @@ def suffixes(self, new_suffixes: str):
 
 class OriginalFile(BaseFile):
     """
-    File model containing all information used by the rest of the archival suite of tools.
+    File model for OriginalDocuments files.
 
     :ivar action: The name of the main action for the file's PUID, if one exists.
     :ivar action_data: The data for the action for the file's PUID, if one exists.
@@ -477,6 +477,12 @@ def get_action(
 
 
 class ConvertedFile(BaseFile):
+    """
+    File model for output of file conversion.
+
+    :ivar original_uuid: UUID of the parent file that was converted.
+    """
+
     original_uuid: UUID4 | None = None
 
     @classmethod
@@ -517,9 +523,17 @@ def from_file(
 
 
 class MasterFile(ConvertedFile):
+    """
+    File model for MasterDocuments files converted from OriginalDocuments.
+
+    :ivar convert_access: Convert action to generate access file.
+    :ivar convert_statutory: Convert action to generate statutory file.
+    :ivar processed: Processed status as a bit flag: 0 none, 1 access, 2 statutory.
+    """
+
     convert_access: ConvertAction | None = None
     convert_statutory: ConvertAction | None = None
-    processed: bool = False
+    processed: int = Field(0, ge=0, le=3)
 
     @classmethod
     def from_file(
@@ -531,7 +545,7 @@ def from_file(
         custom_signatures: list[CustomSignature] | None = None,
         actions: dict[str, MasterConvertAction] | None = None,
         uuid: UUID | None = None,
-        processed: bool = False,
+        processed: int = 0,
     ) -> Self:
         """
         Create a file object from a given path.
@@ -544,7 +558,7 @@ def from_file(
             not provided or fails to find a match.
         :param actions: Optionally, a dictionary of ``MasterConvertAction`` objects to assign an action to the file.
         :param uuid: Optionally, the UUID of the file.
-        :param processed: Optionally, a boolean indicating if the file was processed.
+        :param processed: Optionally, a bit flag indicating if the file was processed: 0 none, 1 access, 2 statutory.
         :return: A ``MasterFile`` object.
         """
         file_base = super().from_file(path, root, original_uuid, siegfried, custom_signatures, uuid)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "acacore"
-version = "4.0.0"
+version = "4.1.0"
 description = ""
 authors = ["Matteo Campinoti <[email protected]>"]
 license = "GPL-3.0"

tests/test_database.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from shutil import copy2
 from sqlite3 import DatabaseError
 from uuid import uuid4
 
@@ -172,3 +173,36 @@ def test_database_update_delete(database_file: Path):
         db.original_files.delete(file1)
         assert db.original_files[file1] is None
         assert len(db.original_files) == 0
+
+
+def test_database_upgrade(test_folder: Path, temp_folder: Path):
+    database_file: Path = test_folder / "databases" / "v4_0_0.db"
+    database_file_copy: Path = temp_folder / database_file.name
+    database_file_copy.unlink(missing_ok=True)
+    database_file_copy.parent.mkdir(parents=True, exist_ok=True)
+
+    copy2(database_file, database_file_copy)
+
+    with FilesDB(database_file_copy, check_version=False) as db:
+        assert db.version() < Version(__version__)
+
+        db.upgrade()
+
+        assert db.version() == Version(__version__)
+
+        assert db.original_files.select(limit=1).fetchone()
+        assert db.master_files.select(limit=1).fetchone()
+
+        for master_file in db.master_files:
+            if master_file.processed == 0:
+                assert db.access_files[{"original_uuid": str(master_file.uuid)}] is None
+                assert db.statutory_files[{"original_uuid": str(master_file.uuid)}] is None
+            elif master_file.processed == 1:
+                assert db.access_files[{"original_uuid": str(master_file.uuid)}] is not None
+                assert db.statutory_files[{"original_uuid": str(master_file.uuid)}] is None
+            elif master_file.processed == 2:
+                assert db.access_files[{"original_uuid": str(master_file.uuid)}] is None
+                assert db.statutory_files[{"original_uuid": str(master_file.uuid)}] is not None
+            elif master_file.processed == 3:
+                assert db.access_files[{"original_uuid": str(master_file.uuid)}] is not None
+                assert db.statutory_files[{"original_uuid": str(master_file.uuid)}] is not None