From a392eb1c63327f757995789ee74f0bf389c5f3c8 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Fri, 25 Oct 2024 11:34:51 +0200 Subject: [PATCH 1/8] models.file:File - add original_name and converted_name fields --- acacore/models/file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acacore/models/file.py b/acacore/models/file.py index 0d07067..751e69a 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -89,6 +89,8 @@ class File(BaseModel): parent: UUID4 | None = None processed: bool = False lock: bool = False + original_name: str + converted_name: str | None = None root: Path | None = DBField(None, ignore=True) # noinspection PyNestedDecorators From 45ba3acc72268813021e64729dfd5f0017a33fc9 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Fri, 25 Oct 2024 11:35:02 +0200 Subject: [PATCH 2/8] tests - use `@pytest.fixture` over `@pytest.fixture()` --- tests/test_database.py | 2 +- tests/test_siegfried.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index fc09c52..953cda3 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -26,7 +26,7 @@ from acacore.models.reference_files import ConvertAction -@pytest.fixture() +@pytest.fixture def database_path(temp_folder: Path) -> Path: path: Path = temp_folder / "files.db" path.unlink(missing_ok=True) diff --git a/tests/test_siegfried.py b/tests/test_siegfried.py index 2b66a0a..65c540e 100644 --- a/tests/test_siegfried.py +++ b/tests/test_siegfried.py @@ -7,12 +7,12 @@ from acacore.siegfried import Siegfried -@pytest.fixture() +@pytest.fixture def siegfried_folder(test_folder: Path) -> Path: return test_folder / "siegfried" -@pytest.fixture() +@pytest.fixture def siegfried(siegfried_folder: Path) -> Siegfried: return Siegfried(Path(environ["GOPATH"], "bin", "sf"), "pronom.sig", siegfried_folder) From 8ea55f0ef5674dafcf660bba5a9bf3eb939371ec Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Fri, 25 Oct 2024 11:36:19 +0200 Subject: [PATCH 3/8] models.file:File._model_validator - add validator that sets original name --- acacore/models/file.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/acacore/models/file.py b/acacore/models/file.py index 751e69a..1fdfcec 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -6,6 +6,7 @@ from pydantic import BaseModel from pydantic import Field from pydantic import field_validator +from pydantic import model_validator from pydantic import UUID4 from acacore.database.column import DBField @@ -93,6 +94,14 @@ class File(BaseModel): converted_name: str | None = None root: Path | None = DBField(None, ignore=True) + # noinspection PyNestedDecorators + @model_validator(mode="before") + @classmethod + def _model_validator(cls, data: dict): + if isinstance(data, dict): + data["original_name"] = data.get("original_name", "").strip() or Path(data["relative_path"]).name + return data + # noinspection PyNestedDecorators @field_validator("action_data", mode="before") @classmethod From dd98338176fd3125d6df492dd60273444bf3f9bb Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Fri, 25 Oct 2024 11:36:46 +0200 Subject: [PATCH 4/8] version - minor 3.2.0 > 3.3.0 --- acacore/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acacore/__version__.py b/acacore/__version__.py index 1173108..88c513e 100644 --- a/acacore/__version__.py +++ b/acacore/__version__.py @@ -1 +1 @@ -__version__ = "3.2.0" +__version__ = "3.3.0" diff --git a/pyproject.toml b/pyproject.toml index 58e88c3..f59e5b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "acacore" -version = "3.2.0" +version = "3.3.0" description = "" authors = ["Matteo Campinoti "] license = "GPL-3.0" From 72d22e380f44181662cad625ad9248e09fa0b8bb Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Fri, 25 Oct 2024 11:47:43 +0200 Subject: [PATCH 5/8] models.file:File.converted_name - rename to processed_name --- acacore/models/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acacore/models/file.py b/acacore/models/file.py index 1fdfcec..737d1be 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -91,7 +91,7 @@ class File(BaseModel): processed: bool = False lock: bool = False original_name: str - converted_name: str | None = None + processed_name: str | None = None root: Path | None = DBField(None, ignore=True) # noinspection PyNestedDecorators From 2751aea9221b4a326bc84632075fabd2c090e5e2 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 29 Oct 2024 07:18:06 +0100 Subject: [PATCH 6/8] database.upgrade:upgrade_3_2to3_3 - add upgrade from 3.2.x to 3.3.0 --- acacore/database/upgrade.py | 66 ++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/acacore/database/upgrade.py b/acacore/database/upgrade.py index 1d37139..e8c5bc8 100644 --- a/acacore/database/upgrade.py +++ b/acacore/database/upgrade.py @@ -1,6 +1,8 @@ from functools import reduce from json import dumps +from json import JSONDecodeError from json import loads +from pathlib import Path from sqlite3 import Connection from sqlite3 import DatabaseError from typing import Any @@ -15,7 +17,6 @@ "is_latest", ] - from .files_db import FileDB @@ -214,6 +215,67 @@ def convert_action_data(data: dict) -> dict: return set_db_version(conn, Version("3.2.0")) +# noinspection SqlResolve +def upgrade_3_2to3_3(conn: Connection) -> Version: + if not conn.execute("select 1 from pragma_table_info('Files') where name = 'original_name'").fetchone(): + conn.execute("alter table Files add column original_name text not null default ''") + if not conn.execute("select 1 from pragma_table_info('Files') where name = 'processed_name'").fetchone(): + conn.execute("alter table Files add column processed_name text default null") + + def _find_original_name(uuid: str, relative_path: str) -> str: + original_path: Path = Path(relative_path) + original_name: str = original_path.name + + for [data_raw] in conn.execute( + "select data from History" + " where uuid = ? and data is not null and data not in ('', '\"\"', 'null', '[]', '{}')" + " order by time desc", + (uuid,), + ): + try: + data = loads(data_raw) + except JSONDecodeError: + continue + + if ( + not isinstance(data, list) + or len(data) != 2 + or not isinstance(data[0], str) + or not isinstance(data[1], str) + ): + continue + + a: str = data[0] + b: str = data[1] + + if a == str(original_path): + original_path = Path(b) + original_name = original_path.name + elif b == str(original_path): + original_path = Path(a) + original_name = original_path.name + elif a == original_name: + original_name = b + original_path = original_path.with_name(original_name) + elif b == original_name: + original_name = a + original_path = original_path.with_name(original_name) + + return original_name + + conn.executemany( + "update Files set original_name = ? where uuid = ?", + ( + (_find_original_name(uuid, relative_path), uuid) + for uuid, relative_path in conn.execute( + "select distinct f.uuid, f.relative_path from Files f join History h where h.uuid = f.uuid" + ) + ), + ) + + return set_db_version(conn, Version("3.3.0")) + + def get_upgrade_function(current_version: Version, latest_version: Version) -> Callable[[Connection], Version]: if current_version < Version("2.0.0"): return upgrade_1to2 @@ -229,6 +291,8 @@ def get_upgrade_function(current_version: Version, latest_version: Version) -> C return upgrade_3_0_6to3_0_7 elif current_version < Version("3.2.0"): return upgrade_3_1to3_2 + elif current_version < Version("3.3.0"): + return upgrade_3_2to3_3 elif current_version < latest_version: return lambda c: set_db_version(c, Version(__version__)) else: From 39930403408244083219d571bb7eb31715e325fe Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 29 Oct 2024 07:31:14 +0100 Subject: [PATCH 7/8] database.upgrade:upgrade_3_2to3_3 - set `original_name` for files without history entries --- acacore/database/upgrade.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acacore/database/upgrade.py b/acacore/database/upgrade.py index e8c5bc8..c9cedeb 100644 --- a/acacore/database/upgrade.py +++ b/acacore/database/upgrade.py @@ -273,6 +273,14 @@ def _find_original_name(uuid: str, relative_path: str) -> str: ), ) + conn.executemany( + "update Files set original_name = ? where uuid = ?", + ( + (Path(relative_path).name, uuid) + for uuid, relative_path in conn.execute("select uuid, relative_path from Files where original_name = ''") + ), + ) + return set_db_version(conn, Version("3.3.0")) From d755d57abc42da389f270c61a270e99439399a04 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 29 Oct 2024 07:51:47 +0100 Subject: [PATCH 8/8] models.file:File.processed_name - rename to processed_names and turn into a list --- acacore/database/upgrade.py | 2 +- acacore/models/file.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acacore/database/upgrade.py b/acacore/database/upgrade.py index c9cedeb..a2e34f4 100644 --- a/acacore/database/upgrade.py +++ b/acacore/database/upgrade.py @@ -220,7 +220,7 @@ def upgrade_3_2to3_3(conn: Connection) -> Version: if not conn.execute("select 1 from pragma_table_info('Files') where name = 'original_name'").fetchone(): conn.execute("alter table Files add column original_name text not null default ''") if not conn.execute("select 1 from pragma_table_info('Files') where name = 'processed_name'").fetchone(): - conn.execute("alter table Files add column processed_name text default null") + conn.execute("alter table Files add column processed_name text default '[]'") def _find_original_name(uuid: str, relative_path: str) -> str: original_path: Path = Path(relative_path) diff --git a/acacore/models/file.py b/acacore/models/file.py index 737d1be..9675651 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -91,7 +91,7 @@ class File(BaseModel): processed: bool = False lock: bool = False original_name: str - processed_name: str | None = None + processed_names: list[str] = Field(default_factory=list) root: Path | None = DBField(None, ignore=True) # noinspection PyNestedDecorators