From 9ad63974b34f2ada143c9a07723dfd06a4f4ed3f Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Mon, 5 Aug 2024 13:59:22 +0200 Subject: [PATCH 01/13] database.upgrade:upgrade_1to2 - replace null values in Files.action_data with "{}" --- acacore/database/upgrade.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acacore/database/upgrade.py b/acacore/database/upgrade.py index 475ab0e..a918405 100644 --- a/acacore/database/upgrade.py +++ b/acacore/database/upgrade.py @@ -40,6 +40,7 @@ def upgrade_1to2(db: FileDB) -> Version: db.execute("alter table Files add column lock boolean default false") db.execute("update Files set lock = false where lock is null") db.execute("update Files set action = 'template' where action = 'replace'") + db.execute("update Files set action_data = '{}' where action_data is null") for file in db.files.select(): db.files.update(file) return set_db_version(db, Version("2.0.0")) From 4773ea0b0310a47b4be2cbe9daa3906cf2124fc8 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Mon, 5 Aug 2024 14:01:21 +0200 Subject: [PATCH 02/13] database.column - save None as null and read null as None --- acacore/database/column.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/acacore/database/column.py b/acacore/database/column.py index 142396d..41d4eb0 100644 --- a/acacore/database/column.py +++ b/acacore/database/column.py @@ -248,14 +248,21 @@ def _schema_to_column(name: str, schema: dict, defs: dict[str, dict] | None = No to_entry, from_entry = schema["enum"][0].__class__ if schema["enum"] else str, str elif schema_type in ("object", "array"): sql_type = "text" - to_entry, from_entry = lambda o: dumps(dump_object(o), default=str), lambda o: loads(o) + to_entry, from_entry = ( + lambda o: None if o is None else dumps(dump_object(o), default=str), + lambda o: None if o is None else loads(o), + ) elif type_name in _sql_schema_type_converters: to_entry, from_entry = _sql_schema_type_converters[type_name] else: raise TypeError(f"Cannot recognize type from schema {schema!r}") elif schema_any_of: if not schema_any_of[0] or len(schema_any_of) > 2: - sql_type, to_entry, from_entry = "text", lambda o: dumps(dump_object(o), default=str), lambda x: loads(x) + sql_type, to_entry, from_entry = ( + "text", + lambda x: None if x is None else dumps(dump_object(x), default=str), + lambda x: None if x is None else loads(x), + ) else: return _schema_to_column(name, {**schema_any_of[0], **schema}, defs) else: From 8ee3fb40933a4b1b4885c7eb8f57487dcd984b2b Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 12:53:46 +0200 Subject: [PATCH 03/13] database.upgrade:upgrade_1to2 - update _IdentificationWarnings view --- acacore/database/upgrade.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/acacore/database/upgrade.py b/acacore/database/upgrade.py index a918405..076e100 100644 --- a/acacore/database/upgrade.py +++ b/acacore/database/upgrade.py @@ -41,8 +41,13 @@ def upgrade_1to2(db: FileDB) -> Version: db.execute("update Files set lock = false where lock is null") db.execute("update Files set action = 'template' where action = 'replace'") db.execute("update Files set action_data = '{}' where action_data is null") + + db.execute("drop view if exists _IdentificationWarnings") + db.identification_warnings.create() + for file in db.files.select(): db.files.update(file) + return set_db_version(db, Version("2.0.0")) From c8ed5957c9ecd49a6281c931c8327eb59838ab01 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 12:55:49 +0200 Subject: [PATCH 04/13] pyproject.tool.ruff.lint.ignore - add SIM118 Use `key in dict` instead of `key in dict.keys()` --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index c6c1496..c080747 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,7 @@ ignore = [ "PT012", # ptest.raises should contain a simple statement "RET505", # unnecessary {branch} after return statement "S101", # use of assert, + "SIM118", # Use `key in dict` instead of `key in dict.keys()` "TRY003", # avoid using long messages outside exception class "UP007", # not using | in type anotations "INP001", # implicit namespace without __init__ (throws errors in tests) From 74f84805a22f1f3eba0aa5ff362e7eca973fb845 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 12:57:29 +0200 Subject: [PATCH 05/13] models.reference_files.Action - add "alternatives" property --- acacore/models/reference_files.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/acacore/models/reference_files.py b/acacore/models/reference_files.py index 314ef38..7ed36f5 100644 --- a/acacore/models/reference_files.py +++ b/acacore/models/reference_files.py @@ -1,11 +1,13 @@ """Data models for the data on saved to different .json files on the `reference_files` repo.""" +from re import match from typing import get_args as get_type_args from typing import Literal from pydantic import AliasChoices from pydantic import BaseModel from pydantic import Field +from pydantic import field_validator from .base import NoDefaultsModel @@ -208,9 +210,22 @@ class Action(ActionData): name: str description: str | None = None + alternatives: dict[str, str] = Field(default_factory=dict) action: TActionType ignore_warnings: list[str] = Field(default_factory=list, alias="ignore-warnings") + # noinspection PyNestedDecorators + @field_validator("alternatives", mode="before") + @classmethod + def _validate_alternatives(cls, value: dict[str, str]) -> dict[str, str]: + if not value: + return value + elif not all(isinstance(k, str) and match(r"^(\.[a-z0-9]+)+$", k) for k in value.keys()): + raise ValueError("Keys are not valid extensions '(\\.[a-z0-9]+)+'.") + elif not all(isinstance(v, str) and match(r"^[a-zA-Z0-9_/-]+$", v) for v in value.values()): + raise ValueError("Keys are not valid PUIDs '(\\.[a-z0-9]+)+'.") + return value + @property def action_data(self) -> ActionData: """ From 027f2840c145cec4f35fe8c61c0d2f3b61529343 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 12:58:44 +0200 Subject: [PATCH 06/13] models.reference_files.Action.ignore_warnings - allow for both "ignore_warnings" and "ignore-warnings" as name --- acacore/models/reference_files.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/acacore/models/reference_files.py b/acacore/models/reference_files.py index 7ed36f5..238b2ce 100644 --- a/acacore/models/reference_files.py +++ b/acacore/models/reference_files.py @@ -212,7 +212,10 @@ class Action(ActionData): description: str | None = None alternatives: dict[str, str] = Field(default_factory=dict) action: TActionType - ignore_warnings: list[str] = Field(default_factory=list, alias="ignore-warnings") + ignore_warnings: list[str] = Field( + default_factory=list, + validation_alias=AliasChoices("ignore_warnings", "ignore-warnings"), + ) # noinspection PyNestedDecorators @field_validator("alternatives", mode="before") From f6b3d723a936bb595a10fbc4ec4e1339e5139b0d Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 13:09:02 +0200 Subject: [PATCH 07/13] models.reference_files:Action - use lowercase for "alternatives" keys --- acacore/models/reference_files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/acacore/models/reference_files.py b/acacore/models/reference_files.py index 238b2ce..6adcc95 100644 --- a/acacore/models/reference_files.py +++ b/acacore/models/reference_files.py @@ -221,13 +221,13 @@ class Action(ActionData): @field_validator("alternatives", mode="before") @classmethod def _validate_alternatives(cls, value: dict[str, str]) -> dict[str, str]: - if not value: - return value + if not isinstance(value, dict): + raise ValueError("Is not a dictionary.") elif not all(isinstance(k, str) and match(r"^(\.[a-z0-9]+)+$", k) for k in value.keys()): raise ValueError("Keys are not valid extensions '(\\.[a-z0-9]+)+'.") elif not all(isinstance(v, str) and match(r"^[a-zA-Z0-9_/-]+$", v) for v in value.values()): raise ValueError("Keys are not valid PUIDs '(\\.[a-z0-9]+)+'.") - return value + return {k.lower(): v for k, v in value.items()} @property def action_data(self) -> ActionData: From 4963e2372d5c3be63a7d9be073414ae8f52678b8 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 13:12:06 +0200 Subject: [PATCH 08/13] models.reference_files:Action - remove unneeded elif --- acacore/models/reference_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acacore/models/reference_files.py b/acacore/models/reference_files.py index 6adcc95..7b94384 100644 --- a/acacore/models/reference_files.py +++ b/acacore/models/reference_files.py @@ -223,9 +223,9 @@ class Action(ActionData): def _validate_alternatives(cls, value: dict[str, str]) -> dict[str, str]: if not isinstance(value, dict): raise ValueError("Is not a dictionary.") - elif not all(isinstance(k, str) and match(r"^(\.[a-z0-9]+)+$", k) for k in value.keys()): + if not all(isinstance(k, str) and match(r"^(\.[a-z0-9]+)+$", k) for k in value.keys()): raise ValueError("Keys are not valid extensions '(\\.[a-z0-9]+)+'.") - elif not all(isinstance(v, str) and match(r"^[a-zA-Z0-9_/-]+$", v) for v in value.values()): + if not all(isinstance(v, str) and match(r"^[a-zA-Z0-9_/-]+$", v) for v in value.values()): raise ValueError("Keys are not valid PUIDs '(\\.[a-z0-9]+)+'.") return {k.lower(): v for k, v in value.items()} From 7b03c9178d2013a22cb06f9d9cf93377a6c3713c Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 13:30:30 +0200 Subject: [PATCH 09/13] models.file:File.get_action - support Action.alternatives --- acacore/models/file.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/acacore/models/file.py b/acacore/models/file.py index bd8d6e8..7009f36 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -276,6 +276,13 @@ def get_action( action: Action | None = reduce(lambda acc, cur: acc or actions.get(cur), identifiers, None) + if action and action.alternatives and (new_puid := action.alternatives.get(self.suffix.lower(), None)): + puid: str = self.puid + self.puid = new_puid + if new_action := self.get_action(actions, file_classes, set_match=set_match): + return new_action + self.puid = puid + if set_match: self.action, self.action_data = ( action.action if action else None, From faae3b5eccc710e21985c7ff6d9a01e1a3aba4af Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 13:30:57 +0200 Subject: [PATCH 10/13] database.files_db:FileDB.identification_warnings - ignore files with size 0 --- acacore/database/files_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acacore/database/files_db.py b/acacore/database/files_db.py index 4fb0d6a..f327e3b 100644 --- a/acacore/database/files_db.py +++ b/acacore/database/files_db.py @@ -102,7 +102,8 @@ def __init__( "_IdentificationWarnings", self.files, self.files.model, - f'"{self.files.name}".warning is not null or "{self.files.name}".puid is NULL', + f'("{self.files.name}".warning is not null or "{self.files.name}".puid is null)' + f' and "{self.files.name}".size != 0', ) self.checksum_count = self.create_view( "_ChecksumCount", From eeb35986d901a64044986c4989e6b96f377df937 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 13:34:20 +0200 Subject: [PATCH 11/13] database.upgrade:upgrade_2to2_0_2 - add upgrade function from 2.0.x to 2.0.2 --- acacore/database/upgrade.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acacore/database/upgrade.py b/acacore/database/upgrade.py index 076e100..0b43184 100644 --- a/acacore/database/upgrade.py +++ b/acacore/database/upgrade.py @@ -28,6 +28,8 @@ def set_db_version(db: FileDB, version: Version) -> Version: def get_upgrade_function(current_version: Version, latest_version: Version) -> Callable[[FileDB], Version]: if current_version < Version("2.0.0"): return upgrade_1to2 + elif current_version < Version("2.0.2"): + return upgrade_2to2_0_2 elif current_version < latest_version: return upgrade_last else: @@ -51,6 +53,12 @@ def upgrade_1to2(db: FileDB) -> Version: return set_db_version(db, Version("2.0.0")) +def upgrade_2to2_0_2(db: FileDB) -> Version: + db.execute("drop view if exists _IdentificationWarnings") + db.identification_warnings.create() + return set_db_version(db, Version("2.0.2")) + + def upgrade_last(db: FileDB) -> Version: db.init() return set_db_version(db, Version(__version__)) From 2bec779620c16ddc78cad6f52d5df1fd6d676c71 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 13:44:37 +0200 Subject: [PATCH 12/13] models.reference_files:Action - simplify validation of Action.alternatives --- acacore/models/reference_files.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/acacore/models/reference_files.py b/acacore/models/reference_files.py index 7b94384..2021cd2 100644 --- a/acacore/models/reference_files.py +++ b/acacore/models/reference_files.py @@ -1,6 +1,5 @@ """Data models for the data on saved to different .json files on the `reference_files` repo.""" -from re import match from typing import get_args as get_type_args from typing import Literal @@ -223,10 +222,6 @@ class Action(ActionData): def _validate_alternatives(cls, value: dict[str, str]) -> dict[str, str]: if not isinstance(value, dict): raise ValueError("Is not a dictionary.") - if not all(isinstance(k, str) and match(r"^(\.[a-z0-9]+)+$", k) for k in value.keys()): - raise ValueError("Keys are not valid extensions '(\\.[a-z0-9]+)+'.") - if not all(isinstance(v, str) and match(r"^[a-zA-Z0-9_/-]+$", v) for v in value.values()): - raise ValueError("Keys are not valid PUIDs '(\\.[a-z0-9]+)+'.") return {k.lower(): v for k, v in value.items()} @property From 3ace8f6e4cc8ca109aca16079ade6fb49b201186 Mon Sep 17 00:00:00 2001 From: Matteo Campinoti Date: Tue, 6 Aug 2024 14:25:35 +0200 Subject: [PATCH 13/13] version - patch 2.0.1 > 2.0.2 --- acacore/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acacore/__version__.py b/acacore/__version__.py index 159d48b..0309ae2 100644 --- a/acacore/__version__.py +++ b/acacore/__version__.py @@ -1 +1 @@ -__version__ = "2.0.1" +__version__ = "2.0.2" diff --git a/pyproject.toml b/pyproject.toml index c080747..9898795 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "acacore" -version = "2.0.1" +version = "2.0.2" description = "" authors = ["Matteo Campinoti "] license = "GPL-3.0"