Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

v3.0.2 #40

Merged
merged 16 commits into from
Aug 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
b52f2dd
models.reference_files:TActionType - remove "template"
MatteoCampinoti94 Aug 16, 2024
7c6b515
database.upgrade:upgrade - do not print versions
MatteoCampinoti94 Aug 19, 2024
870c48e
database.base:Table - add offset parameter to select methods
MatteoCampinoti94 Aug 20, 2024
2fbb10f
models.reference_files:TTemplateType - add "extracted-archive"
MatteoCampinoti94 Aug 20, 2024
0d42c55
database.upgrade - only use Connection methods to select and update d…
MatteoCampinoti94 Aug 20, 2024
ffecb9d
database.upgrade:is_latest - remove unneeded f-string
MatteoCampinoti94 Aug 20, 2024
946af91
database.base:View.create_statement - use * for select if view column…
MatteoCampinoti94 Aug 20, 2024
39cd76a
database.upgrade:set_db_version - fix incorrect parameter type
MatteoCampinoti94 Aug 20, 2024
d253b86
database.upgrade - ignore update statements without WHERE
MatteoCampinoti94 Aug 20, 2024
88f2612
database.upgrade:upgrade_3to3_0_2 - add function to upgrade from 3.0.…
MatteoCampinoti94 Aug 20, 2024
60b02bb
version - patch 3.0.1 > 3.0.2
MatteoCampinoti94 Aug 20, 2024
11e26c1
database.upgrade:upgrade_3to3_0_2 - fix WHERE for `_IdentificationWar…
MatteoCampinoti94 Aug 20, 2024
119e59f
database.upgrade:upgrade_2_0_2to3 - add `_IdentificationWarnings` reset
MatteoCampinoti94 Aug 20, 2024
35e8dcb
database.upgrade:get_upgrade_function - move below upgrade functions
MatteoCampinoti94 Aug 20, 2024
8334d38
database.upgrade:upgrade_last - remove unneeded function
MatteoCampinoti94 Aug 20, 2024
4320a15
database.files_db:FileDB.upgrade - run init after upgrade
MatteoCampinoti94 Aug 20, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion acacore/__version__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "3.0.1"
__version__ = "3.0.2"
45 changes: 22 additions & 23 deletions acacore/database/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,7 @@ def select(
columns: list[Column | SelectColumn] | None = None,
where: str | None = None,
order_by: list[tuple[str | Column, str]] | None = None,
offset: int | None = None,
limit: int | None = None,
parameters: list[Any | None] | None = None,
) -> Cursor:
Expand All @@ -293,6 +294,7 @@ def select(
:param where: A WHERE expression, defaults to None.
:param order_by: A list tuples containing one column (either as Column or string) and a sorting direction
("ASC", or "DESC"), defaults to None.
:param offset: The number of rows skip, defaults to None.
:param limit: The number of rows to limit the results to, defaults to None.
:param parameters: Values to substitute in the SELECT expression, both in the `where` and SelectColumn
statements, defaults to None.
Expand All @@ -316,6 +318,9 @@ def select(
order_statements = [f"{c.name if isinstance(c, Column) else c} {s}" for c, s in order_by]
statement += f" ORDER BY {','.join(order_statements)}"

if offset is not None:
statement += f" OFFSET {offset}"

if limit is not None:
statement += f" LIMIT {limit}"

Expand Down Expand Up @@ -432,6 +437,7 @@ def select(
model: Type[M] | None = None,
where: str | None = None,
order_by: list[tuple[str | Column, str]] | None = None,
offset: int | None = None,
limit: int | None = None,
parameters: list[Any] | None = None,
) -> ModelCursor[M]:
Expand All @@ -442,21 +448,14 @@ def select(
:param where: A WHERE expression, defaults to None.
:param order_by: A list tuples containing one column (either as Column or string) and a sorting direction
("ASC", or "DESC"), defaults to None.
:param offset: The number of rows skip, defaults to None.
:param limit: The number of rows to limit the results to, defaults to None.
:param parameters: Values to substitute in the SELECT expression, both in the `where` and SelectColumn
statements, defaults to None.
:return: A Cursor object wrapping the SQLite cursor returned by the SELECT transaction.
"""
return ModelCursor[M](
super()
.select(
model_to_columns(model or self.model),
where,
order_by,
limit,
parameters,
)
.cursor,
super().select(model_to_columns(model or self.model), where, order_by, offset, limit, parameters).cursor,
model or self.model,
self,
)
Expand Down Expand Up @@ -655,10 +654,15 @@ def create_statement(self, exist_ok: bool = True) -> str:

elements.append("AS")

select_names = [
f"{c.name} as {c.alias}" if c.alias else f"{on_table}.{c.name}"
for c in [SelectColumn.from_column(c) for c in self.columns]
]
if not any(isinstance(c, SelectColumn) for c in self.columns) and [c.name for c in self.columns] == [
c.name for c in self.on.columns
]:
select_names = ["*"]
else:
select_names = [
f"{c.name} as {c.alias}" if c.alias else f"{on_table}.{c.name}"
for c in [SelectColumn.from_column(c) for c in self.columns]
]

elements.append(
f"SELECT {','.join(select_names)} " f"FROM {on_table}",
Expand Down Expand Up @@ -695,6 +699,7 @@ def select(
columns: list[Column | SelectColumn] | None = None,
where: str | None = None,
order_by: list[tuple[str | Column, str]] | None = None,
offset: int | None = None,
limit: int | None = None,
parameters: list[Any] | None = None,
) -> Cursor:
Expand All @@ -705,6 +710,7 @@ def select(
:param where: A WHERE expression, defaults to None.
:param order_by: A list tuples containing one column (either as Column or string) and a sorting direction
("ASC", or "DESC"), defaults to None.
:param offset: The number of rows skip, defaults to None.
:param limit: The number of rows to limit the results to, defaults to None.
:param parameters: Values to substitute in the SELECT expression, both in the `where` and SelectColumn
statements, defaults to None.
Expand All @@ -724,7 +730,7 @@ def select(
)
for c in map(SelectColumn.from_column, self.columns)
]
return super().select(columns, where, order_by, limit, parameters)
return super().select(columns, where, order_by, offset, limit, parameters)

def insert(self, *_args, **_kwargs):
"""
Expand Down Expand Up @@ -793,19 +799,12 @@ def select(
model: Type[M] | None = None,
where: str | None = None,
order_by: list[tuple[str | Column, str]] | None = None,
offset: int | None = None,
limit: int | None = None,
parameters: list[Any] | None = None,
) -> ModelCursor[M]:
return ModelCursor[M](
super()
.select(
model_to_columns(model or self.model),
where,
order_by,
limit,
parameters,
)
.cursor,
super().select(model_to_columns(model or self.model), where, order_by, offset, limit, parameters).cursor,
model or self.model,
self,
)
Expand Down
1 change: 1 addition & 0 deletions acacore/database/files_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ def upgrade(self):
from acacore.database.upgrade import upgrade

upgrade(self)
self.init()

def is_empty(self) -> bool:
"""Check if the database contains any files."""
Expand Down
130 changes: 78 additions & 52 deletions acacore/database/upgrade.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from json import dumps
from json import loads
from sqlite3 import Connection
from sqlite3 import DatabaseError
from sqlite3 import Row
from typing import Any
Expand All @@ -9,55 +10,47 @@

from acacore.__version__ import __version__

from .files_db import FileDB

__all__ = [
"upgrade",
"is_latest",
]


def get_db_version(db: FileDB) -> Version:
return Version(db.metadata.select().version)
from .files_db import FileDB


def set_db_version(db: FileDB, version: Version) -> Version:
metadata = db.metadata.select()
metadata.version = str(version)
db.metadata.update(metadata)
db.commit()
return version
def get_db_version(conn: Connection) -> Version | None:
if res := conn.execute("select VALUE from Metadata where KEY like 'version'").fetchone():
return Version(res[0])
return None


def get_upgrade_function(current_version: Version, latest_version: Version) -> Callable[[FileDB], Version]:
if current_version < Version("2.0.0"):
return upgrade_1to2
elif current_version < Version("2.0.2"):
return upgrade_2to2_0_2
elif current_version < Version("3.0.0"):
return upgrade_2_0_2to3
elif current_version < latest_version:
return upgrade_last
else:
return lambda _: latest_version
def set_db_version(conn: Connection, version: Version) -> Version:
conn.execute("insert or replace into Metadata (KEY, VALUE) values (?, ?)", ("version", str(version)))
conn.commit()
return version


# noinspection SqlResolve
def upgrade_1to2(db: FileDB) -> Version:
def upgrade_1to2(conn: Connection) -> Version:
# Add "lock" column if not already present
if not db.execute("select 1 from pragma_table_info('Files') where name = 'lock'").fetchone():
db.execute("alter table Files add column lock boolean")
db.execute("update Files set lock = false")
if not conn.execute("select 1 from pragma_table_info('Files') where name = 'lock'").fetchone():
conn.execute("alter table Files add column lock boolean")
# noinspection SqlWithoutWhere
conn.execute("update Files set lock = false")
# Rename "replace" action to "template"
db.execute("update Files set action = 'template' where action = 'replace'")
conn.execute("update Files set action = 'template' where action = 'replace'")
# Ensure action_data is always a readable JSON
db.execute("update Files set action_data = '{}' where action_data is null or action_data = ''")
conn.execute("update Files set action_data = '{}' where action_data is null or action_data = ''")

# Reset _IdentificationWarnings view
db.execute("drop view if exists _IdentificationWarnings")
db.identification_warnings.create()
conn.execute("drop view if exists _IdentificationWarnings")
conn.execute(
"CREATE VIEW _IdentificationWarnings AS"
' SELECT * FROM Files WHERE "Files".warning is not null or "Files".puid is NULL;'
)

cursor = db.execute("select * from files where action_data != '{}'")
cursor = conn.execute("select * from files where action_data != '{}'")
cursor.row_factory = Row

for file in cursor:
Expand All @@ -66,22 +59,26 @@ def upgrade_1to2(db: FileDB) -> Version:
action_data["template"] = action_data.get("replace")
# Remove None and empty lists (default values)
action_data = {k: v for k, v in action_data.items() if v}
db.execute("update Files set action_data = ? where uuid = ?", [dumps(action_data), file["uuid"]])
conn.execute("update Files set action_data = ? where uuid = ?", [dumps(action_data), file["uuid"]])

db.commit()
conn.commit()

return set_db_version(db, Version("2.0.0"))
return set_db_version(conn, Version("2.0.0"))


def upgrade_2to2_0_2(db: FileDB) -> Version:
db.execute("drop view if exists _IdentificationWarnings")
db.identification_warnings.create()
db.commit()
return set_db_version(db, Version("2.0.2"))
# noinspection SqlResolve
def upgrade_2to2_0_2(conn: Connection) -> Version:
conn.execute("drop view if exists _IdentificationWarnings")
conn.execute(
"CREATE VIEW _IdentificationWarnings AS"
' SELECT * FROM Files WHERE "Files".warning is not null or "Files".puid is NULL;'
)
conn.commit()
return set_db_version(conn, Version("2.0.2"))


# noinspection SqlResolve
def upgrade_2_0_2to3(db: FileDB) -> Version:
def upgrade_2_0_2to3(conn: Connection) -> Version:
def convert_action_data(data: dict):
new_data: dict[str, Any] = {}

Expand Down Expand Up @@ -114,27 +111,56 @@ def convert_action_data(data: dict):
return new_data

# Add "parent" column if not already present
if not db.execute("select 1 from pragma_table_info('Files') where name = 'parent'").fetchone():
db.execute("alter table Files add column parent text")
db.execute("update Files set parent = null")
if not conn.execute("select 1 from pragma_table_info('Files') where name = 'parent'").fetchone():
conn.execute("alter table Files add column parent text")
# noinspection SqlWithoutWhere
conn.execute("update Files set parent = null")

cursor = db.execute("select * from Files where action_data != '{}'")
# Reset _IdentificationWarnings view
conn.execute("drop view if exists _IdentificationWarnings")
conn.execute(
"CREATE VIEW _IdentificationWarnings AS"
' SELECT * FROM Files WHERE "Files".warning is not null or "Files".puid is NULL;'
)

cursor = conn.execute("select * from Files where action_data != '{}'")
cursor.row_factory = Row

for file in cursor:
db.execute(
conn.execute(
"update Files set action_data = ? where uuid is ?",
[dumps(convert_action_data(loads(file["action_data"]))), file["uuid"]],
)

db.commit()
conn.commit()

return set_db_version(db, Version("3.0.0"))
return set_db_version(conn, Version("3.0.0"))


def upgrade_last(db: FileDB) -> Version:
db.init()
return set_db_version(db, Version(__version__))
# noinspection SqlResolve
def upgrade_3to3_0_2(conn: Connection) -> Version:
conn.execute("drop view if exists _IdentificationWarnings")
conn.execute(
"CREATE VIEW _IdentificationWarnings AS"
' SELECT * FROM Files WHERE ("Files".warning is not null or "Files".puid is null) and "Files".size != 0'
)
conn.commit()
return set_db_version(conn, Version("3.0.2"))


def get_upgrade_function(current_version: Version, latest_version: Version) -> Callable[[Connection], Version]:
if current_version < Version("2.0.0"):
return upgrade_1to2
elif current_version < Version("2.0.2"):
return upgrade_2to2_0_2
elif current_version < Version("3.0.0"):
return upgrade_2_0_2to3
elif current_version < Version("3.0.2"):
return upgrade_3to3_0_2
elif current_version < latest_version:
return lambda c: set_db_version(c, Version(__version__))
else:
return lambda _: latest_version


def is_latest(db: FileDB, *, raise_on_difference: bool = False) -> bool:
Expand All @@ -151,9 +177,11 @@ def is_latest(db: FileDB, *, raise_on_difference: bool = False) -> bool:
if not db.is_initialised(check_views=False, check_indices=False):
raise DatabaseError("Database is not initialised")

current_version: Version = get_db_version(db)
current_version: Version | None = get_db_version(db)
latest_version: Version = Version(__version__)

if not current_version:
raise DatabaseError("Database does not contain version information")
if current_version > latest_version:
raise DatabaseError(f"Database version is greater than latest version: {current_version} > {latest_version}")
if current_version < latest_version and raise_on_difference:
Expand Down Expand Up @@ -181,6 +209,4 @@ def upgrade(db: FileDB):

while current_version < latest_version:
update_function = get_upgrade_function(current_version, latest_version)
print(current_version, end=" ")
current_version = update_function(db)
print(current_version)
2 changes: 1 addition & 1 deletion acacore/models/reference_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
TActionType = Literal[
"convert",
"extract",
"template",
"manual",
"rename",
"ignore",
Expand All @@ -28,6 +27,7 @@
"duplicate",
"not-preservable",
"not-convertable",
"extracted-archive",
]

ActionTypeEnum: tuple[TActionType, ...] = get_type_args(TActionType)
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "acacore"
version = "3.0.1"
version = "3.0.2"
description = ""
authors = ["Matteo Campinoti <[email protected]>"]
license = "GPL-3.0"
Expand Down
Loading