From 276da8511398e59e7bf629f1864fcb5698ab899a Mon Sep 17 00:00:00 2001 From: Magnus Lindholm Date: Tue, 29 Aug 2023 14:27:28 +0200 Subject: [PATCH 1/5] fixed some small type hinting mistake --- acacore/database/base.py | 360 ++++++++++++++++++++++++----------- acacore/database/column.py | 65 ++++--- acacore/database/files_db.py | 78 ++++++-- acacore/exceptions/files.py | 3 + acacore/models/file.py | 4 +- acacore/utils/log.py | 4 +- 6 files changed, 356 insertions(+), 158 deletions(-) diff --git a/acacore/database/base.py b/acacore/database/base.py index fbb2795..bb92c3a 100644 --- a/acacore/database/base.py +++ b/acacore/database/base.py @@ -26,8 +26,12 @@ class Cursor: - def __init__(self, cursor: SQLiteCursor, columns: list[Union[Column, SelectColumn]], - table: Optional['Table'] = None): + def __init__( + self, + cursor: SQLiteCursor, + columns: list[Union[Column, SelectColumn]], + table: Optional["Table"] = None, + ): """ A wrapper class for an SQLite cursor that returns its results as dicts (or objects). @@ -67,7 +71,11 @@ def fetchonetuple(self) -> Optional[tuple]: """ vs: tuple = self.cursor.fetchone() - return tuple(c.from_entry(v) for c, v in zip(self.columns, vs, strict=True)) if vs else None + return ( + tuple(c.from_entry(v) for c, v in zip(self.columns, vs, strict=True)) + if vs + else None + ) @overload def fetchall(self) -> Generator[dict[str, Any], None, None]: @@ -77,7 +85,9 @@ def fetchall(self) -> Generator[dict[str, Any], None, None]: def fetchall(self, model: Type[M]) -> Generator[M, None, None]: ... - def fetchall(self, model: Optional[Type[M]] = None) -> Generator[Union[dict[str, Any], M], None, None]: + def fetchall( + self, model: Optional[Type[M]] = None + ) -> Generator[Union[dict[str, Any], M], None, None]: """ Fetch all results from the cursor and return them as dicts, with the columns' names/aliases used as keys. @@ -87,14 +97,18 @@ def fetchall(self, model: Optional[Type[M]] = None) -> Generator[Union[dict[str, Returns: Generator: A generator for converted dicts (or models). """ - select_columns: list[SelectColumn] = [SelectColumn.from_column(c) for c in self.columns] + select_columns: list[SelectColumn] = [ + SelectColumn.from_column(c) for c in self.columns + ] if model: return ( - model.model_validate({ - c.alias or c.name: v - for c, v in zip(select_columns, vs, strict=True) - }) + model.model_validate( + { + c.alias or c.name: v + for c, v in zip(select_columns, vs, strict=True) + } + ) for vs in self.cursor.fetchall() ) @@ -114,7 +128,9 @@ def fetchone(self) -> Optional[dict[str, Any]]: def fetchone(self, model: Type[M]) -> Optional[M]: ... - def fetchone(self, model: Optional[Type[M]] = None) -> Optional[Union[dict[str, Any], M]]: + def fetchone( + self, model: Optional[Type[M]] = None + ) -> Optional[Union[dict[str, Any], M]]: """ Fetch one result from the cursor and return it as a dict, with the columns' names/aliases used as keys. @@ -124,22 +140,25 @@ def fetchone(self, model: Optional[Type[M]] = None) -> Optional[Union[dict[str, Returns: dict: A single dict (or model) if the cursor is not exhausted, otherwise None. """ - select_columns: list[SelectColumn] = [SelectColumn.from_column(c) for c in self.columns] + select_columns: list[SelectColumn] = [ + SelectColumn.from_column(c) for c in self.columns + ] vs: tuple = self.cursor.fetchone() if vs is None: return None entry: dict[str, Any] = { - c.name: c.from_entry(v) - for c, v in zip(select_columns, vs, strict=True) + c.name: c.from_entry(v) for c, v in zip(select_columns, vs, strict=True) } return model.model_validate(entry) if model else entry class ModelCursor(Cursor, Generic[M]): - def __init__(self, cursor: SQLiteCursor, model: Type[M], table: Optional['Table'] = None): + def __init__( + self, cursor: SQLiteCursor, model: Type[M], table: Optional["Table"] = None + ): """ A wrapper class for an SQLite cursor that returns its results as model objects. @@ -184,7 +203,7 @@ def fetchone(self, model: Optional[Type[M]] = None) -> Optional[M]: # noinspection SqlNoDataSourceInspection class Table: - def __init__(self, connection: 'FileDBBase', name: str, columns: list[Column]): + def __init__(self, connection: "FileDBBase", name: str, columns: list[Column]): """ A class that holds information about a table. @@ -193,7 +212,7 @@ def __init__(self, connection: 'FileDBBase', name: str, columns: list[Column]): name: The name of the table. columns: The columns of the table. """ - self.connection: 'FileDBBase' = connection + self.connection: "FileDBBase" = connection self.name: str = name self.columns: list[Column] = columns @@ -201,7 +220,9 @@ def __repr__(self): return f'{self.__class__.__name__}("{self.name}")' def __len__(self) -> int: - return self.connection.execute(f"select count(*) from {self.name}").fetchone()[0] + return self.connection.execute(f"select count(*) from {self.name}").fetchone()[ + 0 + ] def __iter__(self) -> Generator[dict[str, Any], None, None]: return self.select().fetchall() @@ -235,10 +256,14 @@ def create_statement(self, exist_ok: bool = True) -> str: if self.columns: elements.append( - "(" + - ", ".join(c.create_statement() for c in self.columns) + - (f", primary key ({', '.join(c.name for c in keys)})" if (keys := self.keys) else "") + - ")" + "(" + + ", ".join(c.create_statement() for c in self.columns) + + ( + f", primary key ({', '.join(c.name for c in keys)})" + if (keys := self.keys) + else "" + ) + + ")" ) return " ".join(elements) @@ -246,11 +271,14 @@ def create_statement(self, exist_ok: bool = True) -> str: def create(self, exist_ok: bool = True): self.connection.execute(self.create_statement(exist_ok)) - def select(self, columns: Optional[list[Union[Column, SelectColumn]]] = None, - where: Optional[str] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, - limit: Optional[int] = None, - parameters: Optional[list[Any]] = None) -> Cursor: + def select( + self, + columns: Optional[list[Union[Column, SelectColumn]]] = None, + where: Optional[str] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + parameters: Optional[list[Any]] = None, + ) -> Cursor: """ Select entries from the table. @@ -270,10 +298,12 @@ def select(self, columns: Optional[list[Union[Column, SelectColumn]]] = None, assert columns, "Columns cannot be empty" - select_columns: list[SelectColumn] = [SelectColumn.from_column(c) for c in columns] + select_columns: list[SelectColumn] = [ + SelectColumn.from_column(c) for c in columns + ] select_names = [ - '{} as {}'.format(c.name, c.alias) if c.alias else c.name + "{} as {}".format(c.name, c.alias) if c.alias else c.name for c in select_columns ] @@ -284,8 +314,7 @@ def select(self, columns: Optional[list[Union[Column, SelectColumn]]] = None, if order_by: order_statements = [ - f"{c.name if isinstance(c, Column) else c} {s}" - for c, s in order_by + f"{c.name if isinstance(c, Column) else c} {s}" for c, s in order_by ] statement += f" ORDER BY {','.join(order_statements)}" @@ -294,7 +323,9 @@ def select(self, columns: Optional[list[Union[Column, SelectColumn]]] = None, return Cursor(self.connection.execute(statement, parameters), columns, self) - def insert(self, entry: dict[str, Any], exist_ok: bool = False, replace: bool = False): + def insert( + self, entry: dict[str, Any], exist_ok: bool = False, replace: bool = False + ): """ Insert a row in the table. Existing rows with matching keys can be ignored or replaced. @@ -322,7 +353,7 @@ def insert(self, entry: dict[str, Any], exist_ok: bool = False, replace: bool = class ModelTable(Table, Generic[M]): - def __init__(self, connection: 'FileDBBase', name: str, model: Type[M]): + def __init__(self, connection: "FileDBBase", name: str, model: Type[M]): """ A class that holds information about a table using a model. @@ -340,11 +371,14 @@ def __repr__(self): def __iter__(self) -> Generator[M, None, None]: return self.select().fetchall() - def select(self, model: Type[M] = None, - where: Optional[str] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, - limit: Optional[int] = None, - parameters: Optional[list[Any]] = None) -> ModelCursor[M]: + def select( + self, + model: Type[M] = None, + where: Optional[str] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + parameters: Optional[list[Any]] = None, + ) -> ModelCursor[M]: """ Select entries from the table. @@ -360,8 +394,17 @@ def select(self, model: Type[M] = None, Cursor: A Cursor object wrapping the SQLite cursor returned by the SELECT transaction. """ return ModelCursor[M]( - super().select(model_to_columns(model or self.model), where, order_by, limit, parameters).cursor, - model or self.model, self + super() + .select( + model_to_columns(model or self.model), + where, + order_by, + limit, + parameters, + ) + .cursor, + model or self.model, + self, ) def insert(self, entry: M, exist_ok: bool = False, replace: bool = False): @@ -378,10 +421,17 @@ def insert(self, entry: M, exist_ok: bool = False, replace: bool = False): # noinspection SqlNoDataSourceInspection class View(Table): - def __init__(self, connection: 'FileDBBase', name: str, on: Union[Table, str], - columns: list[Union[Column, SelectColumn]], where: Optional[str] = None, - group_by: Optional[list[Union[Column, SelectColumn]]] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, limit: Optional[int] = None): + def __init__( + self, + connection: "FileDBBase", + name: str, + on: Union[Table, str], + columns: list[Union[Column, SelectColumn]], + where: Optional[str] = None, + group_by: Optional[list[Union[Column, SelectColumn]]] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + ): """ A subclass of Table to handle views. @@ -427,7 +477,7 @@ def create_statement(self, exist_ok: bool = True) -> str: elements.append("AS") select_names = [ - f'{c.name} as {c.alias}' if c.alias else c.name + f"{c.name} as {c.alias}" if c.alias else c.name for c in [SelectColumn.from_column(c) for c in self.columns] ] @@ -441,10 +491,14 @@ def create_statement(self, exist_ok: bool = True) -> str: if self.group_by: elements.append("GROUP BY") - elements.append(",".join([ - c.alias or c.name - for c in [SelectColumn.from_column(c) for c in self.group_by] - ])) + elements.append( + ",".join( + [ + c.alias or c.name + for c in [SelectColumn.from_column(c) for c in self.group_by] + ] + ) + ) if self.order_by: order_statements = [ @@ -458,11 +512,14 @@ def create_statement(self, exist_ok: bool = True) -> str: return " ".join(elements) - def select(self, columns: Optional[list[Union[Column, SelectColumn]]] = None, - where: Optional[str] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, - limit: Optional[int] = None, - parameters: Optional[list[Any]] = None) -> Cursor: + def select( + self, + columns: Optional[list[Union[Column, SelectColumn]]] = None, + where: Optional[str] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + parameters: Optional[list[Any]] = None, + ) -> Cursor: """ Select entries from the view. @@ -478,8 +535,17 @@ def select(self, columns: Optional[list[Union[Column, SelectColumn]]] = None, Cursor: A Cursor object wrapping the SQLite cursor returned by the SELECT transaction. """ columns = columns or [ - Column(c.alias or c.name, c.sql_type, c.to_entry, c.from_entry, c.unique, c.primary_key, c.not_null, - c.check, c.default) + Column( + c.alias or c.name, + c.sql_type, + c.to_entry, + c.from_entry, + c.unique, + c.primary_key, + c.not_null, + c.check, + c.default, + ) for c in map(SelectColumn.from_column, self.columns) ] return super().select(columns, where, order_by, limit, parameters) @@ -493,47 +559,84 @@ def insert(self, *_args, **_kwargs): class ModelView(View, Generic[M]): - def __init__(self, connection: 'FileDBBase', name: str, on: Union[Table, str], model: Type[M], - columns: list[Union[Column, SelectColumn]] = None, where: Optional[str] = None, - group_by: Optional[list[Union[Column, SelectColumn]]] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, limit: Optional[int] = None): - """ - A subclass of Table to handle views with models. - - Args: - connection: A FileDBBase object connected to the database the view belongs to. - name: The name of the table. - on: The table the view is based on. - model: A BaseModel subclass. - columns: Optionally, the columns of the view if the model is too limited. - where: A WHERE expression for the view. - group_by: A GROUP BY expression for the view. - order_by: A list tuples containing one column (either as Column or string) - and a sorting direction ("ASC", or "DESC"). - limit: The number of rows to limit the results to. - """ - super().__init__(connection, name, on, columns or model_to_columns(model), where, group_by, order_by, limit) + def __init__( + self, + connection: "FileDBBase", + name: str, + on: Union[Table, str], + model: Type[M], + columns: list[Union[Column, SelectColumn]] = None, + where: Optional[str] = None, + group_by: Optional[list[Union[Column, SelectColumn]]] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + ): + """ + A subclass of Table to handle views with models. + + Args: + connection: A FileDBBase object connected to the database the view belongs to. + name: The name of the table. + on: The table the view is based on. + model: A BaseModel subclass. + columns: Optionally, the columns of the view if the model is too limited. + where: A WHERE expression for the view. + group_by: A GROUP BY expression for the view. + order_by: A list tuples containing one column (either as Column or string) + and a sorting direction ("ASC", or "DESC"). + limit: The number of rows to limit the results to. + """ + super().__init__( + connection, + name, + on, + columns or model_to_columns(model), + where, + group_by, + order_by, + limit, + ) self.model: Type[M] = model def __repr__(self): return f'{self.__class__.__name__}[{self.model.__name__}]("{self.name}", on={self.on!r})' - def select(self, model: Type[M] = None, - where: Optional[str] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, - limit: Optional[int] = None, - parameters: Optional[list[Any]] = None) -> ModelCursor[M]: + def select( + self, + model: Type[M] = None, + where: Optional[str] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + parameters: Optional[list[Any]] = None, + ) -> ModelCursor[M]: return ModelCursor[M]( - super().select(model_to_columns(model or self.model), where, order_by, limit, parameters).cursor, - model or self.model, self + super() + .select( + model_to_columns(model or self.model), + where, + order_by, + limit, + parameters, + ) + .cursor, + model or self.model, + self, ) class FileDBBase(Connection): - def __init__(self, database: str | bytes | PathLike[str] | PathLike[bytes], *, - timeout: float = 5.0, - detect_types: int = 0, isolation_level: Optional[str] = 'DEFERRED', check_same_thread: bool = True, - factory: Optional[Type[Connection]] = Connection, cached_statements: int = 100, uri: bool = False): + def __init__( + self, + database: Union[str, bytes, PathLike[str], PathLike[bytes]], + *, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Optional[str] = "DEFERRED", + check_same_thread: bool = True, + factory: Optional[Type[Connection]] = Connection, + cached_statements: int = 100, + uri: bool = False, + ): """ A wrapper class for an SQLite connection. @@ -553,8 +656,16 @@ def __init__(self, database: str | bytes | PathLike[str] | PathLike[bytes], *, to avoid parsing overhead. uri: If set to True, database is interpreted as a URI with a file path and an optional query string. """ - super().__init__(database, timeout, detect_types, isolation_level, check_same_thread, factory, - cached_statements, uri) + super().__init__( + database, + timeout, + detect_types, + isolation_level, + check_same_thread, + factory, + cached_statements, + uri, + ) def __repr__(self): return f"{self.__class__.__name__}({self.path})" @@ -575,7 +686,9 @@ def create_table(self, name: str, columns: Type[M]) -> ModelTable[M]: def create_table(self, name: str, columns: list[Column]) -> Table: ... - def create_table(self, name: str, columns: Union[Type[M], list[Column]]) -> Union[Table, ModelTable[M]]: + def create_table( + self, name: str, columns: Union[Type[M], list[Column]] + ) -> Union[Table, ModelTable[M]]: """ Create a table in the database. When the `columns` argument is a subclass of BadeModel, a ModelTable object is returned. @@ -590,34 +703,45 @@ def create_table(self, name: str, columns: Union[Type[M], list[Column]]) -> Unio return Table(self, name, columns) @overload - def create_view(self, name: str, on: Union[Table, str], - columns: Type[M], - where: Optional[str] = None, - group_by: Optional[list[Union[Column, SelectColumn]]] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, - limit: Optional[int] = None, - *, select_columns: Optional[list[Union[Column, SelectColumn]]] = None - ) -> ModelView[M]: + def create_view( + self, + name: str, + on: Union[Table, str], + columns: Type[M], + where: Optional[str] = None, + group_by: Optional[list[Union[Column, SelectColumn]]] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + *, + select_columns: Optional[list[Union[Column, SelectColumn]]] = None, + ) -> ModelView[M]: ... @overload - def create_view(self, name: str, on: Union[Table, str], - columns: list[Union[Column, SelectColumn]], - where: Optional[str] = None, - group_by: Optional[list[Union[Column, SelectColumn]]] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, - limit: Optional[int] = None - ) -> View: + def create_view( + self, + name: str, + on: Union[Table, str], + columns: list[Union[Column, SelectColumn]], + where: Optional[str] = None, + group_by: Optional[list[Union[Column, SelectColumn]]] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + ) -> View: ... - def create_view(self, name: str, on: Union[Table, str], - columns: Union[list[Union[Column, SelectColumn]], Type[M]], - where: Optional[str] = None, - group_by: Optional[list[Union[Column, SelectColumn]]] = None, - order_by: Optional[list[tuple[Union[str, Column], str]]] = None, - limit: Optional[int] = None, - *, select_columns: Optional[list[Union[Column, SelectColumn]]] = None - ) -> Union[View, ModelView[M]]: + def create_view( + self, + name: str, + on: Union[Table, str], + columns: Union[list[Union[Column, SelectColumn]], Type[M]], + where: Optional[str] = None, + group_by: Optional[list[Union[Column, SelectColumn]]] = None, + order_by: Optional[list[tuple[Union[str, Column], str]]] = None, + limit: Optional[int] = None, + *, + select_columns: Optional[list[Union[Column, SelectColumn]]] = None, + ) -> Union[View, ModelView[M]]: """ Create a view in the database. When the `columns` argument is a subclass of BadeModel, a ModelView object is returned. @@ -634,6 +758,16 @@ def create_view(self, name: str, on: Union[Table, str], select_columns: Optionally, the columns of the view if a model is given and is too limited. """ if issubclass(columns, BaseModel): - return ModelView[M](self, name, on, columns, select_columns, where, group_by, order_by, limit) + return ModelView[M]( + self, + name, + on, + columns, + select_columns, + where, + group_by, + order_by, + limit, + ) else: return View(self, name, on, columns, where, group_by, order_by, limit) diff --git a/acacore/database/column.py b/acacore/database/column.py index c455fec..7ae6f46 100644 --- a/acacore/database/column.py +++ b/acacore/database/column.py @@ -22,7 +22,9 @@ "null": "text", } -_sql_schema_type_converters: dict[str, tuple[Callable[[Optional[T]], V], Callable[[V], Optional[T]]]] = { +_sql_schema_type_converters: dict[ + str, tuple[Callable[[Optional[T]], V], Callable[[V], Optional[T]]] +] = { "path": (str, Path), "date-time": (datetime.isoformat, datetime.fromisoformat), "uuid4": (str, UUID), @@ -35,7 +37,7 @@ } -def _schema_to_column(name: str, schema: dict) -> 'Column': +def _schema_to_column(name: str, schema: dict) -> "Column": schema_type: Optional[str] = schema.get("type", None) schema_any_of: list[dict] = schema.get("anyOf", []) @@ -63,23 +65,37 @@ def _schema_to_column(name: str, schema: dict) -> 'Column': raise TypeError(f"Cannot recognize type from schema {schema!r}") return Column( - name, sql_type, lambda x: None if x is None else to_entry(x), lambda x: None if x is None else from_entry(x), + name, + sql_type, + lambda x: None if x is None else to_entry(x), + lambda x: None if x is None else from_entry(x), unique=schema.get("default", False), primary_key=schema.get("primary_key", False), not_null=not_null, - default=schema.get("default", ...) + default=schema.get("default", ...), ) -def model_to_columns(model: Type[BaseModel]) -> list['Column']: - return [_schema_to_column(p, s) for p, s in model.model_json_schema()["properties"].items()] +def model_to_columns(model: Type[BaseModel]) -> list["Column"]: + return [ + _schema_to_column(p, s) + for p, s in model.model_json_schema()["properties"].items() + ] class Column(Generic[T]): - def __init__(self, name: str, sql_type: str, - to_entry: Callable[[T], V], from_entry: Callable[[V], T], - unique: bool = False, primary_key: bool = False, not_null: bool = False, - check: Optional[str] = None, default: Optional[T] = ...): + def __init__( + self, + name: str, + sql_type: str, + to_entry: Callable[[T], V], + from_entry: Callable[[V], T], + unique: bool = False, + primary_key: bool = False, + not_null: bool = False, + check: Optional[str] = None, + default: Optional[T] = ..., + ): """ A class that stores information regarding a table column. @@ -109,17 +125,19 @@ def __init__(self, name: str, sql_type: str, self.default: Union[Optional[T], Ellipsis] = default def __repr__(self): - return (f"{self.__class__.__name__}(" - f"{self.name}" - f", {self.sql_type!r}" - f", unique={self.unique}" - f", primary_key={self.primary_key}" - f", not_null={self.not_null}" - f"{f', default={repr(self.default)}' if self.default is not Ellipsis else ''}" - f")") + return ( + f"{self.__class__.__name__}(" + f"{self.name}" + f", {self.sql_type!r}" + f", unique={self.unique}" + f", primary_key={self.primary_key}" + f", not_null={self.not_null}" + f"{f', default={repr(self.default)}' if self.default is not Ellipsis else ''}" + f")" + ) @classmethod - def from_model(cls, model: Type[BaseModel]) -> list['Column']: + def from_model(cls, model: Type[BaseModel]) -> list["Column"]: return model_to_columns(model) @property @@ -145,7 +163,8 @@ def create_statement(self) -> str: if self.default is not Ellipsis: default_value: V = self.to_entry(self.default) elements.append( - "default '{}'".format(default_value.replace("'", "\\'")) if isinstance(default_value, str) + "default '{}'".format(default_value.replace("'", "\\'")) + if isinstance(default_value, str) else f"default {'null' if default_value is None else default_value}" ) if self.check: @@ -155,7 +174,9 @@ def create_statement(self) -> str: class SelectColumn(Column): - def __init__(self, name: str, from_entry: Callable[[V], T], alias: Optional[str] = None): + def __init__( + self, name: str, from_entry: Callable[[V], T], alias: Optional[str] = None + ): """ A subclass of Column for SELECT expressions that need complex statements and/or an alias. @@ -170,7 +191,7 @@ def __init__(self, name: str, from_entry: Callable[[V], T], alias: Optional[str] self.alias: Optional[str] = alias @classmethod - def from_column(cls, column: Column, alias: Optional[str] = None) -> 'SelectColumn': + def from_column(cls, column: Column, alias: Optional[str] = None) -> "SelectColumn": """ Take a Column object and create a SelectColumn with the given alias. diff --git a/acacore/database/files_db.py b/acacore/database/files_db.py index af9332c..c0933fd 100644 --- a/acacore/database/files_db.py +++ b/acacore/database/files_db.py @@ -10,9 +10,18 @@ class FileDB(FileDBBase): - def __init__(self, database: str | bytes | PathLike[str] | PathLike[bytes], *, timeout: float = 5.0, - detect_types: int = 0, isolation_level: Optional[str] = 'DEFERRED', check_same_thread: bool = True, - factory: Optional[Type[Connection]] = Connection, cached_statements: int = 100, uri: bool = False): + def __init__( + self, + database: str | bytes | PathLike[str] | PathLike[bytes], + *, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Optional[str] = "DEFERRED", + check_same_thread: bool = True, + factory: Optional[Type[Connection]] = Connection, + cached_statements: int = 100, + uri: bool = False, + ): """ A class that handles the SQLite database used by AArhus City Archives to process data archives. @@ -36,21 +45,39 @@ def __init__(self, database: str | bytes | PathLike[str] | PathLike[bytes], *, t from ..models.identification import SignatureCount from ..models.metadata import Metadata - super().__init__(database, timeout=timeout, detect_types=detect_types, isolation_level=isolation_level, - check_same_thread=check_same_thread, factory=factory, cached_statements=cached_statements, - uri=uri) + super().__init__( + database, + timeout=timeout, + detect_types=detect_types, + isolation_level=isolation_level, + check_same_thread=check_same_thread, + factory=factory, + cached_statements=cached_statements, + uri=uri, + ) self.files = self.create_table("Files", File) self.metadata = self.create_table("Metadata", Metadata) self.converted_files = self.create_table("_ConvertedFiles", ConvertedFile) - self.not_converted = self.create_view("_NotConverted", self.files, self.files.model, - f'"{self.files.name}".uuid NOT IN ' - f'(SELECT uuid from {self.converted_files.name})') - self.identification_warnings = self.create_view("_IdentificationWarnings", self.files, self.files.model, - f'"{self.files.name}".warning IS NOT null') + self.not_converted = self.create_view( + "_NotConverted", + self.files, + self.files.model, + f'"{self.files.name}".uuid NOT IN ' + f"(SELECT uuid from {self.converted_files.name})", + ) + self.identification_warnings = self.create_view( + "_IdentificationWarnings", + self.files, + self.files.model, + f'"{self.files.name}".warning IS NOT null', + ) self.signature_count = self.create_view( - "_SignatureCount", self.files, SignatureCount, None, + "_SignatureCount", + self.files, + SignatureCount, + None, [ Column("puid", "varchar", str, str, False, False, False), ], @@ -58,16 +85,29 @@ def __init__(self, database: str | bytes | PathLike[str] | PathLike[bytes], *, t (Column("count", "int", str, str), "ASC"), ], select_columns=[ - Column("puid", "varchar", or_none(str), or_none(str), False, False, False), - Column("signature", "varchar", or_none(str), or_none(str), False, False, False), + Column( + "puid", "varchar", or_none(str), or_none(str), False, False, False + ), + Column( + "signature", + "varchar", + or_none(str), + or_none(str), + False, + False, + False, + ), SelectColumn( - f'count(' + f"count(" f'CASE WHEN ("{self.files.name}".puid IS NULL) ' - f'THEN \'None\' ' + f"THEN 'None' " f'ELSE "{self.files.name}".puid ' - f'END)', - int, "count") - ]) + f"END)", + int, + "count", + ), + ], + ) def init(self): """ diff --git a/acacore/exceptions/files.py b/acacore/exceptions/files.py index b0a14bc..d8748dc 100644 --- a/acacore/exceptions/files.py +++ b/acacore/exceptions/files.py @@ -3,14 +3,17 @@ class IdentificationError(ACAException): """Implements an error to raise if identification or related functionality fails.""" + pass class FileCollectionError(ACAException): """Implements an error to raise if File discovery/collection or related functionality fails.""" + pass class FileParseError(ACAException): """Implements an error to raise if file parsing fails.""" + pass diff --git a/acacore/models/file.py b/acacore/models/file.py index f12e4b4..ea6bced 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -30,7 +30,9 @@ class File(ACABase): warning: Optional[str] def get_absolute_path(self, root: Optional[Path] = None) -> Path: - return root.joinpath(self.relative_path) if root else self.relative_path.resolve() + return ( + root.joinpath(self.relative_path) if root else self.relative_path.resolve() + ) def read_text(self) -> str: """Expose read text functionality from pathlib. diff --git a/acacore/utils/log.py b/acacore/utils/log.py index 91d3b76..e0c35b4 100644 --- a/acacore/utils/log.py +++ b/acacore/utils/log.py @@ -19,9 +19,7 @@ def setup_logger(log_name: str, log_path: Path) -> Logger: Path.mkdir(log_path.parent, parents=True, exist_ok=True) log: Logger = getLogger(log_name) - file_handler: FileHandler = FileHandler( - log_path, "a", encoding="utf-8" - ) + file_handler: FileHandler = FileHandler(log_path, "a", encoding="utf-8") # noinspection SpellCheckingInspection log_fmt: Formatter = Formatter( fmt="%(asctime)s %(levelname)s: %(message)s", From 2c978a5a06fdf89a9a4644383738e2d6f9ae86c6 Mon Sep 17 00:00:00 2001 From: Magnus Lindholm Date: Tue, 29 Aug 2023 14:39:01 +0200 Subject: [PATCH 2/5] Revert "models:file - add status field to ConvertedFile" This reverts commit 4804dd92faac706772e98db450a557bcdf8ce8e8. --- acacore/models/file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/acacore/models/file.py b/acacore/models/file.py index ea6bced..10aa3b0 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -104,4 +104,3 @@ class ArchiveFile(Identification, File): class ConvertedFile(ACABase): file_id: int = Field(primary_key=True) uuid: UUID4 = Field(primary_key=True) - status: str From 9009151a5465258c44fa8361e9da5d73145dcaf8 Mon Sep 17 00:00:00 2001 From: Magnus Lindholm Date: Tue, 29 Aug 2023 14:44:15 +0200 Subject: [PATCH 3/5] Fix:_ Re-added status field to convertedfile (was removed in a revert --- acacore/models/file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acacore/models/file.py b/acacore/models/file.py index 10aa3b0..eb84dcf 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -104,3 +104,4 @@ class ArchiveFile(Identification, File): class ConvertedFile(ACABase): file_id: int = Field(primary_key=True) uuid: UUID4 = Field(primary_key=True) + status: str \ No newline at end of file From 20712b800257a8d370ae5eafa14e030761900a4d Mon Sep 17 00:00:00 2001 From: Magnus Lindholm Date: Tue, 29 Aug 2023 14:47:39 +0200 Subject: [PATCH 4/5] merged main --- acacore/database/files_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acacore/database/files_db.py b/acacore/database/files_db.py index c0933fd..d3e8689 100644 --- a/acacore/database/files_db.py +++ b/acacore/database/files_db.py @@ -1,6 +1,6 @@ from os import PathLike from sqlite3 import Connection -from typing import Optional +from typing import Optional, Union from typing import Type from .base import Column @@ -12,7 +12,7 @@ class FileDB(FileDBBase): def __init__( self, - database: str | bytes | PathLike[str] | PathLike[bytes], + database: Union[str, bytes, PathLike[str], PathLike[bytes]], *, timeout: float = 5.0, detect_types: int = 0, From 0502af7b5f5830497b33f376e2c28e532980cca5 Mon Sep 17 00:00:00 2001 From: Magnus Lindholm Date: Tue, 29 Aug 2023 14:59:19 +0200 Subject: [PATCH 5/5] linting --- acacore/database/base.py | 36 +++++++++--------------------------- acacore/database/column.py | 5 +---- acacore/models/file.py | 6 ++---- 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/acacore/database/base.py b/acacore/database/base.py index 21b121c..55fa5c7 100644 --- a/acacore/database/base.py +++ b/acacore/database/base.py @@ -21,7 +21,7 @@ def __init__( cursor: SQLiteCursor, columns: list[Union[Column, SelectColumn]], table: Optional["Table"] = None, - ): + ) -> None: """ A wrapper class for an SQLite cursor that returns its results as dicts (or objects). @@ -58,11 +58,7 @@ def fetchonetuple(self) -> Optional[tuple]: """ vs: tuple = self.cursor.fetchone() - return ( - tuple(c.from_entry(v) for c, v in zip(self.columns, vs, strict=True)) - if vs - else None - ) + return tuple(c.from_entry(v) for c, v in zip(self.columns, vs, strict=True)) if vs else None @overload def fetchall(self) -> Generator[dict[str, Any], None, None]: @@ -72,9 +68,7 @@ def fetchall(self) -> Generator[dict[str, Any], None, None]: def fetchall(self, model: Type[M]) -> Generator[M, None, None]: ... - def fetchall( - self, model: Optional[Type[M]] = None - ) -> Generator[Union[dict[str, Any], M], None, None]: + def fetchall(self, model: Optional[Type[M]] = None) -> Generator[Union[dict[str, Any], M], None, None]: """ Fetch all results from the cursor and return them as dicts, with the columns' names/aliases used as keys. @@ -84,9 +78,7 @@ def fetchall( Returns: Generator: A generator for converted dicts (or models). """ - select_columns: list[SelectColumn] = [ - SelectColumn.from_column(c) for c in self.columns - ] + select_columns: list[SelectColumn] = [SelectColumn.from_column(c) for c in self.columns] if model: return ( @@ -122,17 +114,13 @@ def fetchone( Returns: dict: A single dict (or model) if the cursor is not exhausted, otherwise None. """ - select_columns: list[SelectColumn] = [ - SelectColumn.from_column(c) for c in self.columns - ] + select_columns: list[SelectColumn] = [SelectColumn.from_column(c) for c in self.columns] vs: tuple = self.cursor.fetchone() if vs is None: return None - entry: dict[str, Any] = { - c.name: c.from_entry(v) for c, v in zip(select_columns, vs, strict=True) - } + entry: dict[str, Any] = {c.name: c.from_entry(v) for c, v in zip(select_columns, vs, strict=True)} return model.model_validate(entry) if model else entry @@ -205,9 +193,7 @@ def __repr__(self) -> str: return f'{self.__class__.__name__}("{self.name}")' def __len__(self) -> int: - return self.connection.execute(f"select count(*) from {self.name}").fetchone()[ - 0 - ] + return self.connection.execute(f"select count(*) from {self.name}").fetchone()[0] def __iter__(self) -> Generator[dict[str, Any], None, None]: return self.select().fetchall() @@ -279,9 +265,7 @@ def select( assert columns, "Columns cannot be empty" - select_columns: list[SelectColumn] = [ - SelectColumn.from_column(c) for c in columns - ] + select_columns: list[SelectColumn] = [SelectColumn.from_column(c) for c in columns] select_names = [f"{c.name} as {c.alias}" if c.alias else c.name for c in select_columns] @@ -299,9 +283,7 @@ def select( return Cursor(self.connection.execute(statement, parameters), columns, self) - def insert( - self, entry: dict[str, Any], exist_ok: bool = False, replace: bool = False - ): + def insert(self, entry: dict[str, Any], exist_ok: bool = False, replace: bool = False): """ Insert a row in the table. Existing rows with matching keys can be ignored or replaced. diff --git a/acacore/database/column.py b/acacore/database/column.py index c265d5e..249c452 100644 --- a/acacore/database/column.py +++ b/acacore/database/column.py @@ -70,10 +70,7 @@ def _schema_to_column(name: str, schema: dict) -> "Column": def model_to_columns(model: Type[BaseModel]) -> list["Column"]: - return [ - _schema_to_column(p, s) - for p, s in model.model_json_schema()["properties"].items() - ] + return [_schema_to_column(p, s) for p, s in model.model_json_schema()["properties"].items()] class Column(Generic[T]): diff --git a/acacore/models/file.py b/acacore/models/file.py index 0fe0eb8..7b8d4f6 100644 --- a/acacore/models/file.py +++ b/acacore/models/file.py @@ -30,9 +30,7 @@ class File(ACABase): warning: Optional[str] def get_absolute_path(self, root: Optional[Path] = None) -> Path: - return ( - root.joinpath(self.relative_path) if root else self.relative_path.resolve() - ) + return root.joinpath(self.relative_path) if root else self.relative_path.resolve() def read_text(self) -> str: """Expose read text functionality from pathlib. @@ -104,4 +102,4 @@ class ArchiveFile(Identification, File): class ConvertedFile(ACABase): file_id: int = Field(primary_key=True) uuid: UUID4 = Field(primary_key=True) - status: str \ No newline at end of file + status: str