feat: add fd support and bump all versions (py3.12 included)

.github/workflows/run-linter-and-tests.yaml

@@ -16,10 +16,10 @@ jobs:
       - name: Set up python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - name: Install deps
-        run: python3 -m pip install poetry==1.6.1 && make init
+        run: python3 -m pip install poetry==1.8.2 && make init
 
       - name: Run all linters and formatters
         run: make lint

.pre-commit-config.yaml

@@ -28,13 +28,6 @@ repos:
         args: [ --fix, --exit-non-zero-on-fix ]
       - id: ruff-format
 
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.1.1
-    hooks:
-      - id: black
-        types: [ file, python ]
-        stages: [ commit ]
-
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.8.0
     hooks:

Dockerfile

@@ -5,9 +5,9 @@
 
 # References: using official Python images
 # https://hub.docker.com/_/python
-ARG OFFICIAL_PYTHON_IMAGE=python:3.11.5-slim-bullseye
+ARG OFFICIAL_PYTHON_IMAGE=python:3.12.2-slim-bullseye
 FROM ${OFFICIAL_PYTHON_IMAGE} as poetry-install-build-stage
-ARG POETRY_VERSION=1.6.1
+ARG POETRY_VERSION=1.8.2
 
 # References:
 # https://pip.pypa.io/en/stable/topics/caching/#avoiding-caching

Makefile

@@ -40,7 +40,7 @@ downgrade:
 
 ## Run celery worker in watch mode
 worker:
-	. .venv/bin/activate && watchmedo auto-restart --directory=./ --pattern='*.py' --recursive -- celery -A app.worker worker --loglevel=info --concurrency=1
+	watchmedo auto-restart --directory=./ --pattern='*.py' --recursive -- celery -A app.worker worker --loglevel=info --concurrency=1
 
 ## Run application server in watch mode
 app:
@@ -52,14 +52,19 @@ init:
 
 ## Run all formatters and linters in project
 lint:
-	poetry run ruff ./tests/*.py ./app/*.py
-	poetry run ruff format --check ./tests/*.py ./app/*.py
-	poetry run black --check ./tests/*.py ./app/*.py
-	poetry run mypy --ignore-missing-imports ./app/*.py
+	ls
+	poetry run ruff tests app \
+	& poetry run ruff format --check tests app \
+	& poetry run black --check tests app \
+	& poetry run mypy --ignore-missing-imports app
+
+format:
+	poetry run ruff format tests app & poetry run black tests app
+
 
 ## Run all tests in project
 test:
-	poetry run pytest --verbosity=2 --showlocals -log-level=DEBUG --cov=app --cov-report term
+	poetry run pytest -o log_cli=true --verbosity=2 --showlocals --log-cli-level=INFO --cov=app --cov-report term
 
 .DEFAULT_GOAL := help
 # See <https://gist.github.com/klmr/575726c7e05d8780505a> for explanation.

app/db/session.py

@@ -1,3 +1,4 @@
+from typing import Generator
 from sqlalchemy.orm import Session, sessionmaker
 from sqlalchemy import create_engine
 
@@ -9,6 +10,6 @@
 SessionLocal = sessionmaker(bind=engine, autoflush=False)
 
 
-def get_session() -> Session:
+def get_session() -> Generator[Session, None, None]:
     with SessionLocal() as session:
         yield session

app/domain/task/abstract_task.py

@@ -0,0 +1,31 @@
+from typing import Generic, TypeVar, Type
+from abc import ABC, abstractmethod
+from abcattrs import abstractattrs, Abstract
+from pydantic import BaseModel
+from desbordante import Algorithm
+import pandas
+
+Conf = TypeVar("Conf", bound=BaseModel)
+Res = TypeVar("Res", bound=BaseModel)
+Algo = TypeVar("Algo", bound=Algorithm)
+
+
+@abstractattrs
+class AbstractTask(Generic[Algo, Conf, Res], ABC):
+    algorithm: Abstract[Algo]
+    config_model_cls: Abstract[Type[Conf]]
+    result_model_cls: Abstract[Type[Res]]
+
+    def __init__(self, table: pandas.DataFrame) -> None:
+        super().__init__()
+        self.table = table
+        self.algorithm.load_data(table=table)
+
+    def execute(self, config: Conf = None) -> Res:
+        options = config.model_dump(exclude_unset=True) if config else {}
+        self.algorithm.execute(**options)
+        return self.collect_result()
+
+    @abstractmethod
+    def collect_result(self) -> Res:
+        ...

app/domain/task/model/task.py

@@ -5,6 +5,7 @@
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from app.db import Base
+from typing import Any
 
 
 class TaskStatus(StrEnum):
@@ -36,8 +37,8 @@ class Task(Base):
     error_msg: Mapped[int | None] = mapped_column(default=None)
     id_executed: Mapped[bool] = mapped_column(default=False)
     elapsed_time: Mapped[float | None] = mapped_column(default=None)
-    config: Mapped[dict] = mapped_column(JSONB)
-    result: Mapped[dict] = mapped_column(JSONB)
+    config: Mapped[dict[Any, Any]] = mapped_column(JSONB)
+    result: Mapped[dict[Any, Any]] = mapped_column(JSONB)
     created_at: Mapped[datetime]
     deleted_at: Mapped[datetime | None] = mapped_column(default=None)
 

app/domain/task/schema/README.md

@@ -0,0 +1,3 @@
+Тут хранятся схемы конфигурации и результата для задач, а так же сами задачи по выбору примитива
+
+Чтобы добавить новую задачу, нужно внести ее в TaskAlgoType, описать конфигурацию по аналогии.

app/domain/task/schema/afd.py

@@ -0,0 +1,18 @@
+from app.domain.task.schema.fd import (
+    TaneTask,
+    TaneConfig,
+    PyroConfig,
+    PyroTask,
+    FDAlgoResult,
+)
+from enum import StrEnum, auto
+
+
+class TaskAlgoType(StrEnum):
+    Pyro = auto()
+    Tane = auto()
+
+
+AFDAlgoResult = FDAlgoResult
+
+__all__ = ["TaneTask", "TaneConfig", "PyroConfig", "PyroTask", "AFDAlgoResult"]

app/domain/task/schema/fd.py

@@ -0,0 +1,156 @@
+from app.domain.task.abstract_task import AbstractTask
+from enum import StrEnum, auto
+from pydantic import BaseModel, Field
+from desbordante.fd import FD, FdAlgorithm
+from desbordante.fd.algorithms import (
+    Aid,
+    DFD,
+    Depminer,
+    FDep,
+    FUN,
+    FastFDs,
+    FdMine,
+    HyFD,
+    Pyro,
+    Tane,
+)
+from typing import Annotated, Generic, TypeVar
+from abc import ABC
+
+
+class TaskAlgoType(StrEnum):
+    Aid = auto()
+    DFD = auto()
+    Depminer = auto()
+    FDep = auto()
+    FUN = auto()
+    FastFDs = auto()
+    FdMine = auto()
+    HyFD = auto()
+    Pyro = auto()
+    Tane = auto()
+
+
+class FDModel(BaseModel):
+    @classmethod
+    def from_fd(cls, fd: FD):
+        return cls(lhs_indices=fd.lhs_indices, rhs_index=fd.rhs_index)
+
+    lhs_indices: list[int]
+    rhs_index: int
+
+
+class FDAlgoResult(BaseModel):
+    fds: list[FDModel]
+
+
+# TODO: replace with 3.12 generics after when supported by mypy
+Conf = TypeVar("Conf", bound=BaseModel)
+FDAlgo = TypeVar("FDAlgo", bound=FdAlgorithm)
+
+
+class FDTask(AbstractTask[FDAlgo, Conf, FDAlgoResult], ABC):
+    result_model_cls = FDAlgoResult
+
+    def collect_result(self) -> FDAlgoResult:
+        fds = self.algorithm.get_fds()
+        return FDAlgoResult(fds=list(map(FDModel.from_fd, fds)))
+
+
+class AidConfig(BaseModel):
+    is_null_equal_null: bool
+
+
+class DFDConfig(BaseModel):
+    is_null_equal_null: bool
+    threads: Annotated[int, Field(ge=1, le=8)]
+
+
+class DepminerConfig(BaseModel):
+    is_null_equal_null: bool
+
+
+class FDepConfig(BaseModel):
+    is_null_equal_null: bool
+
+
+class FUNConfig(BaseModel):
+    is_null_equal_null: bool
+
+
+class FastFDsConfig(BaseModel):
+    is_null_equal_null: bool
+    max_lhs: Annotated[int, Field(ge=1, le=10)]
+    threads: Annotated[int, Field(ge=1, le=8)]
+
+
+class FdMineConfig(BaseModel):
+    is_null_equal_null: bool
+
+
+class HyFDConfig(BaseModel):
+    is_null_equal_null: bool
+
+
+class PyroConfig(BaseModel):
+    is_null_equal_null: bool
+    error: Annotated[float, Field(ge=0, le=1)]
+    max_lhs: Annotated[int, Field(ge=1, le=10)]
+    threads: Annotated[int, Field(ge=1, le=8)]
+    seed: int
+
+
+class TaneConfig(BaseModel):
+    is_null_equal_null: bool
+    error: Annotated[float, Field(ge=0, le=1)]
+    max_lhs: Annotated[int, Field(ge=1, le=10)]
+
+
+class AidTask(FDTask[Aid, AidConfig, FDAlgoResult]):
+    config_model_cls = AidConfig
+    algorithm = Aid()
+
+
+class DFDTask(FDTask[DFD, DFDConfig, FDAlgoResult]):
+    config_model_cls = DFDConfig
+    algorithm = DFD()
+
+
+class DepminerTask(FDTask[Depminer, DepminerConfig, FDAlgoResult]):
+    config_model_cls = DepminerConfig
+    algorithm = Depminer()
+
+
+class FDepTask(FDTask[FDep, FDepConfig, FDAlgoResult]):
+    config_model_cls = FDepConfig
+    algorithm = FDep()
+
+
+class FUNTask(FDTask[FUN, FUNConfig, FDAlgoResult]):
+    config_model_cls = FUNConfig
+    algorithm = FUN()
+
+
+class FastFDsTask(FDTask[FastFDs, FastFDsConfig, FDAlgoResult]):
+    config_model_cls = FastFDsConfig
+    algorithm = FastFDs()
+
+
+class FdMineTask(FDTask[FdMine, FdMineConfig, FDAlgoResult]):
+    config_model_cls = FdMineConfig
+    algorithm = FdMine()
+
+
+class HyFDTask(FDTask[HyFD, HyFDConfig, FDAlgoResult]):
+    config_model_cls = HyFDConfig
+    algorithm = HyFD()
+
+
+class PyroTask(FDTask[Pyro, PyroConfig, FDAlgoResult]):
+    config_model_cls = PyroConfig
+    algorithm = Pyro()
+
+
+class TaneTask(FDTask[Tane, TaneConfig, FDAlgoResult]):
+    config_model_cls = TaneConfig
+    algorithm = Tane()