diff --git a/questionpy_server/settings.py b/questionpy_server/settings.py index 726d888c..16cef0ff 100644 --- a/questionpy_server/settings.py +++ b/questionpy_server/settings.py @@ -20,8 +20,8 @@ ) from questionpy_common.constants import MAX_PACKAGE_SIZE, MiB -from questionpy_server.worker.worker import Worker -from questionpy_server.worker.worker.subprocess import SubprocessWorker +from questionpy_server.worker import Worker +from questionpy_server.worker.impl.subprocess import SubprocessWorker REPOSITORY_MINIMUM_INTERVAL: Final[timedelta] = timedelta(minutes=5) diff --git a/questionpy_server/web/_routes/_attempts.py b/questionpy_server/web/_routes/_attempts.py index 2adc7d11..9999905a 100644 --- a/questionpy_server/web/_routes/_attempts.py +++ b/questionpy_server/web/_routes/_attempts.py @@ -15,8 +15,7 @@ from questionpy_server.worker.runtime.package_location import ZipPackageLocation if TYPE_CHECKING: - from questionpy_server.worker.worker import Worker - + from questionpy_server.worker import Worker attempt_routes = web.RouteTableDef() diff --git a/questionpy_server/web/_routes/_files.py b/questionpy_server/web/_routes/_files.py index 0f599baa..3251f594 100644 --- a/questionpy_server/web/_routes/_files.py +++ b/questionpy_server/web/_routes/_files.py @@ -12,7 +12,7 @@ from questionpy_server.worker.runtime.package_location import ZipPackageLocation if TYPE_CHECKING: - from questionpy_server.worker.worker import Worker + from questionpy_server.worker import Worker file_routes = web.RouteTableDef() diff --git a/questionpy_server/web/_routes/_packages.py b/questionpy_server/web/_routes/_packages.py index 53ef6e28..2b353a23 100644 --- a/questionpy_server/web/_routes/_packages.py +++ b/questionpy_server/web/_routes/_packages.py @@ -16,7 +16,7 @@ from questionpy_server.worker.runtime.package_location import ZipPackageLocation if TYPE_CHECKING: - from questionpy_server.worker.worker import Worker + from questionpy_server.worker import Worker package_routes = web.RouteTableDef() diff --git a/questionpy_server/worker/__init__.py b/questionpy_server/worker/__init__.py index 410f0ace..0101c228 100644 --- a/questionpy_server/worker/__init__.py +++ b/questionpy_server/worker/__init__.py @@ -1,9 +1,21 @@ # This file is part of the QuestionPy Server. (https://questionpy.org) # The QuestionPy Server is free software released under terms of the MIT license. See LICENSE.md. # (c) Technische Universität Berlin, innoCampus +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum from pydantic import BaseModel +from questionpy_common.api.attempt import AttemptModel, AttemptScoredModel +from questionpy_common.elements import OptionsFormDefinition +from questionpy_common.environment import RequestUser, WorkerResourceLimits +from questionpy_common.manifest import PackageFile +from questionpy_server.models import AttemptStarted, QuestionCreated +from questionpy_server.utils.manifest import ComparableManifest +from questionpy_server.worker.runtime.messages import MessageToWorker +from questionpy_server.worker.runtime.package_location import PackageLocation + class WorkerResources(BaseModel): """Current resource usage.""" @@ -11,3 +23,159 @@ class WorkerResources(BaseModel): memory: int cpu_time_since_last_call: float total_cpu_time: float + + +class WorkerState(Enum): + NOT_RUNNING = 1 + IDLE = 2 + SERVER_AWAITS_RESPONSE = 3 # server send a message to worker and is waiting for a response + WORKER_AWAITS_RESPONSE = 4 # worker send a request/message to server and server is now processing the request + + +@dataclass +class PackageFileData: + """Represents a file read from a package.""" + + size: int + """The total size of the file in bytes.""" + mime_type: str | None + """Mime type as reported by the package. + + Usually this is derived from the file extension at build time and listed in the manifest. + """ + data: bytes + + +class Worker(ABC): + """Interface for worker implementations.""" + + def __init__(self, package: PackageLocation, limits: WorkerResourceLimits | None) -> None: + self.package = package + self.limits = limits + self.state = WorkerState.NOT_RUNNING + + @abstractmethod + async def start(self) -> None: + """Start and initialize the worker process. + + Only after this method finishes is the worker ready to accept other messages. + """ + + @abstractmethod + async def stop(self, timeout: float) -> None: + """Ask the worker to exit gracefully and wait for at most timeout seconds before killing it. + + If the worker is not running for any reason, this method should do nothing. + """ + + @abstractmethod + async def kill(self) -> None: + """Kill the worker process without waiting for it to exit. + + If the worker is not running for any reason, this method should do nothing. + """ + + @abstractmethod + def send(self, message: MessageToWorker) -> None: + """Send a message to the worker.""" + + @abstractmethod + async def get_resource_usage(self) -> WorkerResources | None: + """Get the worker's current resource usage. If unknown or unsupported, return None.""" + + @abstractmethod + async def get_manifest(self) -> ComparableManifest: + """Get manifest of the main package in the worker.""" + + @abstractmethod + async def get_options_form( + self, request_user: RequestUser, question_state: str | None + ) -> tuple[OptionsFormDefinition, dict[str, object]]: + """Get the form used to create a new or edit an existing question. + + Args: + request_user: Information on the user this request is for. + question_state: The current question state if editing, or ``None`` if creating a new question. + + Returns: + Tuple of the form definition and the current data of the inputs. + """ + + @abstractmethod + async def create_question_from_options( + self, request_user: RequestUser, old_state: str | None, form_data: dict[str, object] + ) -> QuestionCreated: + """Create or update the question (state) with the form data from a submitted question edit form. + + Args: + request_user: Information on the user this request is for. + old_state: The current question state if editing, or ``None`` if creating a new question. + form_data: Form data from a submitted question edit form. + + Returns: + New question. + """ + + @abstractmethod + async def start_attempt(self, request_user: RequestUser, question_state: str, variant: int) -> AttemptStarted: + """Start an attempt at this question with the given variant. + + Args: + request_user: Information on the user this request is for. + question_state: The question that is to be attempted. + variant: Not implemented. + + Returns: + The started attempt consisting of opaque attempt state and metadata. + """ + + @abstractmethod + async def get_attempt( + self, + *, + request_user: RequestUser, + question_state: str, + attempt_state: str, + scoring_state: str | None = None, + response: dict | None = None, + ) -> AttemptModel: + """Create an attempt object for a previously started attempt. + + Args: + request_user: Information on the user this request is for. + question_state: The question the attempt belongs to. + attempt_state: The `attempt_state` attribute of an attempt which was previously returned by + :meth:`start_attempt`. + scoring_state: Not implemented. + response: The response currently entered by the student. + + Returns: + Metadata of the attempt. + """ + + @abstractmethod + async def score_attempt( + self, + *, + request_user: RequestUser, + question_state: str, + attempt_state: str, + scoring_state: str | None = None, + response: dict, + ) -> AttemptScoredModel: + """TODO: write docstring.""" + + @abstractmethod + async def get_static_file(self, path: str) -> PackageFileData: + """Reads the static file at the given path in the package. + + Args: + path: Path relative to the `dist` directory of the package. + + Raises: + FileNotFoundError: If no static file exists at the given path. + """ + + @abstractmethod + async def get_static_file_index(self) -> dict[str, PackageFile]: + """Returns the index of static files as declared in the package's manifest.""" diff --git a/questionpy_server/worker/exception.py b/questionpy_server/worker/exception.py index a1e4515b..75f52f61 100644 --- a/questionpy_server/worker/exception.py +++ b/questionpy_server/worker/exception.py @@ -13,3 +13,7 @@ class WorkerStartError(Exception): class WorkerCPUTimeLimitExceededError(Exception): pass + + +class StaticFileSizeMismatchError(Exception): + pass diff --git a/tests/questionpy_server/worker/worker/__init__.py b/questionpy_server/worker/impl/__init__.py similarity index 100% rename from tests/questionpy_server/worker/worker/__init__.py rename to questionpy_server/worker/impl/__init__.py diff --git a/questionpy_server/worker/worker/base.py b/questionpy_server/worker/impl/_base.py similarity index 98% rename from questionpy_server/worker/worker/base.py rename to questionpy_server/worker/impl/_base.py index 5da0f8a8..19963247 100644 --- a/questionpy_server/worker/worker/base.py +++ b/questionpy_server/worker/impl/_base.py @@ -17,7 +17,8 @@ from questionpy_common.manifest import Manifest, PackageFile from questionpy_server.models import AttemptStarted, QuestionCreated from questionpy_server.utils.manifest import ComparableManifest -from questionpy_server.worker.exception import WorkerNotRunningError, WorkerStartError +from questionpy_server.worker import PackageFileData, Worker, WorkerState +from questionpy_server.worker.exception import StaticFileSizeMismatchError, WorkerNotRunningError, WorkerStartError from questionpy_server.worker.runtime.messages import ( CreateQuestionFromOptions, Exit, @@ -39,7 +40,6 @@ PackageLocation, ZipPackageLocation, ) -from questionpy_server.worker.worker import PackageFileData, Worker, WorkerState if TYPE_CHECKING: from pathlib import Path @@ -290,7 +290,3 @@ async def get_static_file(self, path: str) -> PackageFileData: async def get_static_file_index(self) -> dict[str, PackageFile]: return (await self.get_manifest()).static_files - - -class StaticFileSizeMismatchError(Exception): - pass diff --git a/questionpy_server/worker/worker/subprocess.py b/questionpy_server/worker/impl/subprocess.py similarity index 98% rename from questionpy_server/worker/worker/subprocess.py rename to questionpy_server/worker/impl/subprocess.py index cf095348..f23edbe7 100644 --- a/questionpy_server/worker/worker/subprocess.py +++ b/questionpy_server/worker/impl/subprocess.py @@ -17,9 +17,9 @@ from questionpy_server.worker import WorkerResources from questionpy_server.worker.connection import ServerToWorkerConnection from questionpy_server.worker.exception import WorkerNotRunningError, WorkerStartError +from questionpy_server.worker.impl._base import BaseWorker from questionpy_server.worker.runtime.messages import MessageToServer, MessageToWorker from questionpy_server.worker.runtime.package_location import PackageLocation -from questionpy_server.worker.worker.base import BaseWorker if TYPE_CHECKING: from asyncio.subprocess import Process diff --git a/questionpy_server/worker/worker/thread.py b/questionpy_server/worker/impl/thread.py similarity index 98% rename from questionpy_server/worker/worker/thread.py rename to questionpy_server/worker/impl/thread.py index 59ac5372..01d5ea72 100644 --- a/questionpy_server/worker/worker/thread.py +++ b/questionpy_server/worker/impl/thread.py @@ -13,11 +13,11 @@ from questionpy_common.environment import WorkerResourceLimits from questionpy_server.worker.connection import ServerToWorkerConnection from questionpy_server.worker.exception import WorkerNotRunningError +from questionpy_server.worker.impl._base import BaseWorker from questionpy_server.worker.runtime.connection import WorkerToServerConnection from questionpy_server.worker.runtime.manager import WorkerManager from questionpy_server.worker.runtime.package_location import PackageLocation from questionpy_server.worker.runtime.streams import AsyncReadAdapter, DuplexPipe -from questionpy_server.worker.worker.base import BaseWorker log = logging.getLogger(__name__) diff --git a/questionpy_server/worker/pool.py b/questionpy_server/worker/pool.py index 3cdd0014..1494de79 100644 --- a/questionpy_server/worker/pool.py +++ b/questionpy_server/worker/pool.py @@ -8,11 +8,11 @@ from questionpy_common.constants import MiB from questionpy_common.environment import WorkerResourceLimits +from questionpy_server.worker.impl.subprocess import SubprocessWorker from questionpy_server.worker.runtime.package_location import PackageLocation +from . import Worker from .exception import WorkerStartError -from .worker import Worker -from .worker.subprocess import SubprocessWorker class WorkerPool: diff --git a/questionpy_server/worker/worker/__init__.py b/questionpy_server/worker/worker/__init__.py deleted file mode 100644 index c255fb95..00000000 --- a/questionpy_server/worker/worker/__init__.py +++ /dev/null @@ -1,173 +0,0 @@ -# This file is part of the QuestionPy Server. (https://questionpy.org) -# The QuestionPy Server is free software released under terms of the MIT license. See LICENSE.md. -# (c) Technische Universität Berlin, innoCampus - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from enum import Enum - -from questionpy_common.api.attempt import AttemptModel, AttemptScoredModel -from questionpy_common.elements import OptionsFormDefinition -from questionpy_common.environment import RequestUser, WorkerResourceLimits -from questionpy_common.manifest import PackageFile -from questionpy_server.models import AttemptStarted, QuestionCreated -from questionpy_server.utils.manifest import ComparableManifest -from questionpy_server.worker import WorkerResources -from questionpy_server.worker.runtime.messages import MessageToWorker -from questionpy_server.worker.runtime.package_location import PackageLocation - - -class WorkerState(Enum): - NOT_RUNNING = 1 - IDLE = 2 - SERVER_AWAITS_RESPONSE = 3 # server send a message to worker and is waiting for a response - WORKER_AWAITS_RESPONSE = 4 # worker send a request/message to server and server is now processing the request - - -@dataclass -class PackageFileData: - """Represents a file read from a package.""" - - size: int - """The total size of the file in bytes.""" - mime_type: str | None - """Mime type as reported by the package. - - Usually this is derived from the file extension at build time and listed in the manifest. - """ - data: bytes - - -class Worker(ABC): - """Interface for worker implementations.""" - - def __init__(self, package: PackageLocation, limits: WorkerResourceLimits | None) -> None: - self.package = package - self.limits = limits - self.state = WorkerState.NOT_RUNNING - - @abstractmethod - async def start(self) -> None: - """Start and initialize the worker process. - - Only after this method finishes is the worker ready to accept other messages. - """ - - @abstractmethod - async def stop(self, timeout: float) -> None: - """Ask the worker to exit gracefully and wait for at most timeout seconds before killing it. - - If the worker is not running for any reason, this method should do nothing. - """ - - @abstractmethod - async def kill(self) -> None: - """Kill the worker process without waiting for it to exit. - - If the worker is not running for any reason, this method should do nothing. - """ - - @abstractmethod - def send(self, message: MessageToWorker) -> None: - """Send a message to the worker.""" - - @abstractmethod - async def get_resource_usage(self) -> WorkerResources | None: - """Get the worker's current resource usage. If unknown or unsupported, return None.""" - - @abstractmethod - async def get_manifest(self) -> ComparableManifest: - """Get manifest of the main package in the worker.""" - - @abstractmethod - async def get_options_form( - self, request_user: RequestUser, question_state: str | None - ) -> tuple[OptionsFormDefinition, dict[str, object]]: - """Get the form used to create a new or edit an existing question. - - Args: - request_user: Information on the user this request is for. - question_state: The current question state if editing, or ``None`` if creating a new question. - - Returns: - Tuple of the form definition and the current data of the inputs. - """ - - @abstractmethod - async def create_question_from_options( - self, request_user: RequestUser, old_state: str | None, form_data: dict[str, object] - ) -> QuestionCreated: - """Create or update the question (state) with the form data from a submitted question edit form. - - Args: - request_user: Information on the user this request is for. - old_state: The current question state if editing, or ``None`` if creating a new question. - form_data: Form data from a submitted question edit form. - - Returns: - New question. - """ - - @abstractmethod - async def start_attempt(self, request_user: RequestUser, question_state: str, variant: int) -> AttemptStarted: - """Start an attempt at this question with the given variant. - - Args: - request_user: Information on the user this request is for. - question_state: The question that is to be attempted. - variant: Not implemented. - - Returns: - The started attempt consisting of opaque attempt state and metadata. - """ - - @abstractmethod - async def get_attempt( - self, - *, - request_user: RequestUser, - question_state: str, - attempt_state: str, - scoring_state: str | None = None, - response: dict | None = None, - ) -> AttemptModel: - """Create an attempt object for a previously started attempt. - - Args: - request_user: Information on the user this request is for. - question_state: The question the attempt belongs to. - attempt_state: The `attempt_state` attribute of an attempt which was previously returned by - :meth:`start_attempt`. - scoring_state: Not implemented. - response: The response currently entered by the student. - - Returns: - Metadata of the attempt. - """ - - @abstractmethod - async def score_attempt( - self, - *, - request_user: RequestUser, - question_state: str, - attempt_state: str, - scoring_state: str | None = None, - response: dict, - ) -> AttemptScoredModel: - """TODO: write docstring.""" - - @abstractmethod - async def get_static_file(self, path: str) -> PackageFileData: - """Reads the static file at the given path in the package. - - Args: - path: Path relative to the `dist` directory of the package. - - Raises: - FileNotFoundError: If no static file exists at the given path. - """ - - @abstractmethod - async def get_static_file_index(self) -> dict[str, PackageFile]: - """Returns the index of static files as declared in the package's manifest.""" diff --git a/tests/conftest.py b/tests/conftest.py index fadc623c..84a14873 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,8 +26,8 @@ WorkerSettings, ) from questionpy_server.utils.manifest import ComparableManifest +from questionpy_server.worker.impl.thread import ThreadWorker from questionpy_server.worker.runtime.package_location import DirPackageLocation, ZipPackageLocation -from questionpy_server.worker.worker.thread import ThreadWorker def get_file_hash(path: Path) -> str: diff --git a/tests/questionpy_server/worker/impl/__init__.py b/tests/questionpy_server/worker/impl/__init__.py new file mode 100644 index 00000000..879bb94b --- /dev/null +++ b/tests/questionpy_server/worker/impl/__init__.py @@ -0,0 +1,3 @@ +# This file is part of the QuestionPy Server. (https://questionpy.org) +# The QuestionPy Server is free software released under terms of the MIT license. See LICENSE.md. +# (c) Technische Universität Berlin, innoCampus diff --git a/tests/questionpy_server/worker/worker/test_base.py b/tests/questionpy_server/worker/impl/test_base.py similarity index 94% rename from tests/questionpy_server/worker/worker/test_base.py rename to tests/questionpy_server/worker/impl/test_base.py index 892edbc6..cd9878df 100644 --- a/tests/questionpy_server/worker/worker/test_base.py +++ b/tests/questionpy_server/worker/impl/test_base.py @@ -8,9 +8,9 @@ from questionpy_common.constants import MiB from questionpy_server import WorkerPool -from questionpy_server.worker.worker.base import StaticFileSizeMismatchError -from questionpy_server.worker.worker.subprocess import SubprocessWorker -from questionpy_server.worker.worker.thread import ThreadWorker +from questionpy_server.worker.exception import StaticFileSizeMismatchError +from questionpy_server.worker.impl.subprocess import SubprocessWorker +from questionpy_server.worker.impl.thread import ThreadWorker from tests.conftest import PACKAGE, TestPackageFactory if TYPE_CHECKING: diff --git a/tests/questionpy_server/worker/worker/test_subprocess.py b/tests/questionpy_server/worker/impl/test_subprocess.py similarity index 93% rename from tests/questionpy_server/worker/worker/test_subprocess.py rename to tests/questionpy_server/worker/impl/test_subprocess.py index 2f58e25b..8f17e818 100644 --- a/tests/questionpy_server/worker/worker/test_subprocess.py +++ b/tests/questionpy_server/worker/impl/test_subprocess.py @@ -9,7 +9,7 @@ from questionpy_common.constants import MiB from questionpy_server import WorkerPool -from questionpy_server.worker.worker.subprocess import SubprocessWorker +from questionpy_server.worker.impl.subprocess import SubprocessWorker from tests.conftest import PACKAGE diff --git a/tests/questionpy_server/worker/worker/test_thread.py b/tests/questionpy_server/worker/impl/test_thread.py similarity index 96% rename from tests/questionpy_server/worker/worker/test_thread.py rename to tests/questionpy_server/worker/impl/test_thread.py index 1fbec7b7..fa362a18 100644 --- a/tests/questionpy_server/worker/worker/test_thread.py +++ b/tests/questionpy_server/worker/impl/test_thread.py @@ -11,9 +11,9 @@ from questionpy_common.constants import MiB from questionpy_server import WorkerPool from questionpy_server.worker.exception import WorkerStartError +from questionpy_server.worker.impl.thread import ThreadWorker from questionpy_server.worker.runtime.manager import WorkerManager from questionpy_server.worker.runtime.messages import WorkerUnknownError -from questionpy_server.worker.worker.thread import ThreadWorker from tests.conftest import PACKAGE