Merge branch 'dev' into permission-middleware

README.md

@@ -25,6 +25,23 @@ Instructions for the backend are located [here](backend/README.md).
 
 Automated clients can interact with the web application via the [API](https://sel2-5.ugent.be/api/docs).
 
+## Used tools and frameworks
+
+### Database
+ - Database system: [PostgreSQL](https://www.postgresql.org/)
+ - Database migrations: [alembic](https://github.com/sqlalchemy/alembic).
+
+### Backend
+ - Backend framework: [FastAPI](https://fastapi.tiangolo.com/)
+ - Database interface: [SQLAlchemy](https://www.sqlalchemy.org/)
+ - JSON-validation: [Pydantic](https://github.com/pydantic/pydantic)
+ - Test framework: [pytest](https://github.com/pytest-dev/pytest)
+
+### Frontend
+ - Frontend framework: [Vue.js](https://vuejs.org/) (Composition API) + [TypeScript](https://www.typescriptlang.org/)
+ - Component library: [Vuetify](https://dev.vuetifyjs.com/en/)
+ - Test framework: [Vitest](https://vitest.dev/)
+
 ## The team
 
 |                  |                                                   |

backend/README.md

@@ -6,7 +6,7 @@
 
 ```sh
 # Create a python virtual environment
-python -m venv venv
+python3.12 -m venv venv
 # Activate the environment
 source venv/bin/activate
 # Install dependencies

backend/src/docker_tests/docker_tests.py

@@ -1,8 +1,10 @@
 import os
 import shutil
+from dataclasses import dataclass
 from pathlib import Path
 
 from docker import DockerClient
+from docker.errors import APIError, NotFound
 from docker.models.containers import Container
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -15,6 +17,18 @@
 # if a container exits with this code, the test failed (exit 0 means the test succeeded)
 EXIT_TEST_FAILED = 10
 
+# mark test runner containers with this label
+REV_DOMAIN = "be.ugent.sel2-5"
+TEST_RUNNER_LABEL = "test_runner"
+
+
+@dataclass
+class DockerResult:
+    status: Status
+    test_results: list[TestResult]
+    stdout: str | None
+    stderr: str | None
+
 
 def read_feedback_file(path: str) -> list[str]:
     with open(path, 'r') as f:
@@ -37,50 +51,25 @@ async def launch_docker_tests(
     os.makedirs(feedback_dir)
     touch(os.path.join(feedback_dir, "correct"), os.path.join(feedback_dir, "failed"))
 
-    # TODO: zorgen dat tests niet gemount worden als custom docker image gemaakt wordt
-
     if using_default_docker_image(tests_uuid):
         # relative path independent of working dir (tests will break otherwise)
         # path = "./docker_default"
         path = os.path.join(Path(__file__).parent, "docker_default")
         image_tag = "default_image"
+        tests_dir = tests_path(tests_uuid)
 
         # rebuild default image if changes were made
         await build_docker_image(path, image_tag, client)
     else:
         image_tag = tests_uuid
+        tests_dir = None
 
-    container = run_docker_tests(
-        image_tag,
-        submission_path(submission_uuid),
-        artifact_dir,
-        feedback_dir,
-        tests_path(tests_uuid),
-        client,
-    )
-    exit_code = (await wait_until_exit(container))['StatusCode']
-
-    if exit_code == 0:
-        status = Status.Accepted
-    elif exit_code == EXIT_TEST_FAILED:
-        status = Status.Rejected
-    else:
-        status = Status.Crashed
-
-    test_results = []
-    for line in read_feedback_file(os.path.join(feedback_dir, "correct")):
-        test_results.append(TestResult(succeeded=True, value=line))
-    for line in read_feedback_file(os.path.join(feedback_dir, "failed")):
-        test_results.append(TestResult(succeeded=False, value=line))
-
-    stdout = container.logs(stdout=True, stderr=False).decode("utf-8")
-    stderr = container.logs(stdout=False, stderr=True).decode("utf-8")
-    container.remove()
+    result = await run_docker_tests(image_tag, submission_uuid, artifact_dir, feedback_dir, tests_dir, client)
 
     await update_submission_status(
-        db, submission_id, status, test_results,
-        stdout=stdout if stdout else None,
-        stderr=stderr if stderr else None,
+        db, submission_id, result.status, result.test_results,
+        stdout=result.stdout,
+        stderr=result.stderr,
     )
 
     await db.close()
@@ -89,6 +78,52 @@ async def launch_docker_tests(
     shutil.rmtree(feedback_dir)
 
 
+async def run_docker_tests(image_tag: str, submission_uuid: str, artifact_dir: str, feedback_dir: str, tests_dir: str | None,
+                           client: DockerClient) -> DockerResult:
+    try:
+        container = create_container(
+            image_tag,
+            submission_path(submission_uuid),
+            artifact_dir,
+            feedback_dir,
+            tests_dir,
+            client,
+        )
+    except APIError as e:
+        return DockerResult(status=Status.Crashed, test_results=[], stdout=None, stderr=str(e))
+
+    try:
+        container.start()
+        exit_code = (await wait_until_exit(container))['StatusCode']
+
+        if exit_code == 0:
+            status = Status.Accepted
+        elif exit_code == EXIT_TEST_FAILED:
+            status = Status.Rejected
+        else:
+            status = Status.Crashed
+
+        test_results = []
+        for line in read_feedback_file(os.path.join(feedback_dir, "correct")):
+            test_results.append(TestResult(succeeded=True, value=line))
+        for line in read_feedback_file(os.path.join(feedback_dir, "failed")):
+            test_results.append(TestResult(succeeded=False, value=line))
+
+        stdout = container.logs(stdout=True, stderr=False).decode("utf-8")
+        stderr = container.logs(stdout=False, stderr=True).decode("utf-8")
+
+        return DockerResult(status=status, test_results=test_results, stdout=stdout if stdout else None,
+                            stderr=stderr if stderr else None)
+
+    except APIError as e:
+        return DockerResult(status=Status.Crashed, test_results=[], stdout=None, stderr=str(e))
+
+    finally:
+        container.remove(force=True)
+        # remove all stopped containers with test runner tag
+        client.containers.prune(filters={"label": f"{REV_DOMAIN}={TEST_RUNNER_LABEL}"})
+
+
 @to_async
 def build_docker_image(path: str, tag: str, client: DockerClient):
     """Build a docker image from a directory where a file 'Dockerfile' is present"""
@@ -97,24 +132,43 @@ def build_docker_image(path: str, tag: str, client: DockerClient):
         tag=tag,
         forcerm=True
     )
-    client.images.prune()  # cleanup dangling images
+
+    # clean up dangling images
+    client.images.prune()
+
+
+def remove_docker_image_if_exists(tag: str, client: DockerClient):
+    try:
+        client.images.remove(image=tag, force=True)
+    except NotFound:
+        pass
+
+    # clean up dangling images
+    client.images.prune()
 
 
 def using_default_docker_image(tests_uuid: str) -> bool:
     return not os.path.isfile(os.path.join(tests_path(tests_uuid), "Dockerfile"))
 
 
-def run_docker_tests(
-    image_tag: str, submission_dir: str, artifact_dir: str, feedback_dir: str, tests_dir: str, client: DockerClient
+def create_container(
+    image_tag: str, submission_dir: str, artifact_dir: str, feedback_dir: str, tests_dir: str | None,
+    client: DockerClient
 ) -> Container:
-    return client.containers.run(
+    volumes = {
+        submission_dir: {'bind': '/submission', 'mode': 'ro'},
+        artifact_dir: {'bind': '/artifacts', 'mode': 'rw'},
+        feedback_dir: {'bind': '/feedback', 'mode': 'rw'},
+    }
+
+    # only mount test files for default image
+    if tests_dir is not None:
+        volumes[tests_dir] = {'bind': '/tests', 'mode': 'ro'}
+
+    return client.containers.create(
         image=image_tag,
-        volumes={
-            submission_dir: {'bind': '/submission', 'mode': 'ro'},
-            artifact_dir: {'bind': '/artifacts', 'mode': 'rw'},
-            feedback_dir: {'bind': '/feedback', 'mode': 'rw'},
-            tests_dir: {'bind': '/tests', 'mode': 'ro'},
-        },
+        volumes=volumes,
+        labels={REV_DOMAIN: TEST_RUNNER_LABEL},
         environment={
             'SUBMISSION_DIR': '/submission',
             'ARTIFACT_DIR': '/artifacts',
@@ -124,8 +178,6 @@ def run_docker_tests(
             'EXIT_TEST_FAILED': EXIT_TEST_FAILED,
         },
         detach=True,
-        stdout=True,
-        stderr=True,
     )  # pyright: ignore
 
 

backend/src/docker_tests/utils.py

@@ -59,6 +59,7 @@ def write_and_unpack_files(files: list[UploadFile], uuid: str | None) -> str:
 
             if upload_file.content_type == "application/zip":
                 shutil.unpack_archive(path, files_path)
+                os.remove(path)  # don't store zip file
 
     return uuid
 

backend/src/project/dependencies.py

@@ -22,8 +22,6 @@ async def retrieve_project(project_id: int,
 
 
 async def retrieve_test_files_uuid(project: Project = Depends(retrieve_project)):
-    if project.test_files_uuid is None:
-        raise TestsNotFound
     return project.test_files_uuid
 
 

backend/src/project/router.py

@@ -1,13 +1,16 @@
-from typing import Sequence, List
+from typing import Sequence, List, Optional
 
 from docker import DockerClient
 from fastapi import APIRouter, Depends, UploadFile, BackgroundTasks
+from fastapi.responses import StreamingResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from src import dependencies
 from src.auth.dependencies import authentication_validation
 from src.dependencies import get_async_db
 from src.group.dependencies import retrieve_groups_by_project
 from src.group.schemas import GroupList
+from src.project.utils import project_zip_stream
 from src.submission.schemas import Submission
 from src.submission.service import get_submissions_by_project
 from . import service
@@ -23,7 +26,7 @@
     update_project, update_test_files,
 )
 from ..docker_tests.dependencies import get_docker_client
-from ..docker_tests.docker_tests import using_default_docker_image, build_docker_image
+from ..docker_tests.docker_tests import using_default_docker_image, build_docker_image, remove_docker_image_if_exists
 from ..docker_tests.utils import get_files_from_dir, tests_path, write_and_unpack_files, remove_test_files
 
 router = APIRouter(
@@ -84,8 +87,18 @@ async def list_submissions(project_id: int,
     return await get_submissions_by_project(db, project_id)
 
 
+@router.get("/{project_id}/zip", response_class=StreamingResponse, dependencies=[Depends(patch_permission_validation)])
+async def get_submissions_dump(project_id: int, db: AsyncSession = Depends(get_async_db)):
+    """Return zip file containing all submission files and csv"""
+    submissions = await get_submissions_by_project(db, project_id)
+    data = await project_zip_stream(db, submissions, project_id)
+    return StreamingResponse(data, media_type="application/zip")
+
+
 @router.get("/{project_id}/test_files")
-async def get_test_files(test_files_uuid: str = Depends(retrieve_test_files_uuid)):
+async def get_test_files(test_files_uuid: Optional[str] = Depends(retrieve_test_files_uuid)):
+    if not test_files_uuid:
+        return []
     return get_files_from_dir(tests_path(test_files_uuid))
 
 
@@ -105,7 +118,8 @@ async def put_test_files(
 
     if not using_default_docker_image(uuid):
         # build custom docker image if dockerfile is present
-        background_tasks.add_task(build_docker_image, tests_path(uuid), uuid, client)
+        background_tasks.add_task(
+            build_docker_image, tests_path(uuid), uuid, client)
 
     return await update_test_files(db, project.id, uuid)
 
@@ -114,7 +128,10 @@ async def put_test_files(
 async def delete_test_files(
     project: Project = Depends(retrieve_project),
     uuid: str = Depends(retrieve_test_files_uuid),
-    db: AsyncSession = Depends(get_async_db)
+    db: AsyncSession = Depends(get_async_db),
+    client: DockerClient = Depends(get_docker_client)
 ):
+    remove_docker_image_if_exists(uuid, client)
     remove_test_files(uuid)
+
     return await service.update_test_files(db, project.id, None)

backend/src/project/schemas.py

@@ -19,6 +19,8 @@ class ProjectBase(BaseModel):
     is_visible: bool = Field(default=True)
     capacity: int = Field(gt=0)
     requirements: List[Requirement] = []
+    enroll_deadline: Optional[datetime]
+    publish_date: datetime
 
 
 class ProjectCreate(ProjectBase):
@@ -55,7 +57,8 @@ class ProjectUpdate(BaseModel):
     deadline: Optional[datetime] = None
     description: Optional[str] = None
     requirements: Optional[List[Requirement]] = None
-    is_visible: Optional[bool] = None
+    enroll_deadline: Optional[datetime] = None
+    publish_date: Optional[datetime] = None
 
     @field_validator("deadline")
     def validate_deadline(cls, value: datetime) -> datetime:

backend/src/project/service.py

@@ -18,7 +18,8 @@ async def create_project(db: AsyncSession, project_in: ProjectCreate) -> Project
         description=project_in.description,
         is_visible=project_in.is_visible,
         capacity=project_in.capacity,
-        requirements=[Requirement(**r.model_dump()) for r in project_in.requirements],
+        requirements=[Requirement(**r.model_dump())
+                      for r in project_in.requirements],
     )
     db.add(new_project)
     await db.commit()
@@ -77,10 +78,12 @@ async def update_project(
         project.name = project_update.name
     if project_update.deadline is not None:
         project.deadline = project_update.deadline
+    if project_update.publish_date is not None:
+        project.publish_date = project_update.publish_date
+    if project_update.enroll_deadline is not None:
+        project.enroll_deadline = project.enroll_deadline
     if project_update.description is not None:
         project.description = project_update.description
-    if project_update.is_visible is not None:
-        project.is_visible = project_update.is_visible
     if project_update.requirements is not None:
         await delete_requirements_for_project(db, project_id)
         project.requirements = [Requirement(**r.model_dump())