From 36ed720b4f5086da8e4a027dec7ab36adc16dba1 Mon Sep 17 00:00:00 2001 From: Aron Buzogany <108480125+AronBuzogany@users.noreply.github.com> Date: Sun, 14 Apr 2024 12:12:33 +0200 Subject: [PATCH 1/2] Running evaluator in synchron when runner is set (#203) * Fix #200 * created executor * fixed tests * resolved linting * linting * fix wrong evaluator in test * adjusted runner to support test --- backend/project/__init__.py | 2 + .../endpoints/projects/endpoint_parser.py | 2 + .../project/endpoints/projects/projects.py | 10 +++-- backend/project/endpoints/submissions.py | 19 ++++++-- backend/project/executor.py | 9 ++++ .../project/utils/submissions/evaluator.py | 41 ++++++++++++++++-- backend/requirements.txt | 1 + .../submission_evaluators/python_test.py | 6 +-- .../python/tc_1/assignment/run_test.zip | Bin 0 -> 184 bytes 9 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 backend/project/executor.py create mode 100644 backend/tests/utils/submission_evaluators/resources/python/tc_1/assignment/run_test.zip diff --git a/backend/project/__init__.py b/backend/project/__init__.py index 018848d6..2d454467 100644 --- a/backend/project/__init__.py +++ b/backend/project/__init__.py @@ -5,6 +5,7 @@ from flask import Flask from flask_cors import CORS from sqlalchemy_utils import register_composites +from .executor import executor from .db_in import db from .endpoints.index.index import index_bp from .endpoints.users import users_bp @@ -22,6 +23,7 @@ def create_app(): """ app = Flask(__name__) + executor.init_app(app) app.register_blueprint(index_bp) app.register_blueprint(users_bp) app.register_blueprint(courses_bp) diff --git a/backend/project/endpoints/projects/endpoint_parser.py b/backend/project/endpoints/projects/endpoint_parser.py index f4ab93ea..26e23b8a 100644 --- a/backend/project/endpoints/projects/endpoint_parser.py +++ b/backend/project/endpoints/projects/endpoint_parser.py @@ -31,6 +31,8 @@ location="form" ) +parser.add_argument("runner", type=str, help='Projects runner', location="form") + def parse_project_params(): """ diff --git a/backend/project/endpoints/projects/projects.py b/backend/project/endpoints/projects/projects.py index 01873379..dfc04895 100644 --- a/backend/project/endpoints/projects/projects.py +++ b/backend/project/endpoints/projects/projects.py @@ -11,14 +11,14 @@ from project.db_in import db -from project.models.project import Project +from project.models.project import Project, Runner from project.utils.query_agent import query_selected_from_model, create_model_instance from project.utils.authentication import authorize_teacher from project.endpoints.projects.endpoint_parser import parse_project_params API_URL = os.getenv('API_HOST') -UPLOAD_FOLDER = os.getenv('UPLOAD_URL') +UPLOAD_FOLDER = os.getenv("UPLOAD_FOLDER") class ProjectsEndpoint(Resource): @@ -43,7 +43,6 @@ def get(self, teacher_id=None): filters=request.args ) - @authorize_teacher def post(self, teacher_id=None): """ Post functionality for project @@ -84,6 +83,11 @@ def post(self, teacher_id=None): file.save(file_path) with zipfile.ZipFile(file_path) as upload_zip: upload_zip.extractall(project_upload_directory) + + if not new_project.runner and \ + os.path.exists(os.path.join(project_upload_directory, "Dockerfile")): + + new_project.runner = Runner.CUSTOM except zipfile.BadZipfile: os.remove(os.path.join(project_upload_directory, filename)) db.session.rollback() diff --git a/backend/project/endpoints/submissions.py b/backend/project/endpoints/submissions.py index e4d204e7..5b3412f0 100644 --- a/backend/project/endpoints/submissions.py +++ b/backend/project/endpoints/submissions.py @@ -8,6 +8,7 @@ from flask import Blueprint, request from flask_restful import Resource from sqlalchemy import exc +from project.executor import executor from project.db_in import db from project.models.submission import Submission, SubmissionStatus from project.models.project import Project @@ -20,6 +21,8 @@ authorize_submissions_request, authorize_grader, \ authorize_student_submission, authorize_submission_author +from project.utils.submissions.evaluator import run_evaluator + load_dotenv() API_HOST = getenv("API_HOST") UPLOAD_FOLDER = getenv("UPLOAD_FOLDER") @@ -132,13 +135,23 @@ def post(self) -> dict[str, any]: "submissions", str(submission.submission_id)) try: makedirs(submission.submission_path, exist_ok=True) + input_folder = path.join(submission.submission_path, "submission") + makedirs(input_folder, exist_ok=True) for file in files: - file.save(path.join(submission.submission_path, file.filename)) - session.commit() + file.save(path.join(input_folder, file.filename)) except OSError: rmtree(submission.submission_path) session.rollback() + if project.runner: + submission.submission_status = SubmissionStatus.RUNNING + executor.submit( + run_evaluator, + submission, + path.join(UPLOAD_FOLDER, str(project.project_id)), + project.runner.value, + False) + data["message"] = "Successfully fetched the submissions" data["url"] = urljoin(f"{API_HOST}/", f"/submissions/{submission.submission_id}") data["data"] = { @@ -149,7 +162,7 @@ def post(self) -> dict[str, any]: "submission_time": submission.submission_time, "submission_status": submission.submission_status } - return data, 201 + return data, 202 except exc.SQLAlchemyError: session.rollback() diff --git a/backend/project/executor.py b/backend/project/executor.py new file mode 100644 index 00000000..12405d1e --- /dev/null +++ b/backend/project/executor.py @@ -0,0 +1,9 @@ +""" +This file is used to create an instance of the Executor class from the flask_executor package. +This instance is used to create a background task that will run the evaluator. +This is done to prevent the server from being blocked while the model is being trained. +""" + +from flask_executor import Executor + +executor = Executor() diff --git a/backend/project/utils/submissions/evaluator.py b/backend/project/utils/submissions/evaluator.py index e2ebca00..da81aefc 100644 --- a/backend/project/utils/submissions/evaluator.py +++ b/backend/project/utils/submissions/evaluator.py @@ -8,16 +8,18 @@ exit code is returned. The output of the evaluator is written to a log file in the submission output folder. """ -from os import path +from os import path, makedirs import docker +from sqlalchemy.exc import SQLAlchemyError +from project.db_in import db from project.models.submission import Submission DOCKER_IMAGE_MAPPER = { - "python": path.join(path.dirname(__file__), "evaluators", "python"), + "PYTHON": path.join(path.dirname(__file__), "evaluators", "python"), } -def evaluate(submission: Submission, project_path: str, evaluator: str) -> int: +def evaluate(submission: Submission, project_path: str, evaluator: str, is_late: bool) -> int: """ Evaluate a submission using the evaluator. @@ -51,6 +53,7 @@ def evaluate(submission: Submission, project_path: str, evaluator: str) -> int: submission_solution_path) submission_output_path = path.join(submission_path, "output") + makedirs(submission_output_path, exist_ok=True) test_output_path = path.join(submission_output_path, "test_output.log") exit_code = container.wait() @@ -62,6 +65,38 @@ def evaluate(submission: Submission, project_path: str, evaluator: str) -> int: return exit_code['StatusCode'] +def run_evaluator(submission: Submission, project_path: str, evaluator: str, is_late: bool) -> int: + """ + Run the evaluator for the submission. + + Args: + submission (Submission): The submission to evaluate. + project_path (str): The path to the project. + evaluator (str): The evaluator to use. + is_late (bool): Whether the submission is late. + + Returns: + int: The exit code of the evaluator. + """ + status_code = evaluate(submission, project_path, evaluator, is_late) + + if not is_late: + if status_code == 0: + submission.submission_status = 'SUCCESS' + else: + submission.submission_status = 'FAIL' + else: + submission.submission_status = 'LATE' + + try: + db.session.merge(submission) + db.session.commit() + except SQLAlchemyError: + pass + + return status_code + + def create_and_run_evaluator(docker_image: str, submission_id: int, project_path: str, diff --git a/backend/requirements.txt b/backend/requirements.txt index 8733be9e..9b016df4 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -11,3 +11,4 @@ SQLAlchemy~=2.0.27 requests>=2.31.0 waitress flask_swagger_ui +flask_executor diff --git a/backend/tests/utils/submission_evaluators/python_test.py b/backend/tests/utils/submission_evaluators/python_test.py index 8db67349..0d8b548c 100644 --- a/backend/tests/utils/submission_evaluators/python_test.py +++ b/backend/tests/utils/submission_evaluators/python_test.py @@ -23,7 +23,7 @@ def evaluate_python(submission_root, project_path_succes): project_id = 1 submission = Submission(submission_id=1, project_id=project_id) submission.submission_path = create_submission_folder(submission_root, project_id) - return evaluate(submission, project_path_succes, "python"), submission.submission_path + return evaluate(submission, project_path_succes, "PYTHON", False), submission.submission_path def prep_submission_and_clear_after_py(tc_folder: str) -> tuple[Submission, Project]: """ @@ -64,13 +64,13 @@ def test_logs_output(evaluate_python): def test_with_dependency(): """Test whether the evaluator works with a dependency.""" submission, project = prep_submission_and_clear_after_py("tc_2") - exit_code = evaluate(submission, project, "python") + exit_code = evaluate(submission, project, "PYTHON", False) cleanup_after_test(submission.submission_path) assert exit_code == 0 def test_dependency_manifest(): """Test whether the evaluator works with a dependency manifest.""" submission, project = prep_submission_and_clear_after_py("tc_3") - exit_code = evaluate(submission, project, "python") + exit_code = evaluate(submission, project, "PYTHON", False) cleanup_after_test(submission.submission_path) assert exit_code != 0 diff --git a/backend/tests/utils/submission_evaluators/resources/python/tc_1/assignment/run_test.zip b/backend/tests/utils/submission_evaluators/resources/python/tc_1/assignment/run_test.zip new file mode 100644 index 0000000000000000000000000000000000000000..8d1ad5d86ede7f51c979b2291880c946429049cc GIT binary patch literal 184 zcmWIWW@h1H0D%dUJ0ld&HH*mr*&xgf#6_if@g=FnC3?jfsmU4n3Q8WSIXU?{3gP)h zIVp-tTwJLYnI#Ga0p5&E_6)d;Q2`nR0t#Rf#ZWG&AR~hWgMXLkw^?1c7@?}*RDd@t P8;HXQgt0){6)Xb)*T5iZ literal 0 HcmV?d00001 From 710526e713bd9bc977c744db83a77a48192ed2a5 Mon Sep 17 00:00:00 2001 From: Aron Buzogany <108480125+AronBuzogany@users.noreply.github.com> Date: Sun, 14 Apr 2024 14:05:07 +0200 Subject: [PATCH 2/2] Check if submission is LATE (#196) * checking for if deadline is on time * Fix #82 * removed redundant print statement * authenticate endpoint * setting submission status when no runner * fixed inverted boolean --- backend/project/endpoints/submissions.py | 17 +++++++++++++---- .../endpoints/submissions/submission_detail.py | 0 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 backend/project/endpoints/submissions/submission_detail.py diff --git a/backend/project/endpoints/submissions.py b/backend/project/endpoints/submissions.py index 5b3412f0..528b0e94 100644 --- a/backend/project/endpoints/submissions.py +++ b/backend/project/endpoints/submissions.py @@ -3,6 +3,7 @@ from urllib.parse import urljoin from datetime import datetime from os import getenv, path, makedirs +from zoneinfo import ZoneInfo from shutil import rmtree from dotenv import load_dotenv from flask import Blueprint, request @@ -28,6 +29,8 @@ UPLOAD_FOLDER = getenv("UPLOAD_FOLDER") BASE_URL = urljoin(f"{API_HOST}/", "/submissions") +TIMEZONE = getenv("TIMEZONE", "GMT") + submissions_bp = Blueprint("submissions", __name__) class SubmissionsEndpoint(Resource): @@ -108,10 +111,7 @@ def post(self) -> dict[str, any]: submission.project_id = int(project_id) # Submission time - submission.submission_time = datetime.now() - - # Submission status - submission.submission_status = SubmissionStatus.RUNNING + submission.submission_time = datetime.now(ZoneInfo(TIMEZONE)) # Submission files submission.submission_path = "" # Must be set on creation @@ -126,6 +126,12 @@ def post(self) -> dict[str, any]: f"(required files={','.join(project.regex_expressions)})" return data, 400 + deadlines = project.deadlines + is_late = True + for deadline in deadlines: + if submission.submission_time < deadline.deadline: + is_late = False + # Submission_id needed for the file location session.add(submission) session.commit() @@ -151,6 +157,9 @@ def post(self) -> dict[str, any]: path.join(UPLOAD_FOLDER, str(project.project_id)), project.runner.value, False) + else: + submission.submission_status = SubmissionStatus.LATE if is_late \ + else SubmissionStatus.SUCCESS data["message"] = "Successfully fetched the submissions" data["url"] = urljoin(f"{API_HOST}/", f"/submissions/{submission.submission_id}") diff --git a/backend/project/endpoints/submissions/submission_detail.py b/backend/project/endpoints/submissions/submission_detail.py new file mode 100644 index 00000000..e69de29b