From 543d961a40ddde974034edd69eab6ca8d65f846c Mon Sep 17 00:00:00 2001 From: Cedric Mekeirle <143823820+JibrilExe@users.noreply.github.com> Date: Fri, 23 Feb 2024 18:52:42 +0100 Subject: [PATCH] Backend/feature/db models (#19) * Basic db definition in flask * main.py * Defined official db model in SQLalchemy * db model definitions in sqlalchemy * flask_sqlalchemy in requirements * foutje * fixed db_uri * db initialization inside create_app * db_uri and code clean * import order fix * database uri added * .env added to ignore * Updated docs and .gitignore * A first succesfull test for user model * Doc cleanup and test function for courses and course_relations models added * Project and submission test added * added psycopg to dependencies * dockerized tests to host postgres server * created test script * created test directory to test models * waiting for postgres service to start before running test scripts and moved env variables * changed github action to run test script instead * constructing pytests for models * running test script with sudo * adding bash to run script * fixing pytest * pytests fixed, 1 warning left * warning fix * fixed: run github action job on self-hosted runner * fixed: no longer running test script with privileges * added: installing docker-compose to run our backend tests * fixed: no longer running compose install with permissions * using ubuntu-latest runner until docker-compose is installed on our self-hosted runner --------- Co-authored-by: warre Co-authored-by: Aron Buzogany Co-authored-by: abuzogan --- .github/workflows/ci-tests.yml | 6 +- backend/.gitignore | 3 +- backend/Dockerfile.test | 15 +++ backend/project/__init__.py | 21 +++- backend/project/__main__.py | 7 +- backend/project/endpoints/index.py | 11 +- backend/project/models/__init__.py | 0 backend/project/models/course_relations.py | 31 ++++++ backend/project/models/courses.py | 15 +++ backend/project/models/projects.py | 28 +++++ backend/project/models/submissions.py | 25 +++++ backend/project/models/users.py | 16 +++ backend/requirements.txt | 3 + backend/run_tests.sh | 20 ++++ backend/tests.yaml | 31 ++++++ backend/tests/models/conftest.py | 104 ++++++++++++++++++ backend/tests/models/course_test.py | 96 ++++++++++++++++ .../models/projects_and_submissions_test.py | 58 ++++++++++ backend/tests/models/users_test.py | 19 ++++ 19 files changed, 499 insertions(+), 10 deletions(-) create mode 100644 backend/Dockerfile.test create mode 100644 backend/project/models/__init__.py create mode 100644 backend/project/models/course_relations.py create mode 100644 backend/project/models/courses.py create mode 100644 backend/project/models/projects.py create mode 100644 backend/project/models/submissions.py create mode 100644 backend/project/models/users.py create mode 100644 backend/run_tests.sh create mode 100644 backend/tests.yaml create mode 100644 backend/tests/models/conftest.py create mode 100644 backend/tests/models/course_test.py create mode 100644 backend/tests/models/projects_and_submissions_test.py create mode 100644 backend/tests/models/users_test.py diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index c0d4cfa4..2231c368 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -37,7 +37,7 @@ jobs: working-directory: ./frontend run: npm run lint Backend-tests: - runs-on: self-hosted + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -50,10 +50,10 @@ jobs: - name: Install dependencies working-directory: ./backend run: pip3 install -r requirements.txt && pip3 install -r dev-requirements.txt - + - name: Running tests working-directory: ./backend - run: pytest + run: bash ./run_tests.sh - name: Run linting working-directory: ./backend diff --git a/backend/.gitignore b/backend/.gitignore index c5c0cc3f..25343357 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -7,4 +7,5 @@ __pycache__/ htmlcov/ docs/_build/ dist/ -venv/ \ No newline at end of file +venv/ +.env diff --git a/backend/Dockerfile.test b/backend/Dockerfile.test new file mode 100644 index 00000000..f975133c --- /dev/null +++ b/backend/Dockerfile.test @@ -0,0 +1,15 @@ +FROM python:3.9-slim + +# Set the working directory +WORKDIR /app + +# Copy the application code into the container +COPY . /app + +# Install dependencies +RUN apt-get update +RUN apt-get install -y --no-install-recommends python3-pip +RUN pip3 install --no-cache-dir -r requirements.txt -r dev-requirements.txt + +# Command to run the tests +CMD ["pytest"] \ No newline at end of file diff --git a/backend/project/__init__.py b/backend/project/__init__.py index 98017b80..66435003 100644 --- a/backend/project/__init__.py +++ b/backend/project/__init__.py @@ -2,17 +2,34 @@ This file is the base of the Flask API. It contains the basic structure of the API. """ -from flask import Flask, jsonify +from flask import Flask +from flask_sqlalchemy import SQLAlchemy from .endpoints.index import index_bp +db = SQLAlchemy() + def create_app(): """ Create a Flask application instance. Returns: Flask -- A Flask application instance """ - app = Flask(__name__) + app = Flask(__name__) app.register_blueprint(index_bp) return app + +def create_app_with_db(db_uri:str): + """ + Initialize the database with the given uri + and connect it to the app made with create_app. + Parameters: + db_uri (str): The URI of the database to initialize. + Returns: + Flask -- A Flask application instance + """ + app = create_app() + app.config["SQLALCHEMY_DATABASE_URI"] = db_uri + db.init_app(app) + return app diff --git a/backend/project/__main__.py b/backend/project/__main__.py index 0e79612a..9448b0eb 100644 --- a/backend/project/__main__.py +++ b/backend/project/__main__.py @@ -1,9 +1,12 @@ """Main entry point for the application.""" from sys import path +from os import getenv +from dotenv import load_dotenv +from project import create_app_with_db path.append(".") if __name__ == "__main__": - from project import create_app - app = create_app() + load_dotenv() + app = create_app_with_db(getenv("DB_HOST")) app.run(debug=True) diff --git a/backend/project/endpoints/index.py b/backend/project/endpoints/index.py index b5536eaf..e5b68473 100644 --- a/backend/project/endpoints/index.py +++ b/backend/project/endpoints/index.py @@ -1,10 +1,17 @@ +"""Index api point""" from flask import Blueprint from flask_restful import Resource index_bp = Blueprint("index", __name__) + class Index(Resource): + """Api endpoint for the / route""" + def get(self): + """Example of an api endpoint function that will respond to get requests made to / + return a json data structure with key Message and value Hello World!""" return {"Message": "Hello World!"} - -index_bp.add_url_rule("/", view_func=Index.as_view("index")) \ No newline at end of file + + +index_bp.add_url_rule("/", view_func=Index.as_view("index")) diff --git a/backend/project/models/__init__.py b/backend/project/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/project/models/course_relations.py b/backend/project/models/course_relations.py new file mode 100644 index 00000000..0677ef54 --- /dev/null +++ b/backend/project/models/course_relations.py @@ -0,0 +1,31 @@ +"""Models for relation between users and courses""" +# pylint: disable=too-few-public-methods + +from sqlalchemy import Integer, Column, ForeignKey, PrimaryKeyConstraint, String +from project import db +from project.models.users import Users +from project.models.courses import Courses + +class BaseCourseRelation(db.Model): + """Base class for course relation models, + both course relation tables have a + course_id of the course to wich someone is related and + an uid of the related person""" + + __abstract__ = True + + course_id = Column(Integer, ForeignKey('courses.course_id'), nullable=False) + uid = Column(String(255), ForeignKey("users.uid"), nullable=False) + __table_args__ = ( + PrimaryKeyConstraint("course_id", "uid"), + ) + +class CourseAdmins(BaseCourseRelation): + """Admin to course relation model""" + + __tablename__ = "course_admins" + +class CourseStudents(BaseCourseRelation): + """Student to course relation model""" + + __tablename__ = "course_students" diff --git a/backend/project/models/courses.py b/backend/project/models/courses.py new file mode 100644 index 00000000..0881ca56 --- /dev/null +++ b/backend/project/models/courses.py @@ -0,0 +1,15 @@ +"""The Courses model""" +# pylint: disable=too-few-public-methods +from sqlalchemy import Integer, Column, ForeignKey, String +from project import db +from project.models.users import Users + +class Courses(db.Model): + """This class described the courses table, + a course has an id, name, optional ufora id and the teacher that created it""" + + __tablename__ = "courses" + course_id = Column(Integer, primary_key=True) + name = Column(String(50), nullable=False) + ufora_id = Column(String(50), nullable=True) + teacher = Column(String(255), ForeignKey("users.uid"), nullable=False) diff --git a/backend/project/models/projects.py b/backend/project/models/projects.py new file mode 100644 index 00000000..a64f6082 --- /dev/null +++ b/backend/project/models/projects.py @@ -0,0 +1,28 @@ +"""Model for projects""" +# pylint: disable=too-few-public-methods +from sqlalchemy import ARRAY, Boolean, Column, DateTime, ForeignKey, Integer, String, Text +from project import db +from project.models.courses import Courses + +class Projects(db.Model): + """This class describes the projects table, + a projects has an id, a title, a description, + an optional assignment file that can contain more explanation of the projects, + an optional deadline, + the course id of the course to which the project belongs, + visible for students variable so a teacher can decide if the students can see it yet, + archieved var so we can implement the archiving functionality, + a test path,script name and regex experssions for automated testing""" + + __tablename__ = "projects" + project_id = Column(Integer, primary_key=True) + title = Column(String(50), nullable=False, unique=False) + descriptions = Column(Text, nullable=False) + assignment_file = Column(String(50)) + deadline = Column(DateTime(timezone=True)) + course_id = Column(Integer, ForeignKey("courses.course_id"), nullable=False) + visible_for_students = Column(Boolean, nullable=False) + archieved = Column(Boolean, nullable=False) + test_path = Column(String(50)) + script_name = Column(String(50)) + regex_expressions = Column(ARRAY(String(50))) diff --git a/backend/project/models/submissions.py b/backend/project/models/submissions.py new file mode 100644 index 00000000..f9de28b4 --- /dev/null +++ b/backend/project/models/submissions.py @@ -0,0 +1,25 @@ +"""Model for submissions""" +# pylint: disable=too-few-public-methods +from sqlalchemy import Column,String,ForeignKey,Integer,CheckConstraint,DateTime,Boolean +from project import db +from project.models.users import Users + +class Submissions(db.Model): + """This class describes the submissions table, + submissions can be made to a project, a submission has + and id, a uid from the user that uploaded it, + the project id of the related project, + an optional grading, + the submission time, + submission path, + and finally the submission status + so we can easily present in a list which submission succeeded the automated checks""" + + __tablename__ = "submissions" + submission_id = Column(Integer, nullable=False, primary_key=True) + uid = Column(String(255), ForeignKey("users.uid"), nullable=False) + project_id = Column(Integer, ForeignKey("projects.project_id"), nullable=False) + grading = Column(Integer, CheckConstraint("grading >= 0 AND grading <= 20")) + submission_time = Column(DateTime(timezone=True), nullable=False) + submission_path = Column(String(50), nullable=False) + submission_status = Column(Boolean, nullable=False) diff --git a/backend/project/models/users.py b/backend/project/models/users.py new file mode 100644 index 00000000..c5af5287 --- /dev/null +++ b/backend/project/models/users.py @@ -0,0 +1,16 @@ +"""Model for users""" +# pylint: disable=too-few-public-methods +from sqlalchemy import Boolean, Column, String +from project import db + + +class Users(db.Model): + """This class defines the users table, + a user has an uid, + is_teacher and is_admin booleans because a user + can be either a student,admin or teacher""" + + __tablename__ = "users" + uid = Column(String(255), primary_key=True) + is_teacher = Column(Boolean) + is_admin = Column(Boolean) diff --git a/backend/requirements.txt b/backend/requirements.txt index 9687a048..943fb75a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,2 +1,5 @@ flask flask-restful +flask-sqlalchemy +python-dotenv +psycopg2-binary \ No newline at end of file diff --git a/backend/run_tests.sh b/backend/run_tests.sh new file mode 100644 index 00000000..a35d5cb2 --- /dev/null +++ b/backend/run_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Run Docker Compose to build and start the services, and capture the exit code from the test runner service +docker-compose -f tests.yaml up --build --exit-code-from test-runner + +# Store the exit code in a variable +exit_code=$? + +# After the tests are finished, stop and remove the containers +docker-compose -f tests.yaml down + +# Check the exit code to determine whether the tests passed or failed +if [ $exit_code -eq 0 ]; then + echo "Tests passed!" +else + echo "Tests failed!" +fi + +# Exit with the same exit code as the test runner service +exit $exit_code diff --git a/backend/tests.yaml b/backend/tests.yaml new file mode 100644 index 00000000..43e401c9 --- /dev/null +++ b/backend/tests.yaml @@ -0,0 +1,31 @@ +version: '3.8' + +services: + postgres: + image: postgres:latest + environment: + POSTGRES_USER: test_user + POSTGRES_PASSWORD: test_password + POSTGRES_DB: test_database + healthcheck: + test: ["CMD-SHELL", "pg_isready -U test_user -d test_database"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s + + test-runner: + build: + context: . + dockerfile: Dockerfile.test + depends_on: + postgres: + condition: service_healthy + environment: + POSTGRES_HOST: postgres # Use the service name defined in Docker Compose + POSTGRES_USER: test_user + POSTGRES_PASSWORD: test_password + POSTGRES_DB: test_database + volumes: + - .:/app + command: ["pytest"] diff --git a/backend/tests/models/conftest.py b/backend/tests/models/conftest.py new file mode 100644 index 00000000..64e4461c --- /dev/null +++ b/backend/tests/models/conftest.py @@ -0,0 +1,104 @@ +""" + +""" +import os +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from datetime import datetime +from project import db +from project.models.courses import Courses +from project.models.course_relations import CourseAdmins, CourseStudents +from project.models.projects import Projects +from project.models.submissions import Submissions +from project.models.users import Users +from sqlalchemy.engine.url import URL +from dotenv import load_dotenv +import pytest + +load_dotenv() + +DATABSE_NAME = os.getenv('POSTGRES_DB') +DATABASE_USER = os.getenv('POSTGRES_USER') +DATABASE_PASSWORD = os.getenv('POSTGRES_PASSWORD') +DATABASE_HOST = os.getenv('POSTGRES_HOST') + +url = URL.create( + drivername="postgresql", + username=DATABASE_USER, + host=DATABASE_HOST, + database=DATABSE_NAME, + password=DATABASE_PASSWORD +) + +engine = create_engine(url) +Session = sessionmaker(bind=engine) + +@pytest.fixture +def db_session(): + db.metadata.create_all(engine) + session = Session() + yield session + session.rollback() + session.close() + # Truncate all tables + for table in reversed(db.metadata.sorted_tables): + session.execute(table.delete()) + session.commit() + +@pytest.fixture +def valid_user(): + user = Users(uid="student", is_teacher=False, is_admin=False) + return user + +@pytest.fixture +def teachers(): + users = [Users(uid=str(i), is_teacher=True, is_admin=False) for i in range(10)] + return users + +@pytest.fixture +def course_teacher(): + sel2_teacher = Users(uid="Bart", is_teacher=True, is_admin=False) + return sel2_teacher + +@pytest.fixture +def course(course_teacher): + sel2 = Courses(name="Sel2", teacher=course_teacher.uid) + return sel2 + +@pytest.fixture +def course_students(): + students = [ + Users(uid="student_sel2_" + str(i), is_teacher=False, is_admin=False) + for i in range(5) + ] + return students + +@pytest.fixture +def course_students_relation(course,course_students): + course_relations = [ + CourseStudents(course_id=course.course_id, uid=course_students[i].uid) + for i in range(5) + ] + return course_relations + +@pytest.fixture +def assistent(): + assist = Users(uid="assistent_sel2") + return assist + +@pytest.fixture() +def course_admin(course,assistent): + admin_relation = CourseAdmins(uid=assistent.uid, course_id=course.course_id) + return admin_relation + +@pytest.fixture() +def valid_project(course): + deadline = datetime(2024, 2, 25, 12, 0, 0) # February 25, 2024, 12:00 PM + project = Projects( + title="Project", + descriptions="Test project", + deadline=deadline, + visible_for_students=True, + archieved=False, + ) + return project diff --git a/backend/tests/models/course_test.py b/backend/tests/models/course_test.py new file mode 100644 index 00000000..d8b92e1a --- /dev/null +++ b/backend/tests/models/course_test.py @@ -0,0 +1,96 @@ +import pytest +from sqlalchemy.exc import IntegrityError +from psycopg2.errors import ForeignKeyViolation +from project.models.courses import Courses +from project.models.users import Users +from project.models.course_relations import CourseAdmins, CourseStudents + + +class TestCoursesModel: + """Test class for the database models""" + + def test_foreignkey_courses_teacher(self, db_session, course: Courses): + """Tests the foreign key relation between courses and the teacher uid""" + with pytest.raises( + IntegrityError + ): + db_session.add(course) + db_session.commit() + + def test_correct_course(self, db_session, course: Courses, course_teacher: Users): + """Tests wether added course and a teacher are correctly connected""" + db_session.add(course_teacher) + db_session.commit() + + db_session.add(course) + db_session.commit() + assert ( + db_session.query(Courses).filter_by(name=course.name).first().teacher + == course_teacher.uid + ) + + def test_foreignkey_coursestudents_uid( + self, db_session, course, course_teacher, course_students_relation + ): + """Test the foreign key of the CourseStudent related to the student uid""" + db_session.add(course_teacher) + db_session.commit() + + db_session.add(course) + db_session.commit() + for s in course_students_relation: + s.course_id = course.course_id + + with pytest.raises( + IntegrityError + ): + db_session.add_all(course_students_relation) + db_session.commit() + + def test_correct_courserelations( + self, + db_session, + course, + course_teacher, + course_students, + course_students_relation, + assistent, + course_admin, + ): + """Tests if we get the expected results for correct usage of CourseStudents and CourseAdmins""" + db_session.add(course_teacher) + db_session.commit() + + db_session.add(course) + db_session.commit() + + db_session.add_all(course_students) + db_session.commit() + + for s in course_students_relation: + s.course_id = course.course_id + db_session.add_all(course_students_relation) + db_session.commit() + + student_check = [ + s.uid + for s in db_session.query(CourseStudents) + .filter_by(course_id=course.course_id) + .all() + ] + student_uids = [s.uid for s in course_students] + assert student_check == student_uids + + db_session.add(assistent) + db_session.commit() + course_admin.course_id = course.course_id + db_session.add(course_admin) + db_session.commit() + + assert ( + db_session.query(CourseAdmins) + .filter_by(course_id=course.course_id) + .first() + .uid + == assistent.uid + ) diff --git a/backend/tests/models/projects_and_submissions_test.py b/backend/tests/models/projects_and_submissions_test.py new file mode 100644 index 00000000..e79eaf93 --- /dev/null +++ b/backend/tests/models/projects_and_submissions_test.py @@ -0,0 +1,58 @@ +from datetime import datetime +import pytest +from sqlalchemy.exc import IntegrityError +from project.models.courses import Courses +from project.models.course_relations import CourseAdmins, CourseStudents +from project.models.projects import Projects +from project.models.submissions import Submissions +from project.models.users import Users + +class TestProjectsAndSubmissionsModel: + def test_deadline(self,db_session,course,course_teacher,valid_project,valid_user): + db_session.add(course_teacher) + db_session.commit() + db_session.add(course) + db_session.commit() + valid_project.course_id = course.course_id + db_session.add(valid_project) + db_session.commit() + check_project = ( + db_session.query(Projects).filter_by(title=valid_project.title).first() + ) + assert check_project.deadline == valid_project.deadline + + db_session.add(valid_user) + db_session.commit() + submission = Submissions( + uid=valid_user.uid, + project_id=check_project.project_id, + submission_time=datetime.now(), + submission_path="/test/submission/", + submission_status=False, + ) + db_session.add(submission) + db_session.commit() + + submission_check = ( + db_session.query(Submissions) + .filter_by(project_id=check_project.project_id) + .first() + ) + assert submission_check.uid == valid_user.uid + + with pytest.raises( + IntegrityError + ): + submission_check.grading = 100 + db_session.commit() + db_session.rollback() + submission_check.grading = 15 + db_session.commit() + submission_check = ( + db_session.query(Submissions) + .filter_by(project_id=check_project.project_id) + .first() + ) + assert submission_check.grading == 15 + assert submission.grading == 15 + # Interesting! all the model objects are connected \ No newline at end of file diff --git a/backend/tests/models/users_test.py b/backend/tests/models/users_test.py new file mode 100644 index 00000000..e9e1acaf --- /dev/null +++ b/backend/tests/models/users_test.py @@ -0,0 +1,19 @@ +from project.models.users import Users + + +class TestUserModel: + """Test class for the database models""" + + def test_valid_user(self, db_session, valid_user): + db_session.add(valid_user) + db_session.commit() + assert valid_user in db_session.query(Users).all() + + def test_is_teacher(self, db_session, teachers): + db_session.add_all(teachers) + db_session.commit() + teacher_count = 0 + for usr in db_session.query(Users).filter_by(is_teacher=True): + teacher_count += 1 + assert usr.is_teacher + assert teacher_count == 10 \ No newline at end of file