From ff90ff4ac0effd257570c5f5ffb10f77d5157faf Mon Sep 17 00:00:00 2001 From: Cedric Mekeirle <143823820+JibrilExe@users.noreply.github.com> Date: Sat, 4 May 2024 11:20:59 +0200 Subject: [PATCH] Seeder for easier demo and frontend testing (#291) * basic seeder provider, course titles, usernames and random uid * seeder creates and populates a few courses with random students where given uid is teacher * ffix * relocate, weird project deadline bug tho * seeder go bvv * fun with linter * lint * removed leading spaces * batch operations and documentation cleanup * bad env name * more batch and better try except finally * specific error * close * projects have 0-2 deadlines now, also randomized numprojects from fixed 2 to random 1-3 * 1-3 * import not good * toml test * toml * faker is dev * linter mad * titles now in txt file * completed toml info * fixed parsing error --------- Co-authored-by: Aron Buzogany --- backend/db_construct.sql | 4 +- backend/dev-requirements.txt | 1 + backend/pyproject.toml | 13 ++ backend/requirements.txt | 2 +- backend/seeder/__init__.py | 0 backend/seeder/seeder.py | 261 +++++++++++++++++++++++++++++++++++ backend/seeder/titles.txt | 208 ++++++++++++++++++++++++++++ 7 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 backend/pyproject.toml create mode 100644 backend/seeder/__init__.py create mode 100644 backend/seeder/seeder.py create mode 100644 backend/seeder/titles.txt diff --git a/backend/db_construct.sql b/backend/db_construct.sql index f9a31e60..b4614151 100644 --- a/backend/db_construct.sql +++ b/backend/db_construct.sql @@ -47,7 +47,7 @@ CREATE TYPE deadline AS( CREATE TABLE projects ( project_id INT GENERATED ALWAYS AS IDENTITY, - title VARCHAR(50) NOT NULL, + title VARCHAR(100) NOT NULL, description TEXT NOT NULL, deadlines deadline[], course_id INT NOT NULL, @@ -65,7 +65,7 @@ CREATE TABLE submissions ( project_id INT NOT NULL, grading FLOAT CHECK (grading >= 0 AND grading <= 20), submission_time TIMESTAMP WITH TIME ZONE NOT NULL, - submission_path VARCHAR(50) NOT NULL, + submission_path VARCHAR(255) NOT NULL, submission_status submission_status NOT NULL, PRIMARY KEY(submission_id), CONSTRAINT fk_project FOREIGN KEY(project_id) REFERENCES projects(project_id) ON DELETE CASCADE, diff --git a/backend/dev-requirements.txt b/backend/dev-requirements.txt index fa950d3d..0263c7e7 100644 --- a/backend/dev-requirements.txt +++ b/backend/dev-requirements.txt @@ -2,3 +2,4 @@ pytest pylint pylint-flask pyyaml +faker \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 00000000..c1c45b42 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,13 @@ +[tool.poetry] +name = "Peristeronas" +version = "1.0" +description = "Project submission platform" +authors = ["Aron","Gerwoud","Siebe","Matisse","Warre","Cedric"] +packages = [ + { include = "project/models" }, + { include = "seeder" }, +] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt index 12a02f7e..529099ca 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,4 +12,4 @@ SQLAlchemy~=2.0.27 requests>=2.31.0 waitress flask_swagger_ui -flask_executor +flask_executor \ No newline at end of file diff --git a/backend/seeder/__init__.py b/backend/seeder/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/seeder/seeder.py b/backend/seeder/seeder.py new file mode 100644 index 00000000..20b320c0 --- /dev/null +++ b/backend/seeder/seeder.py @@ -0,0 +1,261 @@ +"""Seeder file does the actual seeding of the db""" +import argparse +import os +import random +import string +from datetime import datetime, timedelta + +from dotenv import load_dotenv +from faker import Faker +from faker.providers import DynamicProvider +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy_utils import register_composites + +from project.models.course import Course +from project.models.course_relation import CourseAdmin, CourseStudent +from project.models.project import Project +from project.models.submission import Submission, SubmissionStatus +from project.models.user import User +from project.sessionmaker import Session as session_maker + +load_dotenv() + +UPLOAD_URL = os.getenv("UPLOAD_FOLDER") + +fake = Faker() + +# Get the directory of the current script +script_dir = os.path.dirname(os.path.realpath(__file__)) + +# Construct the path to titles.txt relative to the script directory +titles_path = os.path.join(script_dir, 'titles.txt') + +with open(titles_path, 'r', encoding='utf-8') as file: + # Read the lines of the file and strip newline characters + titles = [line.strip() for line in file] + +course_title_provider = DynamicProvider( # Custom course titles. + provider_name="course_titles", + elements=titles, +) +fake.add_provider(course_title_provider) + + +def generate_course_name(): + """Generates a course name chosen from the predefined provider""" + return fake.course_titles() + + +def generate_random_uid(length=8): + """Generates a random uid of given length""" + characters = string.ascii_letters + string.digits + return ''.join(random.choice(characters) for _ in range(length)) + + +def teacher_generator(): + """Generates a teacher user object""" + return user_generator('TEACHER') + + +def student_generator(): + """Generates a student user object""" + return user_generator('STUDENT') + + +def admin_generator(): + """Generates an admin user object""" + return user_generator('ADMIN') + + +def user_generator(role): + """Generates a user object with the given role""" + user = User(uid=generate_random_uid(), + role=role, + display_name=fake.name()) + return user + + +def course_student_generator(course_id, uid): + """Generates a course student relation object""" + return CourseStudent(course_id=course_id, uid=uid) + + +def course_admin_generator(course_id, uid): + """Generates a course admin relation object""" + return CourseAdmin(course_id=course_id, uid=uid) + + +def generate_course(teacher_uid): + """Generates a course object with a random name and the given teacher uid""" + course = Course(name=generate_course_name(), + teacher=teacher_uid) + return course + + +def generate_projects(course_id, num_projects): + """Generates a list of project objects with random future deadlines""" + projects = [] + for _ in range(num_projects): + deadlines = [] + # Generate a random number of deadlines (0-2) + num_deadlines = random.randint(0, 2) + + for _ in range(num_deadlines): + if random.random() < 1/3: + past_datetime = datetime.now() - timedelta(days=random.randint(1, 30)) + deadline = (fake.catch_phrase(), past_datetime) + else: + future_datetime = datetime.now() + timedelta(days=random.randint(1, 30)) + deadline = (fake.catch_phrase(), future_datetime) + deadlines.append(deadline) + project = Project( + title=fake.catch_phrase(), + description=fake.catch_phrase(), + deadlines=deadlines, + course_id=course_id, + visible_for_students=random.choice([True, False]), + archived=random.choice([True, False]), + regex_expressions=[] + ) + projects.append(project) + return projects + + +def generate_submissions(project_id, student_uid): + """Generates a list of submissions with random status""" + submissions = [] + statusses = [SubmissionStatus.SUCCESS, SubmissionStatus.FAIL, + SubmissionStatus.LATE, SubmissionStatus.RUNNING] + num_submissions = random.randint(0, 2) + for _ in range(num_submissions): + submission = Submission(project_id=project_id, + uid=student_uid, + submission_time=datetime.now(), + submission_path="", + submission_status=random.choice(statusses)) + graded = random.choice([True, False]) + if graded and submission.submission_status == "SUCCESS": + submission.grading = random.randint(0, 20) + submissions.append(submission) + return submissions + + +def into_the_db(my_uid): + """Populates the db with 5 courses where my_uid is teacher and 5 where he is student""" + try: + session = session_maker() # setup the db session + connection = session.connection() + register_composites(connection) + + students = [] + # make a random amount of 100-200 students which we can use later to populate courses + num_students = random.randint(100, 200) + students = [student_generator() for _ in range(num_students)] + session.add_all(students) + session.commit() + + num_teachers = random.randint(5, 10) + teachers = [teacher_generator() for _ in range(num_teachers)] + session.add_all(teachers) + session.commit() # only after commit uid becomes available + + for _ in range(5): # 5 courses where my_uid is teacher + course_id = insert_course_into_db_get_id(session, my_uid) + # Add students to the course + subscribed_students = populate_course_students( + session, course_id, students) + populate_course_projects( + session, course_id, subscribed_students, my_uid) + + for _ in range(5): # 5 courses where my_uid is a student + teacher_uid = teachers[random.randint(0, len(teachers)-1)].uid + course_id = insert_course_into_db_get_id(session, teacher_uid) + subscribed_students = populate_course_students( + session, course_id, students) + subscribed_students.append(my_uid) # my_uid is also a student + populate_course_projects( + session, course_id, subscribed_students, teacher_uid) + except SQLAlchemyError as e: + if session: # possibly error resulted in session being null + session.rollback() + raise e + finally: + session.close() + + +def insert_course_into_db_get_id(session, teacher_uid): + """Inserts a course with teacher_uid as teacher into the db and returns the course_id""" + course = generate_course(teacher_uid) + session.add(course) + session.commit() + return course.course_id + + +def populate_course_students(session, course_id, students): + """Populates the course with students and returns their uids as a list""" + num_students_in_course = random.randint(5, 30) + subscribed_students = random.sample(students, num_students_in_course) + student_relations = [course_student_generator(course_id, student.uid) + for student in subscribed_students] + + session.add_all(student_relations) + session.commit() + + return [student.uid for student in subscribed_students] + + +def populate_course_projects(session, course_id, students, teacher_uid): + """Populates the course with projects and submissions, also creates the files""" + teacher_relation = course_admin_generator(course_id, teacher_uid) + session.add(teacher_relation) + session.commit() + + num_projects = random.randint(1, 3) + projects = generate_projects(course_id, num_projects) + session.add_all(projects) + session.commit() + for project in projects: + project_id = project.project_id + # Write assignment.md file + assignment_content = fake.text() + assignment_file_path = os.path.join( + UPLOAD_URL, "projects", str(project_id), "assignment.md") + os.makedirs(os.path.dirname(assignment_file_path), exist_ok=True) + with open(assignment_file_path, "w", encoding="utf-8") as assignment_file: + assignment_file.write(assignment_content) + populate_project_submissions(session, students, project_id) + + +def populate_project_submissions(session, students, project_id): + """Make submissions, 0 1 or 2 for each project per student""" + for student in students: + submissions = generate_submissions(project_id, student) + session.add_all(submissions) + session.commit() + for submission in submissions: + submission_directory = os.path.join(UPLOAD_URL, "projects", str( + project_id), "submissions", str(submission.submission_id), "submission") + os.makedirs(submission_directory, exist_ok=True) + submission_file_path = os.path.join( + submission_directory, "submission.md") + with open(submission_file_path, "w", encoding="utf-8") as submission_file: + submission_file.write(fake.text()) + + submission.submission_path = submission_directory + session.commit() # update submission path + +# Create a function to parse command line arguments +def parse_args(): + """Parse the given uid from the command line""" + parser = argparse.ArgumentParser(description='Populate the database') + parser.add_argument('my_uid', type=str, help='Your UID') + return parser.parse_args() + +# Main function to run when script is executed +def main(): + """Parse arguments, pass them to into_the_db function""" + args = parse_args() + into_the_db(args.my_uid) + +if __name__ == '__main__': + main() diff --git a/backend/seeder/titles.txt b/backend/seeder/titles.txt new file mode 100644 index 00000000..cd5a42ca --- /dev/null +++ b/backend/seeder/titles.txt @@ -0,0 +1,208 @@ +Computer Science +Principles of Economics +Modern Literature +Organic Chemistry +World History +Calculus and Analytic Geometry +Psychology +Microbiology Fundamentals +Principles of Marketing +Environmental Science +Sociology +Financial Accounting +Political Science and Government +Human Anatomy +Business Ethics +Philosophy +Statistics for Social Sciences +Cell Biology +Anthropology +Principles of Management +Macroeconomics +General Physics +English Composition +Human Physiology +Developmental Psychology +Linguistics +Genetics and Genomics +Principles of Finance +Art History +Microeconomics +Anatomy and Physiology +Marketing +Astronomy +Political Science +Microeconomics +Business +Cultural Anthropology +American History +World Religions +Chemistry +Environmental Science +Communication +General Chemistry +Cultural Anthropology +Human Biology +Theatre +Public Speaking +International Relations +Sociology +Criminal Justice +Statistics +Human Anatomy +Western Civilization +Literature +Biochemistry +Physical Anthropology +Human Physiology +Creative Writing +Film Studies +Music +Ethics +Philosophy of Science +Philosophy of Mind +Philosophy of Language +Political Philosophy +Philosophy of Religion +Epistemology +Metaphysics +Logic +Symbolic Logic +Modal Logic +Mathematical Logic +Computer Science +Programming +Algorithms +Data Structures +Software Engineering +Computer Networks +Operating Systems +Database Systems +Artificial Intelligence +Machine Learning +Computer Vision +Natural Language Processing +Robotics +Human-Computer Interaction +Virtual Reality +Augmented Reality +Web Development +Mobile Development +Development +Cybersecurity +Cryptography +Digital Forensics +Cloud Computing +Big Data +Data Science +Data Analytics +Data Visualization +Business Intelligence +Information Systems +E-commerce +Web Design +User Experience Design +User Interface Design +Graphic Design +Multimedia Design +Animation +Digital Art +Photography +Videography +Audio Production +Sound Design +Music Production +Film Production +Screenwriting +Directing +Cinematography +Film Editing +Visual Effects +Motion Graphics +3D Modeling +3D Animation +Design +Programming +Art +Sound +Narrative +Testing +Marketing +Publishing +Monetization +Analytics +Localization +Development Tools +Engines +AI +Networking +Servers +Security +Design Patterns +Theory +History +Culture +Studies +Journalism +Criticism +Law +Ethics +Philosophy +Psychology +Sociology +Anthropology +Archaeology +Economics +Marketing +Management +Development Methodologies +Production +Design Documents +Prototyping +Testing +Quality Assurance +Localization +Voice Acting +Music Composition +Sound Design +Art Direction +Animation +Character Design +Environment Design +Level Design +Storytelling +Writing +Dialogue +Cutscenes +Worldbuilding +Concept Art +Production Design +User Interface Design +User Experience Design +Interaction Design +Control Design +Camera Design +Interface Design +HUD Design +Menu Design +Navigation Design +Inventory Design +Map Design +Mini-map Design +Tutorial Design +Help System Design +Accessibility Design +Reward System Design +Progression System Design +Achievement System Design +Leaderboard Design +Social System Design +Community System Design +Feedback System Design +Chat System Design +Messaging System Design +Notification System Design +Ranking System Design +Matchmaking System Design +Voting System Design +Auction System Design \ No newline at end of file