Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Seeder (#340) #341

Merged
merged 1 commit into from
Apr 18, 2024
Merged

Seeder (#340) #341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/api/fixtures/large/large.json
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion backend/api/fixtures/medium/medium.json
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion backend/api/fixtures/small/small.json
View file
Open in desktop

Large diffs are not rendered by default.

63 changes: 43 additions & 20 deletions backend/api/seeders/seeder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from django.db import connection
from django.utils import timezone

generated_usernames = set()


def fillFaculties():
with connection.cursor() as cursor:
Expand Down Expand Up @@ -60,20 +62,39 @@ def handle(*args, **kwargs):
@timer
def seed_users(faker, count: int, offset: int = 0, staff_prob: float = 0.001) -> list[list]:
"""Seed users into the database"""
global generated_usernames
with connection.cursor() as cursor:
users = [
[
# Create a set to store generated usernames

users = []
for id in range(offset, count + offset):
# Generate username
first_name = faker.first_name()
last_name = faker.last_name()
username_base = first_name + last_name
username = username_base[:12] # Truncate if longer than 12 characters

# Check if the username is unique
suffix = 1
while username in generated_usernames:
username = username_base[:12 - len(str(suffix))] + str(suffix) # Append a unique suffix
suffix += 1

# Add the username to the set
generated_usernames.add(username)

# Append user data to the list
users.append([
id,
faker.unique.user_name(),
faker.unique.email(),
faker.first_name(),
faker.last_name(),
username,
username + "@ugent.be",
first_name,
last_name,
timezone.now().year,
timezone.now(),
timezone.now(),
faker.boolean(chance_of_getting_true=staff_prob),
] for id in range(offset, count + offset)
]
])

cursor.executemany(
"INSERT INTO authentication_user"
Expand Down Expand Up @@ -133,7 +154,8 @@ def seed_teachers(faker, count: int = 250, offset: int = 0) -> None:
@timer
def seed_courses(faker,
count: int = 1_000,
year_dev: int = 1,
year_dev_min: int = -3,
year_dev_max: int = 1,
max_students: int = 100,
max_teachers: int = 3,
max_assistants: int = 5,
Expand All @@ -155,9 +177,9 @@ def seed_courses(faker,
[
faker.catch_phrase(),
faker.paragraph(),
timezone.now().year + faker.random_int(min=-year_dev, max=year_dev),
timezone.now().year + faker.random_int(min=year_dev_min, max=year_dev_max),
choice(faculties),
faker.sentence()
""
] for _ in range(count)
]

Expand Down Expand Up @@ -257,14 +279,16 @@ def seed_projects(
)

# Create projects
projects = [
[
projects = []
for _ in range(count):
start_date = timezone.now() + timezone.timedelta(
days=faker.random_int(min=min_start_date_dev, max=max_start_date_dev)
)
projects.append([
faker.catch_phrase(),
faker.paragraph(),
timezone.now() + timezone.timedelta(
days=faker.random_int(min=min_start_date_dev, max=max_start_date_dev)
),
timezone.now() + timezone.timedelta(
start_date,
start_date + timezone.timedelta(
days=faker.random_int(min=min_deadline_dev, max=max_deadline_dev)
),
faker.random_int(min=min_max_score, max=max_max_score),
Expand All @@ -274,8 +298,7 @@ def seed_projects(
faker.boolean(chance_of_getting_true=visible_prob),
faker.boolean(chance_of_getting_true=archived_prob),
choice(courses)
] for _ in range(count)
]
])

# Insert projects
cursor.executemany(
Expand Down Expand Up @@ -466,7 +489,7 @@ def seed_submissions(faker, count: int = 4_000):
[
faker.date_this_month(),
choice(groups),
faker.boolean(chance_of_getting_true=80),
faker.boolean(chance_of_getting_true=80)
]
for _ in range(count)
]
Expand Down
2 changes: 1 addition & 1 deletion backend/authentication/fixtures/large/large.json
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion backend/authentication/fixtures/medium/medium.json
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion backend/authentication/fixtures/small/small.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
[{"model": "authentication.user", "pk": "0", "fields": {"last_login": "2024-04-18T18:41:07.505Z", "username": "michael78", "is_staff": false, "email": "[email protected]", "first_name": "Emily", "last_name": "Cline", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.505Z", "faculties": []}}, {"model": "authentication.user", "pk": "1", "fields": {"last_login": "2024-04-18T18:41:07.505Z", "username": "dhernandez", "is_staff": false, "email": "[email protected]", "first_name": "Jennifer", "last_name": "Moreno", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.505Z", "faculties": []}}, {"model": "authentication.user", "pk": "10", "fields": {"last_login": "2024-04-18T18:41:07.676Z", "username": "wross", "is_staff": false, "email": "[email protected]", "first_name": "Kelsey", "last_name": "Moore", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.676Z", "faculties": []}}, {"model": "authentication.user", "pk": "11", "fields": {"last_login": "2024-04-18T18:41:07.676Z", "username": "owallace", "is_staff": false, "email": "[email protected]", "first_name": "Caitlin", "last_name": "Mueller", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.676Z", "faculties": []}}, {"model": "authentication.user", "pk": "12", "fields": {"last_login": "2024-04-18T18:41:07.676Z", "username": "anna47", "is_staff": false, "email": "[email protected]", "first_name": "Kim", "last_name": "Hill", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.676Z", "faculties": []}}, {"model": "authentication.user", "pk": "13", "fields": {"last_login": "2024-04-18T18:41:07.677Z", "username": "jesseperez", "is_staff": false, "email": "[email protected]", "first_name": "Kenneth", "last_name": "Hart", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.677Z", "faculties": []}}, {"model": "authentication.user", "pk": "14", "fields": {"last_login": "2024-04-18T18:41:07.677Z", "username": "hufftheresa", "is_staff": false, "email": "[email protected]", "first_name": "John", "last_name": "Vaughn", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.677Z", "faculties": []}}, {"model": "authentication.user", "pk": "15", "fields": {"last_login": "2024-04-18T18:41:07.752Z", "username": "natalieedwards", "is_staff": false, "email": "[email protected]", "first_name": "Daniel", "last_name": "Rice", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.752Z", "faculties": []}}, {"model": "authentication.user", "pk": "16", "fields": {"last_login": "2024-04-18T18:41:07.753Z", "username": "osanchez", "is_staff": false, "email": "[email protected]", "first_name": "Rachel", "last_name": "Lin", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.753Z", "faculties": []}}, {"model": "authentication.user", "pk": "17", "fields": {"last_login": "2024-04-18T18:41:07.753Z", "username": "hectorerickson", "is_staff": false, "email": "[email protected]", "first_name": "Lori", "last_name": "Sanchez", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.753Z", "faculties": []}}, {"model": "authentication.user", "pk": "2", "fields": {"last_login": "2024-04-18T18:41:07.505Z", "username": "barnesleslie", "is_staff": false, "email": "[email protected]", "first_name": "Adam", "last_name": "Mckay", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.505Z", "faculties": []}}, {"model": "authentication.user", "pk": "3", "fields": {"last_login": "2024-04-18T18:41:07.506Z", "username": "jenny35", "is_staff": false, "email": "[email protected]", "first_name": "Johnny", "last_name": "Howard", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.506Z", "faculties": []}}, {"model": "authentication.user", "pk": "4", "fields": {"last_login": "2024-04-18T18:41:07.506Z", "username": "rachel41", "is_staff": false, "email": "[email protected]", "first_name": "Tyler", "last_name": "Smith", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.506Z", "faculties": []}}, {"model": "authentication.user", "pk": "5", "fields": {"last_login": "2024-04-18T18:41:07.506Z", "username": "chayden", "is_staff": false, "email": "[email protected]", "first_name": "Leslie", "last_name": "Becker", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.506Z", "faculties": []}}, {"model": "authentication.user", "pk": "6", "fields": {"last_login": "2024-04-18T18:41:07.506Z", "username": "priscillajohnson", "is_staff": false, "email": "[email protected]", "first_name": "Patrick", "last_name": "Alvarez", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.506Z", "faculties": []}}, {"model": "authentication.user", "pk": "7", "fields": {"last_login": "2024-04-18T18:41:07.507Z", "username": "amandaadams", "is_staff": false, "email": "[email protected]", "first_name": "Mary", "last_name": "Andersen", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.507Z", "faculties": []}}, {"model": "authentication.user", "pk": "8", "fields": {"last_login": "2024-04-18T18:41:07.507Z", "username": "thomas69", "is_staff": false, "email": "[email protected]", "first_name": "Wesley", "last_name": "Hodges", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.507Z", "faculties": []}}, {"model": "authentication.user", "pk": "9", "fields": {"last_login": "2024-04-18T18:41:07.507Z", "username": "stephanie50", "is_staff": false, "email": "[email protected]", "first_name": "James", "last_name": "Yates", "last_enrolled": 2024, "create_time": "2024-04-18T18:41:07.507Z", "faculties": []}}, {"model": "authentication.faculty", "pk": "Bio-ingenieurswetenschappen", "fields": {"name": "faculties.bioscience_engineering"}}, {"model": "authentication.faculty", "pk": "Diergeneeskunde", "fields": {"name": "faculties.veterinary_medicine"}}, {"model": "authentication.faculty", "pk": "Economie_Bedrijfskunde", "fields": {"name": "faculties.economics_business_administration"}}, {"model": "authentication.faculty", "pk": "Farmaceutische_Wetenschappen", "fields": {"name": "faculties.pharmaceutical_sciences"}}, {"model": "authentication.faculty", "pk": "Geneeskunde_Gezondheidswetenschappen", "fields": {"name": "faculties.medicine_health_sciences"}}, {"model": "authentication.faculty", "pk": "Ingenieurswetenschappen_Architectuur", "fields": {"name": "faculties.engineering_architecture"}}, {"model": "authentication.faculty", "pk": "Letteren_Wijsbegeerte", "fields": {"name": "faculties.arts_philosophy"}}, {"model": "authentication.faculty", "pk": "Politieke_Sociale_Wetenschappen", "fields": {"name": "faculties.political_social_sciences"}}, {"model": "authentication.faculty", "pk": "Psychologie_PedagogischeWetenschappen", "fields": {"name": "faculties.psychology_educational_sciences"}}, {"model": "authentication.faculty", "pk": "Recht_Criminologie", "fields": {"name": "faculties.law_criminology"}}, {"model": "authentication.faculty", "pk": "Wetenschappen", "fields": {"name": "faculties.sciences"}}]
[{"model": "authentication.user", "pk": "0", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "JamesWilson", "is_staff": false, "email": "[email protected]", "first_name": "James", "last_name": "Wilson", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "1", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "ChristinaHal", "is_staff": false, "email": "[email protected]", "first_name": "Christina", "last_name": "Hall", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "10", "fields": {"last_login": "2024-04-18T21:09:05.409Z", "username": "ChristopherW", "is_staff": false, "email": "[email protected]", "first_name": "Christopher", "last_name": "Walls", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.409Z", "faculties": []}}, {"model": "authentication.user", "pk": "11", "fields": {"last_login": "2024-04-18T21:09:05.409Z", "username": "FeliciaScott", "is_staff": false, "email": "[email protected]", "first_name": "Felicia", "last_name": "Scott", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.409Z", "faculties": []}}, {"model": "authentication.user", "pk": "12", "fields": {"last_login": "2024-04-18T21:09:05.409Z", "username": "MadelineWarr", "is_staff": false, "email": "[email protected]", "first_name": "Madeline", "last_name": "Warren", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.409Z", "faculties": []}}, {"model": "authentication.user", "pk": "13", "fields": {"last_login": "2024-04-18T21:09:05.409Z", "username": "KennethSchro", "is_staff": false, "email": "[email protected]", "first_name": "Kenneth", "last_name": "Schroeder", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.409Z", "faculties": []}}, {"model": "authentication.user", "pk": "14", "fields": {"last_login": "2024-04-18T21:09:05.410Z", "username": "ChristopherC", "is_staff": false, "email": "[email protected]", "first_name": "Christopher", "last_name": "Curry", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.410Z", "faculties": []}}, {"model": "authentication.user", "pk": "15", "fields": {"last_login": "2024-04-18T21:09:05.506Z", "username": "MelindaLane", "is_staff": false, "email": "[email protected]", "first_name": "Melinda", "last_name": "Lane", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.506Z", "faculties": []}}, {"model": "authentication.user", "pk": "16", "fields": {"last_login": "2024-04-18T21:09:05.506Z", "username": "AmandaLopez", "is_staff": false, "email": "[email protected]", "first_name": "Amanda", "last_name": "Lopez", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.506Z", "faculties": []}}, {"model": "authentication.user", "pk": "17", "fields": {"last_login": "2024-04-18T21:09:05.506Z", "username": "ChristianRod", "is_staff": false, "email": "[email protected]", "first_name": "Christian", "last_name": "Rodriguez", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.506Z", "faculties": []}}, {"model": "authentication.user", "pk": "2", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "CharlesDunn", "is_staff": false, "email": "[email protected]", "first_name": "Charles", "last_name": "Dunn", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "3", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "BrandonBenne", "is_staff": false, "email": "[email protected]", "first_name": "Brandon", "last_name": "Bennett", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "4", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "BethSmith", "is_staff": false, "email": "[email protected]", "first_name": "Beth", "last_name": "Smith", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "5", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "LauraWoodard", "is_staff": false, "email": "[email protected]", "first_name": "Laura", "last_name": "Woodard", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "6", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "RandyCastro", "is_staff": false, "email": "[email protected]", "first_name": "Randy", "last_name": "Castro", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "7", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "TinaCraig", "is_staff": false, "email": "[email protected]", "first_name": "Tina", "last_name": "Craig", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "8", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "SaraStanton", "is_staff": false, "email": "[email protected]", "first_name": "Sara", "last_name": "Stanton", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.user", "pk": "9", "fields": {"last_login": "2024-04-18T21:09:05.216Z", "username": "LisaSingh", "is_staff": false, "email": "[email protected]", "first_name": "Lisa", "last_name": "Singh", "last_enrolled": 2024, "create_time": "2024-04-18T21:09:05.216Z", "faculties": []}}, {"model": "authentication.faculty", "pk": "Bio-ingenieurswetenschappen", "fields": {"name": "faculties.bioscience_engineering"}}, {"model": "authentication.faculty", "pk": "Diergeneeskunde", "fields": {"name": "faculties.veterinary_medicine"}}, {"model": "authentication.faculty", "pk": "Economie_Bedrijfskunde", "fields": {"name": "faculties.economics_business_administration"}}, {"model": "authentication.faculty", "pk": "Farmaceutische_Wetenschappen", "fields": {"name": "faculties.pharmaceutical_sciences"}}, {"model": "authentication.faculty", "pk": "Geneeskunde_Gezondheidswetenschappen", "fields": {"name": "faculties.medicine_health_sciences"}}, {"model": "authentication.faculty", "pk": "Ingenieurswetenschappen_Architectuur", "fields": {"name": "faculties.engineering_architecture"}}, {"model": "authentication.faculty", "pk": "Letteren_Wijsbegeerte", "fields": {"name": "faculties.arts_philosophy"}}, {"model": "authentication.faculty", "pk": "Politieke_Sociale_Wetenschappen", "fields": {"name": "faculties.political_social_sciences"}}, {"model": "authentication.faculty", "pk": "Psychologie_PedagogischeWetenschappen", "fields": {"name": "faculties.psychology_educational_sciences"}}, {"model": "authentication.faculty", "pk": "Recht_Criminologie", "fields": {"name": "faculties.law_criminology"}}, {"model": "authentication.faculty", "pk": "Wetenschappen", "fields": {"name": "faculties.sciences"}}]
33 changes: 2 additions & 31 deletions development.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,42 +74,13 @@ fi

# Seed database if data size is provided
if [ "$data" != "" ]; then
# Cleanup function
cleanup() {
echo "Ctrl+C detected. Cleaning up..."
deactivate
rm -rf .venv_dev
exit 1
}

# Call cleanup function on SIGINT
trap cleanup SIGINT

echo "--------------------------"
echo "Filling the database."
echo "This can take some time..."
echo "--------------------------"

cd backend

rm -f db.sqlite3 > /dev/null 2>&1

echo "Setting up workspace..."
python3 -m venv .venv_dev > /dev/null
source .venv_dev/bin/activate
pip install poetry > /dev/null
poetry install > /dev/null

echo "Migrating database..."
python manage.py migrate > /dev/null

echo "Filling $data database..."
python manage.py loaddata */fixtures/$data/* > /dev/null 2>&1

echo "Resetting workspace..."
deactivate
rm -rf .venv_dev
cd ..
rm -f backend/db.sqlite3 > /dev/null 2>&1
docker run -v backend:/code ugent-7_backend sh -c "python manage.py migrate; python manage.py loaddata */fixtures/small/*" > /dev/null
fi

# Build Docker images
Expand Down
Loading