diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 50db754..0000000 --- a/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -*.pyc -*.log -.idea -meeting* -*.vscode -.ipynb_checkpoints -*.xlsx -checking_parsing.ipynb diff --git a/Server/__init__.py b/Server/Server/__init__.py similarity index 92% rename from Server/__init__.py rename to Server/Server/__init__.py index b0a8c0f..264137f 100644 --- a/Server/__init__.py +++ b/Server/Server/__init__.py @@ -10,7 +10,7 @@ db = SQLAlchemy(app) bcrypt = Bcrypt(app) login_manager = LoginManager(app) -login_manager.login_view = 'login' +login_manager.login_view = 'users.login' login_manager.login_message_category = 'info' diff --git a/Server/Server/classrooms/__init__.py b/Server/Server/classrooms/__init__.py new file mode 100644 index 0000000..daaea2c --- /dev/null +++ b/Server/Server/classrooms/__init__.py @@ -0,0 +1,5 @@ +from Server.config import ParseConfig +from Server.classrooms.loading_classroom_file import ParseClassFile + + +parser = ParseClassFile.from_object(ParseConfig) \ No newline at end of file diff --git a/Server/Server/classrooms/attendance_check.py b/Server/Server/classrooms/attendance_check.py new file mode 100644 index 0000000..a1bc161 --- /dev/null +++ b/Server/Server/classrooms/attendance_check.py @@ -0,0 +1,109 @@ +import numpy as np +import pandas as pd + + +class Attendance: + """ + receives student class df, the zoom chat and other configuration. returns: + 1. table of attendant students from the student class df + 2. list of table of relevant data from zoom users that didn't add a student return + """ + def __init__(self, chat_df, students_df, filter_modes, time_delta, start_sentence, not_included_zoom_users): + """ + - convert the chat text file to a data frame and arrange columns. + - creates df for each session according the appearance of the start sentence and time delta + :param chat_df: zoom chat in df (df) + :param students_df: student class raw data (df) + :param filter_modes: filters the user picked for parsing the text file (list of str) + :param time_delta: max time from start sentence to the last message to parse in each session in minutes (int) + :param start_sentence: start sentence that initiate sessions for parse (str) + :return: data frame with the data from the chat + """ + + self.first_message_time = chat_df["time"].sort_values().iloc[0] + start_indices = chat_df.index[chat_df['message'].apply(lambda string: start_sentence.lower() in string.lower())] #TODO: slice by time or by next message + df_students_for_report = students_df.set_index("id").astype(str).reset_index() # set all columns to str except the id + + self._sessions = [] + for start_index in start_indices: + df_session = Attendance.get_df_of_time_segment(chat_df, start_index, time_delta) + self._sessions.append(Session(df_students_for_report, df_session, filter_modes, not_included_zoom_users)) + + @staticmethod + def get_df_of_time_segment(df, start_index, time_delta): + + time_delta = np.timedelta64(time_delta, 'm') + time_segment_start = df.loc[start_index, "time"] + time_filt = (df["time"] >= time_segment_start) & \ + (df["time"] <= time_segment_start + time_delta) + + return df.loc[time_filt] + + @property + def report_sessions(self): + return self._sessions + + +class Session: + + def __init__(self, df_students, df_session_chat, filter_modes, not_included_zoom_users): + + self._first_message_time = df_session_chat["time"].sort_values().iloc[0] + self._relevant_chat = self.get_participants_in_session(df_students,filter_modes, df_session_chat, not_included_zoom_users) + + + @ staticmethod + def get_participants_in_session(df_students, filter_modes, df_chat, not_included_zoom_users): + """ + finds students that attendant to the session. runs over each mode which represent different way to declare that + the student attendant (for example: phone number, ID). merges this data to the csv table with the zoom name that + added it + :param df_chat: that table of the chat for the specific session + :return: df of the attendance in the session + """ + final_df = None + for mode in filter_modes: + merged_df = pd.merge(df_students, df_chat.reset_index(), left_on=mode, right_on="message", how="left") + final_df = pd.concat([merged_df, final_df]) + + final_df.sort_values(by="time", inplace=True) + df_participated = final_df.groupby("zoom_name").first().reset_index() + df_participated["index"] = df_participated["index"].astype(int) + df_participated = df_participated.loc[:, ["id", "zoom_name", "time", "message", "index"]].set_index("index") + + filt = df_chat['zoom_name'].str.contains('|'.join(not_included_zoom_users)) + df_relevant_chat = pd.merge(df_chat[~filt], df_participated, how="left") + + df_relevant_chat["relevant"] = df_relevant_chat["id"].apply(lambda x: 1 if x == x else 0) + df_relevant_chat["id"] = df_relevant_chat["id"].apply(lambda x: int(x) if x == x else -1) + return df_relevant_chat + + + def zoom_names_table(self, session_id): + zoom_df = self._relevant_chat.loc[:, ["zoom_name", "id"]].rename(columns={"zoom_name": "name", "id": "student_id"}) + zoom_df['session_id'] = pd.Series([session_id] * zoom_df.shape[0]) + return zoom_df.sort_values(by="student_id", ascending=False).groupby("name").first().reset_index() + + def chat_table(self, zoom_df): + relevant_chat = self._relevant_chat.drop(columns=["id"]) + chat_session_table = pd.merge(relevant_chat, zoom_df, left_on="zoom_name", right_on="name") + return chat_session_table.drop(columns=["zoom_name", "name", "session_id", "student_id"]).rename(columns={"id": "zoom_names_id"}) + + + +if __name__ == '__main__': + from utils import create_chat_df, create_students_df + + chat_file_path = r"C:\Users\Inbar Shirizly\Documents\python\useful\ITC_programs\zoom_attendance_check\chat files\meeting_example_full_name.txt" + excel_file_path = r"C:\Users\Inbar Shirizly\Documents\python\useful\ITC_programs\zoom_attendance_check\student_csv_examples\example_data_already_prepared.xlsx" + + + with open(chat_file_path, "r", encoding="utf-8") as f: + chat_df = create_chat_df(f.readlines()) + df_students = create_students_df(file_name=excel_file_path.split("\\")[-1], file_data=excel_file_path) + + my_class = Attendance(chat_df, df_students, ['name', "id_number", "phone"], 5, "Attendance check", ["ITC", "Tech", "Challenge"]) + + df_part_session = my_class._sessions[0] + df_part_session.zoom_names_table(2) + diff --git a/Server/Server/classrooms/forms.py b/Server/Server/classrooms/forms.py new file mode 100644 index 0000000..e644b50 --- /dev/null +++ b/Server/Server/classrooms/forms.py @@ -0,0 +1,17 @@ +from flask_wtf import FlaskForm +from flask_wtf.file import FileField, FileAllowed, FileRequired +from wtforms.validators import DataRequired +from wtforms import StringField, SubmitField, IntegerField + + +class CreateClassForm(FlaskForm): + name = StringField('Name of the class', validators=[DataRequired()]) + students_file = FileField('Csv/Excel file of students', validators=[FileAllowed(['xlsx', 'csv', 'xls']), FileRequired()]) + submit = SubmitField('Create') + + +class CreateReportForm(FlaskForm): + start_sentence = StringField('Sentence to start the zoom check', validators=[DataRequired()]) + chat_file = FileField('Zoom chat file', validators=[FileRequired(), FileAllowed(['txt'])]) + time = IntegerField('Number of minutes', validators=[DataRequired()]) + submit = SubmitField('Create Report') diff --git a/Server/Server/classrooms/loading_classroom_file.py b/Server/Server/classrooms/loading_classroom_file.py new file mode 100644 index 0000000..a02e126 --- /dev/null +++ b/Server/Server/classrooms/loading_classroom_file.py @@ -0,0 +1,64 @@ +import pandas as pd +import numpy as np +import re + + +class ParseClassFile: + + def __init__(self, file_cols_dict, mashov_cols, gender_dict): + self._file_cols_dict = file_cols_dict + self._mashov_cols = mashov_cols + self._gender_dict = gender_dict + + @classmethod + def from_object(cls, config): + return cls( + config.FILE_COLS_DICT, + config.MASHOV_COLS, + config.GENDER_DICT + ) + + + def parse_df(self, df_students): + relevant_cols = [col for col in df_students.columns if not col.startswith("Unnamed")] + current_excel_dict = {} + + for col in relevant_cols: + for key, col_options in self._file_cols_dict.items(): + if col in col_options: + current_excel_dict[key] = df_students[col] + + if len(current_excel_dict) == 0 and len(relevant_cols) <= 1: + print("Mashov file") + header_index = df_students.notnull().sum(axis=1).argmax() + df_students = pd.DataFrame(df_students.values[header_index + 1:-2], columns=df_students.iloc[header_index]) + df_students.dropna(axis=0, how='all', inplace=True) + df_students.dropna(axis=1, how='all', inplace=True) + df_students.rename(columns={np.nan: 'name', "פרטי תלמיד": 'name', "כיתה": "org_class"}, inplace=True) + df_students = df_students.loc[:, self._mashov_cols] + + mashov_name_pattern = re.compile(r"([\u0590-\u05fe ]+)([(\u0590-\u05fe)]+)") + df_name_gender = df_students['name'].str.extract(mashov_name_pattern, expand=False) + df_students['gender'] = df_name_gender[1].str.extract("\(([\u0590-\u05fe ])\)") + df_students['gender'] = df_students['gender'].apply(self.gender_assign, gender_dict=self._gender_dict) + df_students['name'] = df_name_gender[0] + + else: + df_students = pd.DataFrame(current_excel_dict) + + for col in self._file_cols_dict.keys(): + try: + df_students[col] = df_students[col] + except KeyError: + df_students[col] = pd.Series([np.nan] * df_students.shape[0]) + + final_df = df_students[list(self._file_cols_dict.keys())] + + return final_df.reset_index().drop(columns="index") + + @staticmethod + def gender_assign(string, gender_dict): + for key, vals in gender_dict.items(): + if string in vals: + return key + return "" \ No newline at end of file diff --git a/Server/Server/classrooms/routes.py b/Server/Server/classrooms/routes.py new file mode 100644 index 0000000..4112dfa --- /dev/null +++ b/Server/Server/classrooms/routes.py @@ -0,0 +1,74 @@ +from flask import render_template, redirect, url_for, flash, Blueprint, request, session +from Server import db +from Server.classrooms.forms import CreateClassForm, CreateReportForm +from Server.models import ClassroomModel, StudentModel, ReportModel, SessionModel, ZoomNamesModel, ChatModel +from flask_login import current_user, login_required +import pandas as pd +from Server.classrooms.attendance_check import Attendance +from Server.classrooms.utils import create_chat_df, create_students_df +from Server.classrooms import parser +from datetime import datetime + +classrooms = Blueprint('classrooms', __name__) + + +@classrooms.route('/', methods=['GET', 'POST']) +@classrooms.route('/home', methods=['GET', 'POST']) +@login_required +def home(): + form = CreateClassForm() + if form.validate_on_submit(): + students_df = create_students_df(form.students_file.data.filename, form.students_file.data) + students = parser.parse_df(students_df) + new_class = ClassroomModel(name=form.name.data, teacher_model=current_user) + db.session.add(new_class) + db.session.commit() + students['class_id'] = pd.Series([new_class.id] * students.shape[0]) + students.to_sql('student_model', con=db.engine, if_exists="append", index=False) + return redirect(url_for('classrooms.classroom', class_id=new_class.id)) + return render_template('home.html', form=form) + + +@classrooms.route('/classroom/', methods=['GET', 'POST']) +@login_required +def classroom(class_id): + current_class = ClassroomModel.query.filter_by(id=class_id, teacher_model=current_user).first() # Making sure the class belongs to the current user + if current_class is None: + flash('Invalid class!', 'danger') + return redirect(url_for('classrooms.home')) + + form = CreateReportForm() + if form.validate_on_submit(): # If form was submitted, creating report for the class + description = "this is the best class" # TODO : add from the form file + report_date = datetime.now().date() # TODO : add from the form file + students_df = pd.read_sql(StudentModel.query.filter_by(class_id=class_id).statement, con=db.engine) + chat_file = form.chat_file.data.stream.read().decode("utf-8").split("\n") + chat_df = create_chat_df(chat_file) + + report_object = Attendance(chat_df, students_df, ['name', "id_number", "phone"], form.time.data, form.start_sentence.data, ["ITC", "Tech", "Challenge"]) + + new_report = ReportModel(description=description, start_time=report_object.first_message_time, report_date=report_date, class_id=class_id) + db.session.add(new_report) + db.session.commit() + + # insert relevant data for each session to the database + for session_object in report_object.report_sessions: + session_table = SessionModel(start_time=session_object._first_message_time, report_id=new_report.id) + db.session.add(session_table) + db.session.commit() + + zoom_names_df = session_object.zoom_names_table(session_table.id) + zoom_names_df.to_sql('zoom_names_model', con=db.engine, if_exists="append", index=False) + + zoom_names_df = pd.read_sql(ZoomNamesModel.query.filter_by(session_id=session_table.id).statement, con=db.engine) + session_chat_df = session_object.chat_table(zoom_names_df) + session_chat_df.to_sql('chat_model', con=db.engine, if_exists="append", index=False) + + + + return render_template("report.html") + + + return render_template('classroom.html', current_class=current_class, form=form) + + diff --git a/Server/Server/classrooms/utils.py b/Server/Server/classrooms/utils.py new file mode 100644 index 0000000..b5b82db --- /dev/null +++ b/Server/Server/classrooms/utils.py @@ -0,0 +1,26 @@ +import re +from datetime import datetime +import pandas as pd + +def create_chat_df(chat_file): + + regex_pattern = re.compile(r"(^\d{2}.\d{2}.\d{2})\s+From\s\s([\s\S]+)\s:\s([\s\S]+)") + chat_content = [re.search(regex_pattern, line).groups() for line in chat_file if re.match(regex_pattern, line)] + + chat_df = pd.DataFrame(chat_content, columns=["time", "zoom_name", "message"]) + chat_df['message'] = chat_df['message'].str[:-1].astype(str) + chat_df["time"] = chat_df["time"].apply(lambda string: datetime.strptime(string, "%H:%M:%S")) + + return chat_df + + +def create_students_df(file_name, file_data): + + if file_name.endswith(".csv"): + df_students = pd.read_csv(file_data) + elif file_name.endswith(".xlsx"): + df_students = pd.read_excel(file_data) + else: + df_students = pd.read_html(file_data, header=1)[0] + + return df_students \ No newline at end of file diff --git a/Server/Server/config.py b/Server/Server/config.py new file mode 100644 index 0000000..f2768b6 --- /dev/null +++ b/Server/Server/config.py @@ -0,0 +1,16 @@ +class FlaskConfig: + SECRET_KEY = 'TEMP_SECRET_KEY' + SQLALCHEMY_DATABASE_URI = 'sqlite:///site.db' + + +class ParseConfig: + + FILE_COLS_DICT = { + "name": ["שם התלמיד", "תלמידים", "שמות", "שם", "סטודנט"], + "id_number": ["תעודת זהות", "ת.ז.", "ת.ז", "תז"], + "phone": ["טלפון", "מספר טלפון", "מס טלפון"], + "gender": ["מין"], + "org_class": ["כיתה"] + } + MASHOV_COLS = ["name", "org_class"] + GENDER_DICT = {1: ["זכר", "ז", "(ז)"], 0: ["נקבה", "נ", "(נ)"]} \ No newline at end of file diff --git a/Server/Server/models.py b/Server/Server/models.py new file mode 100644 index 0000000..d47a8d3 --- /dev/null +++ b/Server/Server/models.py @@ -0,0 +1,76 @@ +from Server import db, login_manager, app +from flask_login import UserMixin + + +@login_manager.user_loader +def load_user(user_id): + return TeacherModel.query.get(int(user_id)) + + +class TeacherModel(db.Model, UserMixin): + id = db.Column(db.Integer, primary_key=True) + username = db.Column(db.String(40), unique=True, nullable=False) + email = db.Column(db.String(120), unique=True, nullable=False) + password = db.Column(db.String(60), nullable=False) # 60 chars because of the hashing algo + classrooms = db.relationship('ClassroomModel', backref='teacher_model', lazy=True) + + def __repr__(self): + return f'Teacher({self.username}, {self.email}, {self.password})' + + +class ClassroomModel(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(70), unique=False, nullable=False) + teacher_id = db.Column(db.Integer, db.ForeignKey('teacher_model.id'), nullable=False) + students = db.relationship('StudentModel', backref='classroom_model', lazy=True) + reports = db.relationship('ReportModel', backref='classroom_model', lazy=True) + + +class StudentModel(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(70), unique=False, nullable=False) + id_number = db.Column(db.String(10), unique=False, nullable=True) + org_class = db.Column(db.String(20), unique=False, nullable=True) + gender = db.Column(db.Boolean, unique=False, nullable=True) # True means male + phone = db.Column(db.Integer, unique=False, nullable=True) + + class_id = db.Column(db.Integer, db.ForeignKey('classroom_model.id'), nullable=False) + zoom_names = db.relationship('ZoomNamesModel', backref='student_model', lazy=True) + + +class ReportModel(db.Model): + id = db.Column(db.Integer, primary_key=True) + description = db.Column(db.Text(), unique=False, nullable=True) + start_time = db.Column(db.DateTime(), unique=False, nullable=False) # first timestamp of chat file + report_date = db.Column(db.Date(), unique=False, nullable=True) # date of the report - given by the user + + class_id = db.Column(db.Integer, db.ForeignKey('classroom_model.id'), nullable=False) + sessions = db.relationship('SessionModel', backref='report_model', lazy=True) + + +class SessionModel(db.Model): + id = db.Column(db.Integer, primary_key=True) + start_time = db.Column(db.DateTime(), unique=False, nullable=False) # first timestamp of chat session + + report_id = db.Column(db.Integer, db.ForeignKey('report_model.id'), nullable=False) + zoom_names = db.relationship('ZoomNamesModel', backref='session_model', lazy=True) + + +class ZoomNamesModel(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(50), unique=False, nullable=True) + session_id = db.Column(db.Integer, db.ForeignKey('session_model.id'), nullable=False) + student_id = db.Column(db.Integer, db.ForeignKey('student_model.id'), nullable=True) # if Null - means the student wasn't present in the session + + chat = db.relationship('ChatModel', backref='zoom_names_model', lazy=True) + + +class ChatModel(db.Model): + id = db.Column(db.Integer, primary_key=True) + time = db.Column(db.DateTime, unique=False, nullable=False) # time the message written + message = db.Column(db.Text, unique=False, nullable=True) # message zoom user wrote + relevant = db.Column(db.Boolean, unique=False, nullable=False) # True is message is part of the report + zoom_names_id = db.Column(db.Integer, db.ForeignKey('zoom_names_model.id'), nullable=False) + + + diff --git a/Server/Server/templates/_render_form_field.html b/Server/Server/templates/_render_form_field.html new file mode 100644 index 0000000..e4947f8 --- /dev/null +++ b/Server/Server/templates/_render_form_field.html @@ -0,0 +1,31 @@ +{% macro render_field(field) %} + +
+ {{ field.label }} + {% if field.errors %} + {{ field(class="form-control is-invalid")}} +
+ {% for error in field.errors %} + {{ error }} + {% endfor %} +
+ {% else %} + {{ field(class="form-control")}} + {% endif %} +
+ +{% endmacro %} + +{% macro render_file_field(field) %} + +
+ {{ field.label }} + {{ field(class="form-control-file") }} + {% if field.errors %} + {% for error in field.errors %} + {{ error }} + {% endfor %} +{% endif %} +
+ +{% endmacro %} \ No newline at end of file diff --git a/Server/Server/templates/classroom.html b/Server/Server/templates/classroom.html new file mode 100644 index 0000000..b6fa625 --- /dev/null +++ b/Server/Server/templates/classroom.html @@ -0,0 +1,74 @@ +{% extends "layout.html" %} +{% from "_render_form_field.html" import render_field, render_file_field %} + +{% block content %} +
+
+

Class: {{ current_class.name }}

+
+
+ +
+
+ +
+
+
+
+
+
+ {{ form.hidden_tag() }} + {{ render_field(form.start_sentence) }} + {{ render_file_field(form.chat_file)}} + {{ render_field(form.time) }} +
+ {{ form.submit(class="btn btn-outline-info") }} +
+
+
+
+
+
+
+ + + + + + + + + + + + {% for student in current_class.students %} + + + + {% if student.id_number == None %} + + {% else %} + + {% endif %} + {% if student.phone == None %} + + {% else %} + + {% endif %} + {% if student.gender == None %} + + {% else %} + {% if student.gender %} + + {% else %} + + {% endif %} + {% endif %} + + {% endfor %} + +
Student NameClassId NumberPhonegender
{{ student.name }}{{ student.org_class }}{{ student.id_number }}{{ student.phone }}MaleFemale
+
+
+ +{% endblock content %} \ No newline at end of file diff --git a/Server/Server/templates/home.html b/Server/Server/templates/home.html new file mode 100644 index 0000000..081567c --- /dev/null +++ b/Server/Server/templates/home.html @@ -0,0 +1,46 @@ +{% extends "layout.html" %} +{% from "_render_form_field.html" import render_field, render_file_field %} + +{% block content %} +
+
+
+

My classes

+
+
+ +
+
+
+
+
+
+ {{ form.hidden_tag() }} +
+
+ {{ render_field(form.name) }} +
+
+ {{ render_file_field(form.students_file) }} +
+
+
+ {{ form.submit(class="btn btn-outline-info") }} +
+
+
+
+
+
+
+
+ {% for classroom in current_user.classes %} + + {{ classroom.name }} + + {% endfor %} +
+
+
+
+{% endblock content %} \ No newline at end of file diff --git a/Server/templates/layout.html b/Server/Server/templates/layout.html similarity index 100% rename from Server/templates/layout.html rename to Server/Server/templates/layout.html diff --git a/Server/Server/templates/login.html b/Server/Server/templates/login.html new file mode 100644 index 0000000..97196f5 --- /dev/null +++ b/Server/Server/templates/login.html @@ -0,0 +1,19 @@ +{% extends "layout.html" %} +{% from "_render_form_field.html" import render_field %} + +{% block content %} + +
+ {{ form.hidden_tag() }} + {{ render_field(form.auth) }} + {{ render_field(form.password)}} +
+ {{ form.remember(class="form-check-input") }} + {{ form.remember.label() }} +
+
+ {{ form.submit(class="btn btn-outline-info")}} +
+
+ +{% endblock content %} \ No newline at end of file diff --git a/Server/Server/templates/register.html b/Server/Server/templates/register.html new file mode 100644 index 0000000..0f4cb2b --- /dev/null +++ b/Server/Server/templates/register.html @@ -0,0 +1,15 @@ +{% extends "layout.html" %} +{% from "_render_form_field.html" import render_field %} + +{% block content %} +
+ {{ form.hidden_tag() }} + {{ render_field(form.username) }} + {{ render_field(form.email) }} + {{ render_field(form.password) }} + {{ render_field(form.confirm_password) }} +
+ {{ form.submit(class="btn btn-outline-info")}} +
+
+{% endblock content %} \ No newline at end of file diff --git a/Server/Server/templates/report.html b/Server/Server/templates/report.html new file mode 100644 index 0000000..0f25d3b --- /dev/null +++ b/Server/Server/templates/report.html @@ -0,0 +1,4 @@ +{% extends "layout.html" %} +{% block content %} +

Reports file

+{% endblock content %} \ No newline at end of file diff --git a/Server/classrooms/__init__.py b/Server/Server/users/__init__.py similarity index 100% rename from Server/classrooms/__init__.py rename to Server/Server/users/__init__.py diff --git a/Server/users/forms.py b/Server/Server/users/forms.py similarity index 86% rename from Server/users/forms.py rename to Server/Server/users/forms.py index 9d157c7..e2dd8aa 100644 --- a/Server/users/forms.py +++ b/Server/Server/users/forms.py @@ -1,7 +1,7 @@ from flask_wtf import FlaskForm from wtforms import StringField, PasswordField, SubmitField, BooleanField from wtforms.validators import DataRequired, EqualTo, Length, Email, ValidationError -from Server.models import User +from Server.models import TeacherModel class RegistrationForm(FlaskForm): @@ -12,14 +12,14 @@ class RegistrationForm(FlaskForm): submit = SubmitField('Sign Up') def validate_username(self, username): - user = User.query.filter_by(username=username.data).first() + user = TeacherModel.query.filter_by(username=username.data).first() if user: print(user) raise ValidationError('Username already taken.') # Add special signs validations def validate_email(self, email): - user = User.query.filter_by(email=email.data).first() + user = TeacherModel.query.filter_by(email=email.data).first() if user: raise ValidationError('Email already taken.') diff --git a/Server/users/routes.py b/Server/Server/users/routes.py similarity index 85% rename from Server/users/routes.py rename to Server/Server/users/routes.py index 5c95cd0..7418fdf 100644 --- a/Server/users/routes.py +++ b/Server/Server/users/routes.py @@ -1,7 +1,7 @@ from flask import render_template, redirect, url_for, flash, Blueprint from Server import bcrypt, db from Server.users.forms import LoginForm, RegistrationForm -from Server.models import User +from Server.models import TeacherModel from flask_login import login_user, current_user, logout_user, login_required users = Blueprint('users', __name__) @@ -13,8 +13,8 @@ def login(): form = LoginForm() if form.validate_on_submit(): - user = User.query.filter_by(email=form.auth.data).first() or \ - User.query.filter_by(username=form.auth.data).first() # User can be validated with both username and email + user = TeacherModel.query.filter_by(email=form.auth.data).first() or \ + TeacherModel.query.filter_by(username=form.auth.data).first() # User can be validated with both username and email if user and bcrypt.check_password_hash(user.password, form.password.data): login_user(user, remember=form.remember.data) return redirect(url_for('classrooms.home')) @@ -32,7 +32,7 @@ def register(): form = RegistrationForm() if form.validate_on_submit(): hashed_password = bcrypt.generate_password_hash((form.password.data)).decode('utf-8') - user = User( + user = TeacherModel( username=form.username.data, password=hashed_password, email=form.email.data diff --git a/Server/classrooms/forms.py b/Server/classrooms/forms.py deleted file mode 100644 index 4e7ffca..0000000 --- a/Server/classrooms/forms.py +++ /dev/null @@ -1,10 +0,0 @@ -from flask_wtf import FlaskForm -from flask_wtf.file import FileField, FileAllowed, FileRequired -from wtforms.validators import DataRequired -from wtforms import StringField, SubmitField - - -class CreateClassForm(FlaskForm): - name = StringField('Name of the class', validators=[DataRequired()]) - students_file = FileField('Csv/Excel file of students', validators=[FileAllowed(['xlsx', 'csv']), FileRequired()]) - submit = SubmitField('Create') diff --git a/Server/classrooms/routes.py b/Server/classrooms/routes.py deleted file mode 100644 index 9eb7053..0000000 --- a/Server/classrooms/routes.py +++ /dev/null @@ -1,38 +0,0 @@ -from flask import render_template, redirect, url_for, flash, Blueprint -from Server import db -from Server.classrooms.forms import CreateClassForm -from Server.models import Classroom, Student -from flask_login import current_user, login_required -from Server.classrooms.utils import save_file - -classrooms = Blueprint('classrooms', __name__) - - -@classrooms.route('/', methods=['GET', 'POST']) -@classrooms.route('/home', methods=['GET', 'POST']) -@login_required -def home(): - form = CreateClassForm() - if form.validate_on_submit(): - file_name = save_file(form.students_file.data) - new_class = Classroom(name=form.name.data, teacher=current_user) - db.session.add(new_class) - db.session.commit() - return render_template('home.html', form=form) - - -@classrooms.route('/classroom/') -@login_required -def classroom(class_id): - current_class = Classroom.query.filter_by(id=class_id, teacher=current_user).first() # Making sure the class belongs to the current user - if current_class is None: - flash('Invalid class!', 'danger') - return redirect(url_for('classrooms.home')) - current_class.students = [ # Temporary hardocded data - Student(school_class='יב 1', name='איתי'), - Student(school_class='יב 2', name='ענבר', id_number=212525489), - Student(school_class='יב 3', name='לירן', id_number=3), - Student(school_class='Liran', name='hello') - ] - return render_template('classroom.html', current_class=current_class) - diff --git a/Server/classrooms/utils.py b/Server/classrooms/utils.py deleted file mode 100644 index 6801961..0000000 --- a/Server/classrooms/utils.py +++ /dev/null @@ -1,9 +0,0 @@ -import secrets -import os - -def save_file(form_file): #TODO: create better algorithem to save the files - _, f_ext = os.path.splitext(form_file.filename) - file_name = secrets.token_hex(8) + f_ext - file_path = os.path.join(app.root_path, 'static', 'students', file_name) - form_file.save(file_path) - return file_name diff --git a/Server/config.py b/Server/config.py deleted file mode 100644 index 30a47cf..0000000 --- a/Server/config.py +++ /dev/null @@ -1,3 +0,0 @@ -class FlaskConfig: - SECRET_KEY = 'TEMP_SECRET_KEY' - SQLALCHEMY_DATABASE_URI = 'sqlite:///site.db' \ No newline at end of file diff --git a/Server/models.py b/Server/models.py deleted file mode 100644 index e06ad49..0000000 --- a/Server/models.py +++ /dev/null @@ -1,36 +0,0 @@ -from Server import db, login_manager, app -from flask_login import UserMixin - - -@login_manager.user_loader -def load_user(user_id): - return User.query.get(int(user_id)) - - -class User(db.Model, UserMixin): - id = db.Column(db.Integer, primary_key=True) - username = db.Column(db.String(40), unique=True, nullable=False) - email = db.Column(db.String(120), unique=True, nullable=False) - password = db.Column(db.String(60), nullable=False) # 60 chars because of the hashing algo - classes = db.relationship('Classroom', backref='teacher', lazy=True) - - def __repr__(self): - return f'User({self.username}, {self.email}, {self.password})' - - -class Classroom(db.Model): - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String(70), unique=False, nullable=False) - teacher_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=False) - # TODO: also store the reports in the db - students = db.relationship('Student', backref='classroom', lazy=True) - - -class Student(db.Model): - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String(70), unique=False, nullable=False) - id_number = db.Column(db.String(9), unique=False) - school_class = db.Column(db.String(20), unique=False) - gender = db.Column(db.Boolean, unique=False) # True means male - class_id = db.Column(db.Integer, db.ForeignKey('classroom.id'), nullable=False) - diff --git a/Server/requirements.txt b/Server/requirements.txt new file mode 100644 index 0000000..ebc4e21 --- /dev/null +++ b/Server/requirements.txt @@ -0,0 +1,30 @@ +bcrypt==3.2.0 +beautifulsoup4==4.9.1 +bs4==0.0.1 +cffi==1.14.3 +click==7.1.2 +dnspython==2.0.0 +email-validator==1.1.1 +Flask==1.1.2 +Flask-Bcrypt==0.7.1 +Flask-Login==0.5.0 +Flask-SQLAlchemy==2.4.4 +Flask-WTF==0.14.3 +html5lib==1.1 +idna==2.10 +itsdangerous==1.1.0 +Jinja2==2.11.2 +lxml==4.5.2 +MarkupSafe==1.1.1 +numpy==1.19.2 +pandas==1.1.2 +pycparser==2.20 +python-dateutil==2.8.1 +pytz==2020.1 +six==1.15.0 +soupsieve==2.0.1 +SQLAlchemy==1.3.19 +webencodings==0.5.1 +Werkzeug==1.0.1 +WTForms==2.3.3 +xlrd==1.2.0 diff --git a/Server/run.py b/Server/run.py new file mode 100644 index 0000000..6c193e7 --- /dev/null +++ b/Server/run.py @@ -0,0 +1,8 @@ +from Server import app, db +import os + +if __name__ == '__main__': + if not os.path.exists("/Server/site.db"): + db.create_all() + db.session.commit() + app.run(debug=True) \ No newline at end of file diff --git a/Server/templates/classroom.html b/Server/templates/classroom.html deleted file mode 100644 index c48a7bc..0000000 --- a/Server/templates/classroom.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-

Class: {{ current_class.name }}

-
-
- -
-
-
-
- - - - - - - - - - {% for student in current_class.students %} - - - - {% if student.id_number == None %} - - {% else %} - - {% endif %} - - {% endfor %} - -
Student NameClassId Number
{{ student.name }}{{ student.school_class }}Don't exist{{ student.id_number }}
-
-
-{% endblock content %} \ No newline at end of file diff --git a/Server/templates/home.html b/Server/templates/home.html deleted file mode 100644 index 8664810..0000000 --- a/Server/templates/home.html +++ /dev/null @@ -1,66 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
-

My classes

-
-
- -
-
-
-
-
-
-
- {{ form.hidden_tag() }} -
-
-
- {{ form.name.label() }} - {% if form.name.errors %} - {{ form.name(class="form-control is-invalid")}} -
- {% for error in form.name.errors %} - {{ error }} - {% endfor %} -
- {% else %} - {{ form.name(class="form-control")}} - {% endif %} -
-
-
-
- {{ form.students_file.label}} - {{ form.students_file(class="form-control-file")}} - {% if form.students_file.errors %} - {% for error in form.students_file.errors %} - {{ error }} - {% endfor %} - {% endif %} -
-
-
-
- {{ form.submit(class="btn btn-outline-info") }} -
-
-
-
-
-
-
-
-
- {% for classroom in current_user.classes %} - - {{ classroom.name }} - - {% endfor %} -
-
-
-
-{% endblock content %} \ No newline at end of file diff --git a/Server/templates/login.html b/Server/templates/login.html deleted file mode 100644 index 8db9854..0000000 --- a/Server/templates/login.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "layout.html" %} -{% block content %} - -
- {{ form.hidden_tag() }} -
- {{ form.auth.label() }} - - {% if form.auth.errors %} - {{ form.auth(class="form-control is-invalid")}} -
- {% for error in form.auth.errors %} - {{ error }} - {% endfor %} -
- {% else %} - {{ form.auth(class="form-control")}} - {% endif %} -
-
- {{ form.password.label() }} - - {% if form.password.errors %} - {{ form.password(class="form-control is-invalid")}} -
- {% for error in form.password.errors %} - {{ error }} - {% endfor %} -
- {% else %} - {{ form.password(class="form-control")}} - {% endif %} -
-
- {{ form.remember(class="form-check-input") }} - {{ form.remember.label() }} -
-
- {{ form.submit(class="btn btn-outline-info")}} -
-
- -{% endblock content %} \ No newline at end of file diff --git a/Server/templates/register.html b/Server/templates/register.html deleted file mode 100644 index c24a05d..0000000 --- a/Server/templates/register.html +++ /dev/null @@ -1,66 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
- {{ form.hidden_tag() }} -
- {{ form.username.label() }} - - {% if form.username.errors %} - {{ form.username(class="form-control is-invalid")}} -
- {% for error in form.username.errors %} - {{ error }} - {% endfor %} -
- {% else %} - {{ form.username(class="form-control")}} - {% endif %} -
-
- {{ form.email.label() }} - - {% if form.email.errors %} - {{ form.email(class="form-control is-invalid")}} -
- {% for error in form.email.errors %} - {{ error }} - {% endfor %} -
- {% else %} - {{ form.email(class="form-control")}} - {% endif %} - -
-
- {{ form.password.label() }} - - {% if form.password.errors %} - {{ form.password(class="form-control is-invalid")}} -
- {% for error in form.password.errors %} - {{ error }} - {% endfor %} -
- {% else %} - {{ form.password(class="form-control")}} - {% endif %} -
-
- {{ form.confirm_password.label() }} - - {% if form.confirm_password.errors %} - {{ form.confirm_password(class="form-control is-invalid")}} -
- {% for error in form.confirm_password.errors %} - {{ error }} - {% endfor %} -
- {% else %} - {{ form.confirm_password(class="form-control")}} - {% endif %} -
-
- {{ form.submit(class="btn btn-outline-info")}} -
-
-{% endblock content %} \ No newline at end of file diff --git a/Server/users/__init__.py b/Server/users/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/conf.py b/conf.py deleted file mode 100644 index be734a5..0000000 --- a/conf.py +++ /dev/null @@ -1,13 +0,0 @@ -import re -import numpy as np - - -EXCEL_COLS = {"Name": "שם התלמיד", - "ID": "תעודת זהות", - "Phone": "טלפון"} - - -# constants -CHAT_PATTERN = re.compile(r"(^\d{2}.\d{2}.\d{2})\s+From\s\s([\s\S]+)\s:\s([\s\S]+)") - - diff --git a/run.py b/run.py deleted file mode 100644 index 5777dc4..0000000 --- a/run.py +++ /dev/null @@ -1,4 +0,0 @@ -from Server import app - -if __name__ == '__main__': - app.run(debug=True) \ No newline at end of file diff --git a/zoom_chat_check_attendence.py b/zoom_chat_check_attendence.py deleted file mode 100644 index 182ca72..0000000 --- a/zoom_chat_check_attendence.py +++ /dev/null @@ -1,144 +0,0 @@ -""" ------------ zoom check attendance ----------- -Program implementations: -a. gets a text file - zoom chat from class -b. find session when an attendance check has been initiated and look for all the participators who wrote - something meaningful in the chat before or after the check. function checks allows user to write a name of the - student list only once. in case user had a mistake, it will check all his messages until it find the relevant one -c. return df contains all the users from the csv/excel file with new columns that have: zoom name/ Nan - where zoom name - means that this zoom user added the relevant name and Nan means missing student -d. retrun df from each session with the messages of zoom users that did not write something related to the actuall name - (allows the teacher to check if they had type error and then add them manually) - -""" - -from datetime import datetime -import numpy as np -import pandas as pd -import re -import conf - -# configuration variables -FILE_NAME = ".\chat files\meeting_example_full_name.txt" -EXCEL_PATH = "דוגמה לרשימת תלמידים.xlsx" -SENTENCE_START = "attendance check" -TIME_DELTA = 1 # time period in minuets -WORDS_FOR_NOT_INCLUDED_PARTICIPATORS = ["ITC", "Tech", "Challenge"] - - -class Attendance: - - def __init__(self, chat_path, student_file_path, filter_modes, time_delta, start_sentence): - """ - - convert the chat text file to a data frame and arrange columns. - - creates df for each session according the appearance of the start sentence and time delta - - create df from the excel or csv file of the studnets - :param chat_path: the path of the relevant chat file (str) - :param student_file_path: path for the relevant csv/excel file (str) - :param filter_modes: filters the user picked for parsing the text file (list of str) - :param time_delta: max time from start sentence to the last message to parse in each session in minutes (int) - :param start_sentence: start sentence that initiate sessions for parse (str) - :return: data frame with the data from the chat - """ - with open(chat_path, "r", encoding="utf-8") as file: - data = [re.search(conf.CHAT_PATTERN, line).groups() for line in file if re.match(conf.CHAT_PATTERN, line)] - df = pd.DataFrame(data, columns=["time", "users", "chat"]) - df['chat'] = df['chat'].str[:-1] - df["time"] = df["time"].apply(lambda string: datetime.strptime(string, "%H:%M:%S")) - - start_indices = df.index[df['chat'].apply(lambda string: start_sentence.lower() in string.lower())] - - self.df_sessions = [self.get_df_of_time_segment(df, start_index, time_delta) for start_index in start_indices] - self.filter_modes = filter_modes - if student_file_path.endswith(".csv"): - self.df_students = pd.read_csv(student_file_path, usecols=conf.EXCEL_COLS.values()).astype("str") - else: - self.df_students = pd.read_excel(student_file_path, usecols=conf.EXCEL_COLS.values()).astype("str") - - @staticmethod - def get_df_of_time_segment(df, start_index, time_delta): - - time_delta = np.timedelta64(time_delta, 'm') - time_segment_start = df.loc[start_index, "time"] - time_filt = (df["time"] >= time_segment_start) & \ - (df["time"] <= time_segment_start + time_delta) - - return df.loc[time_filt] - - - def get_participants(self, df_chat): - """ - finds students that attendant to the session. runs over each mode which represent different way to declare that - the student attendant (for example: phone number, ID). merges this data to the csv table with the zoom name that - added it - :param df_chat: that table of the chat for the specific session - :return: df of the attendance in the session - """ - final_df = None - for mode in self.filter_modes: - merged_df = pd.merge(self.df_students, df_chat, left_on=conf.EXCEL_COLS[mode], right_on="chat", how="left") - final_df = pd.concat([merged_df, final_df]) - - final_df.sort_values(by="time", inplace=True) - df_participated = final_df.drop(columns=["chat", "time"]) - df_participated = df_participated.groupby("users").first().reset_index() - - df_participated_updated = pd.merge(self.df_students, df_participated, left_on=conf.EXCEL_COLS["Name"], - right_on=conf.EXCEL_COLS["Name"], how="left", suffixes=["_x", ""]) - overlapping_columns = df_participated_updated.columns[df_participated_updated.columns.str.contains("_x")] - df_participated_updated = df_participated_updated.drop(columns=overlapping_columns) - return df_participated_updated - - @staticmethod - def get_zoom_users_not_included(df_participated, df_chat, not_included_part): - """ - get the zoom users who wrote something in the chat but it was not related to the csv/excel users info - - drops not relevant records from the df of zoom users that are not part of the class - - drops the zoom users who wrote something meaningful - :param df_participated: output of participators - that contains Nan for csv student that weren't mentioned (df) - :param df_chat: full chat of the relevant session (df) - :param not_included_part: zoom user name that will not be included (list of str) - :return: table of zoom users and there messages in the session (df) - """ - filt = ~(df_chat['users'].isin(df_participated['users'].dropna())) & \ - ~(df_chat['users'].str.contains('|'.join(not_included_part))) - df_zoom_not_correct = df_chat[filt] - return df_zoom_not_correct.drop(columns=["time"]).set_index("users") - - def get_attendance(self, not_included_part): - """ - - get table of attendant student from the csv/excel file and list of table of relevant data from zoom users that - didn't add a student. - - runs over all the session, for each one take the zoom user names and add it as a column to the final df - - if there is Nan it means that the student was missing in that session. the zoom user name is the user who added - the specific student. - - for each session append to a list all the df represent table of relevant data from zoom users that - didn't add a student - :param not_included_part: zoom user name that will not be included (list of str) - :return: 1. table of attendant student from the csv/excel file, - 2. list of table of relevant data from zoom users that didn't add a student - """ - df_zoom_not_correct_list = [] - attendance_df = self.df_students - for i, session in enumerate(self.df_sessions): - df_participated = self.get_participants(session) - attendance_df[f'session {i + 1}'] = df_participated['users'] - df_zoom_not_correct = self.get_zoom_users_not_included(df_participated, session, not_included_part) - df_zoom_not_correct_list.append(df_zoom_not_correct) - - return attendance_df, df_zoom_not_correct_list - -def main(): - my_class = Attendance(FILE_NAME, EXCEL_PATH, ['Name', "ID", "Phone"], TIME_DELTA, SENTENCE_START) - attendance_df, df_zoom_not_correct_list = my_class.get_attendance(WORDS_FOR_NOT_INCLUDED_PARTICIPATORS) - - print(attendance_df) - - for i in range(len(df_zoom_not_correct_list)): - print(f"zoom session {i + 1}") - print(df_zoom_not_correct_list[i]) - - -if __name__ == '__main__': - main() -