Merge pull request #13 from InbarShirizly/integration

Integration

Server/__init__.py → Server/Server/__init__.py

@@ -10,7 +10,7 @@
 db = SQLAlchemy(app)
 bcrypt = Bcrypt(app)
 login_manager = LoginManager(app)
-login_manager.login_view = 'login'
+login_manager.login_view = 'users.login'
 login_manager.login_message_category = 'info'
 
 

Server/Server/classrooms/__init__.py

@@ -0,0 +1,5 @@
+from Server.config import ParseConfig
+from Server.classrooms.loading_classroom_file import ParseClassFile
+
+
+parser = ParseClassFile.from_object(ParseConfig)

Server/Server/classrooms/attendance_check.py

@@ -0,0 +1,109 @@
+import numpy as np
+import pandas as pd
+
+
+class Attendance:
+    """
+    receives student class df, the zoom chat and other configuration. returns:
+     1. table of attendant students from the student class df
+     2. list of table of relevant data from zoom users that didn't add a student return
+    """
+    def __init__(self, chat_df, students_df, filter_modes, time_delta, start_sentence, not_included_zoom_users):
+        """
+        - convert the chat text file to a data frame and arrange columns.
+        - creates df for each session according the appearance of the start sentence and time delta
+        :param chat_df: zoom chat in df (df)
+        :param students_df: student class raw data (df)
+        :param filter_modes: filters the user picked for parsing the text file (list of str)
+        :param time_delta: max time from start sentence to the last message to parse in each session in minutes (int)
+        :param start_sentence: start sentence that initiate sessions for parse (str)
+        :return: data frame with the data from the chat
+        """
+
+        self.first_message_time = chat_df["time"].sort_values().iloc[0]
+        start_indices = chat_df.index[chat_df['message'].apply(lambda string: start_sentence.lower() in string.lower())] #TODO: slice by time or by next message
+        df_students_for_report = students_df.set_index("id").astype(str).reset_index()  # set all columns to str except the id
+
+        self._sessions = []
+        for start_index in start_indices:
+            df_session = Attendance.get_df_of_time_segment(chat_df, start_index, time_delta)
+            self._sessions.append(Session(df_students_for_report, df_session, filter_modes, not_included_zoom_users))
+
+    @staticmethod
+    def get_df_of_time_segment(df, start_index, time_delta):
+
+        time_delta = np.timedelta64(time_delta, 'm')
+        time_segment_start = df.loc[start_index, "time"]
+        time_filt = (df["time"] >= time_segment_start) & \
+                    (df["time"] <= time_segment_start + time_delta)
+
+        return df.loc[time_filt]
+
+    @property
+    def report_sessions(self):
+        return self._sessions
+
+
+class Session:
+
+    def __init__(self, df_students, df_session_chat, filter_modes, not_included_zoom_users):
+
+        self._first_message_time = df_session_chat["time"].sort_values().iloc[0]
+        self._relevant_chat = self.get_participants_in_session(df_students,filter_modes, df_session_chat, not_included_zoom_users)
+
+
+    @ staticmethod
+    def get_participants_in_session(df_students, filter_modes, df_chat, not_included_zoom_users):
+        """
+        finds students that attendant to the session. runs over each mode which represent different way to declare that
+        the student attendant (for example: phone number, ID). merges this data to the csv table with the zoom name that
+        added it
+        :param df_chat: that table of the chat for the specific session
+        :return: df of the attendance in the session
+        """
+        final_df = None
+        for mode in filter_modes:
+            merged_df = pd.merge(df_students, df_chat.reset_index(), left_on=mode, right_on="message", how="left")
+            final_df = pd.concat([merged_df, final_df])
+
+        final_df.sort_values(by="time", inplace=True)
+        df_participated = final_df.groupby("zoom_name").first().reset_index()
+        df_participated["index"] = df_participated["index"].astype(int)
+        df_participated = df_participated.loc[:, ["id", "zoom_name", "time", "message", "index"]].set_index("index")
+
+        filt = df_chat['zoom_name'].str.contains('|'.join(not_included_zoom_users))
+        df_relevant_chat = pd.merge(df_chat[~filt], df_participated, how="left")
+
+        df_relevant_chat["relevant"] = df_relevant_chat["id"].apply(lambda x: 1 if x == x else 0)
+        df_relevant_chat["id"] = df_relevant_chat["id"].apply(lambda x: int(x) if x == x else -1)
+        return df_relevant_chat
+
+
+    def zoom_names_table(self, session_id):
+        zoom_df = self._relevant_chat.loc[:, ["zoom_name", "id"]].rename(columns={"zoom_name": "name", "id": "student_id"})
+        zoom_df['session_id'] = pd.Series([session_id] * zoom_df.shape[0])
+        return zoom_df.sort_values(by="student_id", ascending=False).groupby("name").first().reset_index()
+
+    def chat_table(self, zoom_df):
+        relevant_chat = self._relevant_chat.drop(columns=["id"])
+        chat_session_table = pd.merge(relevant_chat, zoom_df, left_on="zoom_name", right_on="name")
+        return chat_session_table.drop(columns=["zoom_name", "name", "session_id", "student_id"]).rename(columns={"id": "zoom_names_id"})
+
+
+
+if __name__ == '__main__':
+    from utils import create_chat_df, create_students_df
+
+    chat_file_path = r"C:\Users\Inbar Shirizly\Documents\python\useful\ITC_programs\zoom_attendance_check\chat files\meeting_example_full_name.txt"
+    excel_file_path = r"C:\Users\Inbar Shirizly\Documents\python\useful\ITC_programs\zoom_attendance_check\student_csv_examples\example_data_already_prepared.xlsx"
+
+
+    with open(chat_file_path, "r", encoding="utf-8") as f:
+        chat_df = create_chat_df(f.readlines())
+    df_students = create_students_df(file_name=excel_file_path.split("\\")[-1], file_data=excel_file_path)
+
+    my_class = Attendance(chat_df, df_students, ['name', "id_number", "phone"], 5, "Attendance check", ["ITC", "Tech", "Challenge"])
+
+    df_part_session = my_class._sessions[0]
+    df_part_session.zoom_names_table(2)
+

Server/Server/classrooms/forms.py

@@ -0,0 +1,17 @@
+from flask_wtf import FlaskForm
+from flask_wtf.file import FileField, FileAllowed, FileRequired
+from wtforms.validators import DataRequired
+from wtforms import StringField, SubmitField, IntegerField
+
+
+class CreateClassForm(FlaskForm):
+    name = StringField('Name of the class', validators=[DataRequired()])
+    students_file = FileField('Csv/Excel file of students', validators=[FileAllowed(['xlsx', 'csv', 'xls']), FileRequired()])
+    submit = SubmitField('Create')
+
+
+class CreateReportForm(FlaskForm):
+    start_sentence = StringField('Sentence to start the zoom check', validators=[DataRequired()])
+    chat_file = FileField('Zoom chat file', validators=[FileRequired(), FileAllowed(['txt'])]) 
+    time = IntegerField('Number of minutes', validators=[DataRequired()])
+    submit = SubmitField('Create Report')

Server/Server/classrooms/loading_classroom_file.py

@@ -0,0 +1,64 @@
+import pandas as pd
+import numpy as np
+import re
+
+
+class ParseClassFile:
+
+    def __init__(self, file_cols_dict, mashov_cols, gender_dict):
+        self._file_cols_dict = file_cols_dict
+        self._mashov_cols = mashov_cols
+        self._gender_dict = gender_dict
+
+    @classmethod
+    def from_object(cls, config):
+        return cls(
+            config.FILE_COLS_DICT,
+            config.MASHOV_COLS,
+            config.GENDER_DICT
+        )
+
+
+    def parse_df(self, df_students):
+        relevant_cols = [col for col in df_students.columns if not col.startswith("Unnamed")]
+        current_excel_dict = {}
+
+        for col in relevant_cols:
+            for key, col_options in self._file_cols_dict.items():
+                if col in col_options:
+                    current_excel_dict[key] = df_students[col]
+
+        if len(current_excel_dict) == 0 and len(relevant_cols) <= 1:
+            print("Mashov file")
+            header_index = df_students.notnull().sum(axis=1).argmax()
+            df_students = pd.DataFrame(df_students.values[header_index + 1:-2], columns=df_students.iloc[header_index])
+            df_students.dropna(axis=0, how='all', inplace=True)
+            df_students.dropna(axis=1, how='all', inplace=True)
+            df_students.rename(columns={np.nan: 'name', "פרטי תלמיד": 'name', "כיתה": "org_class"}, inplace=True)
+            df_students = df_students.loc[:, self._mashov_cols]
+
+            mashov_name_pattern = re.compile(r"([\u0590-\u05fe ]+)([(\u0590-\u05fe)]+)")
+            df_name_gender = df_students['name'].str.extract(mashov_name_pattern, expand=False)
+            df_students['gender'] = df_name_gender[1].str.extract("\(([\u0590-\u05fe ])\)")
+            df_students['gender'] = df_students['gender'].apply(self.gender_assign, gender_dict=self._gender_dict)
+            df_students['name'] = df_name_gender[0]
+
+        else:
+            df_students = pd.DataFrame(current_excel_dict)
+
+        for col in self._file_cols_dict.keys():
+            try:
+                df_students[col] = df_students[col]
+            except KeyError:
+                df_students[col] = pd.Series([np.nan] * df_students.shape[0])
+
+        final_df = df_students[list(self._file_cols_dict.keys())]
+
+        return final_df.reset_index().drop(columns="index")
+
+    @staticmethod
+    def gender_assign(string, gender_dict):
+        for key, vals in gender_dict.items():
+            if string in vals:
+                return key
+        return ""

Server/Server/classrooms/routes.py

@@ -0,0 +1,74 @@
+from flask import render_template, redirect, url_for, flash, Blueprint, request, session
+from Server import db
+from Server.classrooms.forms import CreateClassForm, CreateReportForm
+from Server.models import ClassroomModel, StudentModel, ReportModel, SessionModel, ZoomNamesModel, ChatModel
+from flask_login import current_user, login_required
+import pandas as pd
+from Server.classrooms.attendance_check import Attendance
+from Server.classrooms.utils import create_chat_df, create_students_df
+from Server.classrooms import parser
+from  datetime import datetime
+
+classrooms = Blueprint('classrooms', __name__)
+
+
+@classrooms.route('/', methods=['GET', 'POST'])
+@classrooms.route('/home', methods=['GET', 'POST'])
+@login_required
+def home():
+    form = CreateClassForm()
+    if form.validate_on_submit():
+        students_df = create_students_df(form.students_file.data.filename, form.students_file.data)
+        students = parser.parse_df(students_df)
+        new_class = ClassroomModel(name=form.name.data, teacher_model=current_user)
+        db.session.add(new_class)
+        db.session.commit()
+        students['class_id'] = pd.Series([new_class.id] * students.shape[0])
+        students.to_sql('student_model', con=db.engine, if_exists="append", index=False)
+        return redirect(url_for('classrooms.classroom', class_id=new_class.id))
+    return render_template('home.html', form=form)
+
+
+@classrooms.route('/classroom/<class_id>', methods=['GET', 'POST'])
+@login_required
+def classroom(class_id):
+    current_class = ClassroomModel.query.filter_by(id=class_id, teacher_model=current_user).first() # Making sure the class belongs to the current user
+    if current_class is None:
+        flash('Invalid class!', 'danger')
+        return redirect(url_for('classrooms.home'))
+
+    form = CreateReportForm() 
+    if form.validate_on_submit(): # If form was submitted, creating report for the class
+        description = "this is the best class"  # TODO : add from the form file
+        report_date = datetime.now().date()    # TODO : add from the form file
+        students_df = pd.read_sql(StudentModel.query.filter_by(class_id=class_id).statement, con=db.engine)
+        chat_file = form.chat_file.data.stream.read().decode("utf-8").split("\n")
+        chat_df = create_chat_df(chat_file)
+
+        report_object = Attendance(chat_df, students_df, ['name', "id_number", "phone"], form.time.data, form.start_sentence.data, ["ITC", "Tech", "Challenge"])
+
+        new_report = ReportModel(description=description, start_time=report_object.first_message_time, report_date=report_date, class_id=class_id)
+        db.session.add(new_report)
+        db.session.commit()
+
+        # insert relevant data for each session to the database
+        for session_object in report_object.report_sessions:
+            session_table = SessionModel(start_time=session_object._first_message_time, report_id=new_report.id)
+            db.session.add(session_table)
+            db.session.commit()
+
+            zoom_names_df = session_object.zoom_names_table(session_table.id)
+            zoom_names_df.to_sql('zoom_names_model', con=db.engine, if_exists="append", index=False)
+
+            zoom_names_df = pd.read_sql(ZoomNamesModel.query.filter_by(session_id=session_table.id).statement, con=db.engine)
+            session_chat_df = session_object.chat_table(zoom_names_df)
+            session_chat_df.to_sql('chat_model', con=db.engine, if_exists="append", index=False)
+
+
+
+        return render_template("report.html")
+
+
+    return render_template('classroom.html', current_class=current_class, form=form)
+
+

Server/Server/classrooms/utils.py

@@ -0,0 +1,26 @@
+import re
+from datetime import datetime
+import pandas as pd
+
+def create_chat_df(chat_file):
+
+    regex_pattern = re.compile(r"(^\d{2}.\d{2}.\d{2})\s+From\s\s([\s\S]+)\s:\s([\s\S]+)")
+    chat_content = [re.search(regex_pattern, line).groups() for line in chat_file if re.match(regex_pattern, line)]
+
+    chat_df = pd.DataFrame(chat_content, columns=["time", "zoom_name", "message"])
+    chat_df['message'] = chat_df['message'].str[:-1].astype(str)
+    chat_df["time"] = chat_df["time"].apply(lambda string: datetime.strptime(string, "%H:%M:%S"))
+
+    return chat_df
+
+
+def create_students_df(file_name, file_data):
+
+    if file_name.endswith(".csv"):
+        df_students = pd.read_csv(file_data)
+    elif file_name.endswith(".xlsx"):
+        df_students = pd.read_excel(file_data)
+    else:
+        df_students = pd.read_html(file_data, header=1)[0]
+
+    return df_students

Server/Server/config.py

@@ -0,0 +1,16 @@
+class FlaskConfig:
+    SECRET_KEY = 'TEMP_SECRET_KEY'
+    SQLALCHEMY_DATABASE_URI = 'sqlite:///site.db'
+
+
+class ParseConfig:
+
+    FILE_COLS_DICT = {
+        "name": ["שם התלמיד", "תלמידים", "שמות", "שם", "סטודנט"],
+        "id_number": ["תעודת זהות", "ת.ז.", "ת.ז", "תז"],
+        "phone": ["טלפון", "מספר טלפון", "מס טלפון"],
+        "gender": ["מין"],
+        "org_class": ["כיתה"]
+    }
+    MASHOV_COLS = ["name", "org_class"]
+    GENDER_DICT = {1: ["זכר", "ז", "(ז)"], 0: ["נקבה", "נ", "(נ)"]}