From c190868c35a4814cad0cb419d78955acba9b6c82 Mon Sep 17 00:00:00 2001 From: Inbar Shirizly Date: Sun, 1 Nov 2020 21:45:15 +0200 Subject: [PATCH] #24 delete chat and report data after time period add timestamps for all the modules. currently deleting when posting new report --- server/server/api/reports.py | 7 ++++++- server/server/api/validators.py | 4 ++-- server/server/config.py | 5 +++-- server/server/models/orm.py | 11 +++++++++-- server/server/models/utils.py | 22 +++++++++++++++++++++- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/server/server/api/reports.py b/server/server/api/reports.py index aa2e948..1a4e50e 100644 --- a/server/server/api/reports.py +++ b/server/server/api/reports.py @@ -8,9 +8,10 @@ from server.api.utils import validate_classroom from server.config import RestErrors, ValidatorsConfig from server.models.marshals import report_resource_field, reports_list_fields -from server.models.utils import store_sessions_and_chat_data +from server.models.utils import store_sessions_and_chat_data, delete_old_chat_data from server.config import FlaskConfig + class ReportsResource(Resource): """ Resource responsible for report handling (creating new report, getting report information, deleting report) @@ -73,6 +74,7 @@ def post(self, class_id, report_id=None): db.session.commit() student_status_df = report_object.student_status_table(new_report.id) + student_status_df["timestamp"] = datetime.utcnow() student_status_df.to_sql('student_status', con=db.engine, if_exists="append", index=False) # checks if to store the data for the session, chat and zoom names in the report @@ -80,6 +82,9 @@ def post(self, class_id, report_id=None): # store all the chat, session and zoom names data from a report - currently not supported store_sessions_and_chat_data(report_object.report_sessions, new_report.id) + # delete old chat files # TODO : should be a cron job and not only when we post new report + delete_old_chat_data(ValidatorsConfig.TIME_PERIOD_TO_DELETE_REPORT_DATA) + return new_report def delete(self, class_id, report_id=None): diff --git a/server/server/api/validators.py b/server/server/api/validators.py index 5649f48..eb11551 100644 --- a/server/server/api/validators.py +++ b/server/server/api/validators.py @@ -87,7 +87,7 @@ def date(self, value): def students_file(self, value): """ - The function will make sure the student file has the right extenstion + The function will make sure the student file has the right extension :param value: the student file (FileStorage) :return: all the students from the file (Pandas DataFrame) """ @@ -103,7 +103,7 @@ def students_file(self, value): def chat_file(self, value): """ - The function will make sure the chat file has the right extenstion + The function will make sure the chat file has the right extension :param value: the chat file (FileStorage) :return: The chat as dataframe (Pandas Dataframe) """ diff --git a/server/server/config.py b/server/server/config.py index 2b58775..c886b5d 100644 --- a/server/server/config.py +++ b/server/server/config.py @@ -4,7 +4,7 @@ class FlaskConfig: SECRET_KEY = 'TEMP_SECRET_KEY' SQLALCHEMY_DATABASE_URI = 'sqlite:///site.db' - STORE_CHAT = False # if this variable is True, all the chat, session and zoom names data from a report will be store in the datbase. currently not supported + STORE_CHAT = True # if this variable is True, all the chat, session and zoom names data from a report will be store in the datbase. currently not supported class ParseConfig: @@ -28,8 +28,9 @@ class ValidatorsConfig: CHAT_FILE_EXT = [".txt"] STUDENTS_FILE_EXT = [".xls", ".xlsx", ".csv"] MAX_CLASSROOMS = 10 # Max classrooms per students - MAX_REPORTS = 7 # Max reports per classroom + MAX_REPORTS = 12 # Max reports per classroom MAX_STUDENTS_IN_CLASS = 200 + TIME_PERIOD_TO_DELETE_REPORT_DATA = 60 * 60 # in seconds class RestErrors: INVALID_ROUTE = "route_doesn't_exists" diff --git a/server/server/models/orm.py b/server/server/models/orm.py index 91f76b0..9145380 100644 --- a/server/server/models/orm.py +++ b/server/server/models/orm.py @@ -1,5 +1,5 @@ from server import db - +from datetime import datetime class TeacherModel(db.Model): __tablename__ = 'teacher' @@ -8,7 +8,8 @@ class TeacherModel(db.Model): username = db.Column(db.String(40), unique=True, nullable=False) email = db.Column(db.String(120), unique=True, nullable=False) password = db.Column(db.String(60), nullable=False) # 60 chars because of the hashing algo - + timestamp = db.Column(db.DateTime, default=datetime.utcnow) + classrooms = db.relationship('ClassroomModel', backref='teacher', lazy=True) def __repr__(self): @@ -20,6 +21,7 @@ class ClassroomModel(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(70), unique=False, nullable=False) + timestamp = db.Column(db.DateTime, default=datetime.utcnow) teacher_id = db.Column(db.Integer, db.ForeignKey('teacher.id'), nullable=False) students = db.relationship('StudentModel', backref='classroom', cascade="all,delete", lazy=True) @@ -49,6 +51,7 @@ class ReportModel(db.Model): id = db.Column(db.Integer, primary_key=True) description = db.Column(db.Text(), unique=False, nullable=True) report_time = db.Column(db.DateTime(), unique=False, nullable=False) # first timestamp of chat file, date by user or date of the request + timestamp = db.Column(db.DateTime, default=datetime.utcnow) class_id = db.Column(db.Integer, db.ForeignKey('classroom.id'), nullable=False) sessions = db.relationship('SessionModel', backref='report',cascade="all,delete", lazy=True) @@ -60,6 +63,7 @@ class SessionModel(db.Model): id = db.Column(db.Integer, primary_key=True) start_time = db.Column(db.DateTime(), unique=False, nullable=False) # first timestamp of chat session + timestamp = db.Column(db.DateTime, default=datetime.utcnow) report_id = db.Column(db.Integer, db.ForeignKey('report.id'), nullable=False) zoom_names = db.relationship('ZoomNamesModel', backref='session',cascade="all,delete", lazy=True) @@ -70,6 +74,7 @@ class ZoomNamesModel(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(50), unique=False, nullable=True) + timestamp = db.Column(db.DateTime, default=datetime.utcnow) session_id = db.Column(db.Integer, db.ForeignKey('session.id'), nullable=False) student_id = db.Column(db.Integer, db.ForeignKey('student.id'), nullable=True) # if Null - means the student wasn't present in the session @@ -83,6 +88,7 @@ class ChatModel(db.Model): time = db.Column(db.DateTime, unique=False, nullable=False) # time the message written message = db.Column(db.Text, unique=False, nullable=True) # message zoom user wrote relevant = db.Column(db.Boolean, unique=False, nullable=False) # True is message is part of the report + timestamp = db.Column(db.DateTime, default=datetime.utcnow) zoom_names_id = db.Column(db.Integer, db.ForeignKey('zoom_names.id'), nullable=False) @@ -92,6 +98,7 @@ class StudentStatus(db.Model): id = db.Column(db.Integer, primary_key=True) status = db.Column(db.Integer, unique=False, nullable=False) + timestamp = db.Column(db.DateTime, default=datetime.utcnow) student_id = db.Column(db.Integer, db.ForeignKey('student.id'), nullable=False) report_id = db.Column(db.Integer, db.ForeignKey('report.id'), nullable=False) diff --git a/server/server/models/utils.py b/server/server/models/utils.py index cc65dda..8bddd0b 100644 --- a/server/server/models/utils.py +++ b/server/server/models/utils.py @@ -1,6 +1,8 @@ from server import auth, bcrypt, db -from server.models.orm import TeacherModel, SessionModel,ZoomNamesModel +from server.models.orm import TeacherModel, SessionModel, ZoomNamesModel, ChatModel import pandas as pd +from datetime import datetime, timedelta + def get_user(username_or_email, password): """ @@ -29,13 +31,31 @@ def store_sessions_and_chat_data(report_sessions, report_id): # store sessions db.session.commit() zoom_names_df = session_object.zoom_names_table(session_table.id) + zoom_names_df["timestamp"] = datetime.utcnow() zoom_names_df.to_sql('zoom_names', con=db.engine, if_exists="append", index=False) zoom_names_df = pd.read_sql(ZoomNamesModel.query.filter_by(session_id=session_table.id).statement, con=db.engine) session_chat_df = session_object.chat_table(zoom_names_df) + session_chat_df["timestamp"] = datetime.utcnow() session_chat_df.to_sql('chat', con=db.engine, if_exists="append", index=False) +def delete_old_report_data(time_period): + """ + delete chat, zoom names and session data of all report created before the "time period" delte + :param time_period: input of time threshold - give in the ValidatorsConfig + :return: None + """ + too_old = datetime.utcnow() - timedelta(seconds=time_period) + + chat_reports_to_delete = ChatModel.query.filter(ChatModel.timestamp <= too_old).all() + zoom_names_to_delete = ZoomNamesModel.query.filter(ZoomNamesModel.timestamp <= too_old).all() + sessions_to_delete = SessionModel.query.filter(SessionModel.timestamp <= too_old).all() + to_delete = chat_reports_to_delete + sessions_to_delete + zoom_names_to_delete + for obj in to_delete: + db.session.delete(obj) + db.session.commit() + \ No newline at end of file