#24 delete chat and report data after time period

add timestamps for all the modules. currently deleting when posting new report

server/server/api/reports.py

@@ -8,9 +8,10 @@
 from server.api.utils import validate_classroom
 from server.config import RestErrors, ValidatorsConfig
 from server.models.marshals import report_resource_field, reports_list_fields
-from server.models.utils import store_sessions_and_chat_data
+from server.models.utils import store_sessions_and_chat_data, delete_old_chat_data
 from server.config import FlaskConfig
 
+
 class ReportsResource(Resource):
     """
     Resource responsible for report handling (creating new report, getting report information, deleting report)
@@ -73,13 +74,17 @@ def post(self, class_id, report_id=None):
         db.session.commit()
 
         student_status_df = report_object.student_status_table(new_report.id)
+        student_status_df["timestamp"] = datetime.utcnow()
         student_status_df.to_sql('student_status', con=db.engine, if_exists="append", index=False)
 
         # checks if to store the data for the session, chat and zoom names in the report
         if FlaskConfig.STORE_CHAT:
             # store all the chat, session and zoom names data from a report -  currently not supported
             store_sessions_and_chat_data(report_object.report_sessions, new_report.id)
 
+        # delete old chat files  # TODO : should be a cron job and not only when we post new report
+        delete_old_chat_data(ValidatorsConfig.TIME_PERIOD_TO_DELETE_REPORT_DATA)
+
         return new_report
 
     def delete(self, class_id, report_id=None):

server/server/api/validators.py

@@ -87,7 +87,7 @@ def date(self, value):
 
     def students_file(self, value):
         """
-        The function will make sure the student file has the right extenstion
+        The function will make sure the student file has the right extension
         :param value: the student file (FileStorage)
         :return: all the students from the file (Pandas DataFrame)
         """
@@ -103,7 +103,7 @@ def students_file(self, value):
 
     def chat_file(self, value):
         """
-        The function will make sure the chat file has the right extenstion
+        The function will make sure the chat file has the right extension
         :param value: the chat file (FileStorage)
         :return: The chat as dataframe (Pandas Dataframe)
         """

server/server/config.py

@@ -4,7 +4,7 @@
 class FlaskConfig:
     SECRET_KEY = 'TEMP_SECRET_KEY'
     SQLALCHEMY_DATABASE_URI = 'sqlite:///site.db'
-    STORE_CHAT = False # if this variable is True, all the chat, session and zoom names data from a report will be store in the datbase. currently not supported
+    STORE_CHAT = True # if this variable is True, all the chat, session and zoom names data from a report will be store in the datbase. currently not supported
 
 
 class ParseConfig:
@@ -28,8 +28,9 @@ class ValidatorsConfig:
     CHAT_FILE_EXT = [".txt"]
     STUDENTS_FILE_EXT = [".xls", ".xlsx", ".csv"]
     MAX_CLASSROOMS = 10  # Max classrooms per students
-    MAX_REPORTS = 7  # Max reports per classroom
+    MAX_REPORTS = 12  # Max reports per classroom
     MAX_STUDENTS_IN_CLASS = 200
+    TIME_PERIOD_TO_DELETE_REPORT_DATA = 60 * 60  # in seconds
 
 class RestErrors:
     INVALID_ROUTE = "route_doesn't_exists"

server/server/models/orm.py

@@ -1,5 +1,5 @@
 from server import db
-
+from datetime import datetime
 
 class TeacherModel(db.Model):
     __tablename__ = 'teacher'
@@ -8,7 +8,8 @@ class TeacherModel(db.Model):
     username = db.Column(db.String(40), unique=True, nullable=False)
     email = db.Column(db.String(120), unique=True, nullable=False)
     password = db.Column(db.String(60), nullable=False) # 60 chars because of the hashing algo
-
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow)
+
     classrooms = db.relationship('ClassroomModel', backref='teacher', lazy=True)
 
     def __repr__(self):
@@ -20,6 +21,7 @@ class ClassroomModel(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String(70), unique=False, nullable=False)
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow)
 
     teacher_id = db.Column(db.Integer, db.ForeignKey('teacher.id'), nullable=False)
     students = db.relationship('StudentModel', backref='classroom', cascade="all,delete", lazy=True)
@@ -49,6 +51,7 @@ class ReportModel(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     description = db.Column(db.Text(), unique=False, nullable=True)
     report_time = db.Column(db.DateTime(), unique=False, nullable=False) # first timestamp of chat file, date by user or date of the request
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow)
 
     class_id = db.Column(db.Integer, db.ForeignKey('classroom.id'), nullable=False)
     sessions = db.relationship('SessionModel', backref='report',cascade="all,delete", lazy=True)
@@ -60,6 +63,7 @@ class SessionModel(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     start_time = db.Column(db.DateTime(), unique=False, nullable=False) # first timestamp of chat session
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow)
 
     report_id = db.Column(db.Integer, db.ForeignKey('report.id'), nullable=False)
     zoom_names = db.relationship('ZoomNamesModel', backref='session',cascade="all,delete", lazy=True)
@@ -70,6 +74,7 @@ class ZoomNamesModel(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String(50), unique=False, nullable=True)
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow)
 
     session_id = db.Column(db.Integer, db.ForeignKey('session.id'), nullable=False)
     student_id = db.Column(db.Integer, db.ForeignKey('student.id'), nullable=True) # if Null - means the student wasn't present in the session
@@ -83,6 +88,7 @@ class ChatModel(db.Model):
     time = db.Column(db.DateTime, unique=False, nullable=False)  # time the message written
     message = db.Column(db.Text, unique=False, nullable=True)  # message zoom user wrote
     relevant = db.Column(db.Boolean, unique=False, nullable=False)  # True is message is part of the report
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow)
 
     zoom_names_id = db.Column(db.Integer, db.ForeignKey('zoom_names.id'), nullable=False)
 
@@ -92,6 +98,7 @@ class StudentStatus(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     status = db.Column(db.Integer, unique=False, nullable=False)
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow)
 
     student_id = db.Column(db.Integer, db.ForeignKey('student.id'), nullable=False)
     report_id = db.Column(db.Integer, db.ForeignKey('report.id'), nullable=False)

server/server/models/utils.py

@@ -1,6 +1,8 @@
 from server import auth, bcrypt, db
-from server.models.orm import TeacherModel, SessionModel,ZoomNamesModel
+from server.models.orm import TeacherModel, SessionModel, ZoomNamesModel, ChatModel
 import pandas as pd
+from datetime import datetime, timedelta
+
 
 def get_user(username_or_email, password):
     """
@@ -29,13 +31,31 @@ def store_sessions_and_chat_data(report_sessions, report_id):  # store sessions
         db.session.commit()
 
         zoom_names_df = session_object.zoom_names_table(session_table.id)
+        zoom_names_df["timestamp"] = datetime.utcnow()
         zoom_names_df.to_sql('zoom_names', con=db.engine, if_exists="append", index=False)
 
         zoom_names_df = pd.read_sql(ZoomNamesModel.query.filter_by(session_id=session_table.id).statement,
                                     con=db.engine)
         session_chat_df = session_object.chat_table(zoom_names_df)
+        session_chat_df["timestamp"] = datetime.utcnow()
         session_chat_df.to_sql('chat', con=db.engine, if_exists="append", index=False)
 
 
+def delete_old_report_data(time_period):
+    """
+    delete chat, zoom names and session data of all report created before the "time period" delte
+    :param time_period: input of time threshold - give in the ValidatorsConfig
+    :return: None
+    """
+    too_old = datetime.utcnow() - timedelta(seconds=time_period)
+
+    chat_reports_to_delete = ChatModel.query.filter(ChatModel.timestamp <= too_old).all()
+    zoom_names_to_delete = ZoomNamesModel.query.filter(ZoomNamesModel.timestamp <= too_old).all()
+    sessions_to_delete = SessionModel.query.filter(SessionModel.timestamp <= too_old).all()
+    to_delete = chat_reports_to_delete + sessions_to_delete + zoom_names_to_delete
+    for obj in to_delete:
+        db.session.delete(obj)
+    db.session.commit()
+