feat: backend to supports multiple aws s3 buckets

edx_sga/backends.py

@@ -0,0 +1,43 @@
+from storages.backends.s3boto3 import S3Boto3Storage
+from django.core.files.storage import default_storage
+
+AWS_S3_FIELDS = [
+    'aws_s3_access_key',
+    'aws_s3_secret_key',
+    'aws_s3_bucket_name',
+    'aws_s3_region_name'
+]
+
+class StaffGradedAssignmentStorage:
+    def __init__(self, aws_s3_bucket):
+        self.aws_s3_storage = None
+
+        if aws_s3_bucket:
+            self.check_s3_bucket_keys(aws_s3_bucket)
+            self.aws_s3_storage = S3Boto3Storage(
+                access_key=aws_s3_bucket.get('aws_s3_access_key'),
+                secret_key=aws_s3_bucket.get('aws_s3_secret_key'),
+                bucket_name=aws_s3_bucket.get('aws_s3_bucket_name'),
+                region_name=aws_s3_bucket.get('aws_s3_region_name')
+            )
+
+    def check_s3_bucket_keys(self, settings_dict):
+        """
+        Method checks integrity and structure of the Xblock settings given.
+        """
+        if not isinstance(settings_dict, dict):
+            raise ValueError("Wrong XBlock settings definition. Expected dict.")
+
+        for aws_s3_field in AWS_S3_FIELDS:
+            if aws_s3_field not in settings_dict.keys() or not settings_dict[aws_s3_field]:
+                raise ValueError(f"Error on Xblock settings with field '{aws_s3_field}'.")
+
+        for value, key in settings_dict.items():
+            if not value and isinstance(value, str):
+                raise ValueError(f"Value error on Xblock Settings: {key}.")
+
+    def sga_storage(self):
+        """
+        Returns the storage to be used wheter the default Django's storage or AWS S3 Bucket
+        """
+        return self.aws_s3_storage if self.aws_s3_storage else default_storage

edx_sga/sga.py

@@ -19,7 +19,6 @@
 from django.conf import settings
 from django.core.exceptions import PermissionDenied
 from django.core.files import File
-from django.core.files.storage import default_storage
 from django.template import Context, Template
 from django.utils.encoding import force_text
 from django.utils.timezone import now as django_now
@@ -40,6 +39,7 @@
 from xmodule.contentstore.content import StaticContent
 from xmodule.util.duedate import get_extended_due_date
 
+from edx_sga.backends import StaffGradedAssignmentStorage
 from edx_sga.constants import ITEM_TYPE
 from edx_sga.showanswer import ShowAnswerXBlockMixin
 from edx_sga.tasks import get_zip_file_name, get_zip_file_path, zip_student_submissions
@@ -73,6 +73,7 @@ def getter(inst):
     return property(getter)
 
 
+@XBlock.wants("settings")
 class StaffGradedAssignmentXBlock(
     StudioEditableXBlockMixin, ShowAnswerXBlockMixin, XBlock
 ):
@@ -160,6 +161,27 @@ class StaffGradedAssignmentXBlock(
         help=_("When the annotated file was uploaded"),
     )
 
+    @property
+    def xblock_settings(self):
+        """
+        Method access to LMS/CMS runtime service to extract the settings_bucket for Xblock.
+        """
+        settings_service = self.runtime.service(self, "settings") or {}
+
+        if not settings_service:
+            return {}
+
+        return settings_service.get_settings_bucket(self)
+
+    @property
+    def xblock_storage(self):
+        """
+        Defines the storage to be used in this Xblock instance.
+        """
+        return StaffGradedAssignmentStorage(aws_s3_bucket = self.xblock_settings.get("aws_s3_storage_data", {})).\
+            sga_storage()
+
+
     @classmethod
     def student_upload_max_size(cls):
         """
@@ -283,10 +305,10 @@ def upload_assignment(self, request, suffix=""):
             path,
             user.username,
         )
-        if default_storage.exists(path):
+        if self.xblock_storage.exists(path):
             # save latest submission
-            default_storage.delete(path)
-        default_storage.save(path, File(upload.file))
+            self.xblock_storage.delete(path)
+        self.xblock_storage.save(path, File(upload.file))
         return Response(json_body=self.student_state())
 
     @XBlock.handler
@@ -328,8 +350,8 @@ def staff_upload_annotated(self, request, suffix=""):
         state["annotated_mimetype"] = mimetypes.guess_type(upload.file.name)[0]
         state["annotated_timestamp"] = utcnow().strftime(DateTime.DATETIME_FORMAT)
         path = self.file_storage_path(sha1, filename)
-        if not default_storage.exists(path):
-            default_storage.save(path, File(upload.file))
+        if not self.xblock_storage.exists(path):
+            self.xblock_storage.save(path, File(upload.file))
         module.state = json.dumps(state)
         module.save()
         log.info(
@@ -620,8 +642,8 @@ def clear_student_state(self, *args, **kwargs):
             submission_file_path = self.file_storage_path(
                 submission_file_sha1, submission_filename
             )
-            if default_storage.exists(submission_file_path):
-                default_storage.delete(submission_file_path)
+            if self.xblock_storage.exists(submission_file_path):
+                self.xblock_storage.delete(submission_file_path)
             submissions_api.reset_score(
                 student_id, self.block_course_id, self.block_id, clear_state=True
             )
@@ -977,7 +999,7 @@ def is_zip_file_available(self, user):
         zip_file_path = get_zip_file_path(
             user.username, self.block_course_id, self.block_id, self.location
         )
-        return default_storage.exists(zip_file_path)
+        return self.xblock_storage.exists(zip_file_path)
 
     def count_archive_files(self, user):
         """
@@ -987,7 +1009,7 @@ def count_archive_files(self, user):
         zip_file_path = get_zip_file_path(
             user.username, self.block_course_id, self.block_id, self.location
         )
-        with default_storage.open(zip_file_path, "rb") as zip_file:
+        with self.xblock_storage.open(zip_file_path, "rb") as zip_file:
             with closing(ZipFile(zip_file)) as archive:
                 return len(archive.infolist())
 

edx_sga/tests/test_backends.py

@@ -0,0 +1,100 @@
+
+import unittest
+from unittest.mock import MagicMock, patch
+from django.core.files.storage import default_storage
+from edx_sga.backends import StaffGradedAssignmentStorage
+
+
+class StaffGradedAssignmentStorageTest(unittest.TestCase):
+
+    @patch('edx_sga.backends.S3Boto3Storage')
+    def test_returns_s3_storage_xblock_settings_given(self, mock_s3_aws_storage):
+        """
+        Test StaffGradedAssignmentStorage to return a valid S3 storage when the bucket data is given
+        """
+        xblock_settings = {
+            'aws_s3_access_key': 'access_key',
+            'aws_s3_secret_key': 'secret_key',
+            'aws_s3_bucket_name': 'bucket_name',
+            'aws_s3_region_name': 'region_name'
+        }
+
+        xblock_storage = StaffGradedAssignmentStorage(xblock_settings).sga_storage()
+
+        mock_s3_aws_storage.assert_called_with(
+            access_key='access_key',
+            secret_key='secret_key',
+            bucket_name='bucket_name',
+            region_name='region_name'
+        )
+
+        self.assertEqual(xblock_storage, mock_s3_aws_storage.return_value)
+
+    def test_returns_default_storage_xblock_settings_no_given(self):
+        """
+        Test StaffGradedAssignmentStorage to return the default Django's storage when no settings are defined
+        """
+        xblock_settings = None
+
+        xblock_storage = StaffGradedAssignmentStorage(xblock_settings).sga_storage()
+        self.assertEqual(xblock_storage, default_storage)
+
+    def test_check_s3_bucket_proper_values(self):
+        """
+        Checks for check_s3_bucket_keys method to check and return the data given in the format expected.
+        """
+        xblock_settings = {
+            'aws_s3_access_key': 'access_key',
+            'aws_s3_secret_key': 'secret_key',
+            'aws_s3_bucket_name': 'bucket_name',
+            'aws_s3_region_name': 'region_name'
+        }
+
+        try:
+            xblock_storage_instance = StaffGradedAssignmentStorage(xblock_settings)
+            xblock_storage_instance.check_s3_bucket_keys(xblock_settings)
+        except ValueError:
+            self.fail('Unexpected raise from check_s3_bucket_keys method.')
+
+
+    def test_check_s3_bucket_incomplete_values(self):
+        """
+        Checks for check_s3_bucket_keys method to raises an exception when the data passed doesn't fullfils the needs.
+        """
+        xblock_settings = {
+            'aws_s3_access_key': 'access_key',
+            'aws_s3_secret_key': 'secret_key',
+        }
+        xblock_storage_instance = StaffGradedAssignmentStorage(None)
+        with self.assertRaises(ValueError) as exception:
+            xblock_storage_instance.check_s3_bucket_keys(xblock_settings)
+        self.assertIn("Error on Xblock settings with field 'aws_s3_bucket_name'", str(exception.exception))
+
+    def test_sga_storage_returns_defined_storage(self):
+        """
+        Test for StaffGradedAssignmentStorage to have assigned the expected storage when the data is passed
+        """
+        mock_aws_s3_storage = MagicMock(name='MockS3Storage')
+        xblock_settings = {
+            'aws_s3_access_key': 'access_key',
+            'aws_s3_secret_key': 'secret_key',
+            'aws_s3_bucket_name': 'bucket_name',
+            'aws_s3_region_name': 'region_name'
+        }
+
+        sga_storage_instance = StaffGradedAssignmentStorage(xblock_settings)
+        sga_storage_instance.aws_s3_storage = mock_aws_s3_storage
+
+        result = sga_storage_instance.sga_storage()
+        self.assertEqual(result, mock_aws_s3_storage)
+
+    @patch('edx_sga.backends.default_storage')
+    def test_sga_storage_returns_default_storage(self, mock_default_storage):
+        """
+        Test for StaffGradedAssignmentStorage to uses the Django's default storage.
+        """
+        sga_storage_instance = StaffGradedAssignmentStorage({})
+        sga_storage_instance.aws_s3_storage = None
+
+        result = sga_storage_instance.sga_storage()
+        self.assertEqual(result, mock_default_storage)