Skip to content

Commit

Permalink
Merge pull request #285 from 4dn-dcic/drr_glacier_copy_kms
Browse files Browse the repository at this point in the history
KMS encryption for glacier utils
  • Loading branch information
drio18 authored Sep 19, 2023
2 parents 989593d + c3c4efd commit 0066b66
Show file tree
Hide file tree
Showing 5 changed files with 31 additions and 5 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -121,3 +121,7 @@ ENV/

# PyCharm metadata
.idea/

# Vi
*.swp
*.swo
9 changes: 9 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ Change Log
----------


7.12.0
======

* In ``glacier_utils``:

* Add functionality for KMS key encrypted accounts


7.11.0
======

Expand All @@ -16,6 +24,7 @@ Change Log
* Fix in ``get_schema`` and ``get_schemas`` for the ``portal_vapp`` returning webtest.response.TestResponse
which has a ``json`` object property rather than a function.


7.10.0
======

Expand Down
20 changes: 16 additions & 4 deletions dcicutils/glacier_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ def __init__(self, env_name: str):
self.env_key = self.key_manager.get_keydict_for_env(env_name)
self.health_page = get_health_page(key=self.env_key, ff_env=env_name)

@property
def kms_key_id(self) -> str:
return self.health_page.get("s3_encrypt_key_id", "")

@classmethod
def is_glacier_storage_class(cls, storage_class: S3StorageClass):
return storage_class in S3_GLACIER_CLASSES
Expand Down Expand Up @@ -295,6 +299,9 @@ def _do_multipart_upload(self, bucket: str, key: str, total_size: int, part_size
}
if tags:
cmu['Tagging'] = tags
if self.kms_key_id:
cmu['ServerSideEncryption'] = 'aws:kms'
cmu['SSEKMSKeyId'] = self.kms_key_id
mpu = self.s3.create_multipart_upload(**cmu)
mpu_upload_id = mpu['UploadId']
except Exception as e:
Expand Down Expand Up @@ -381,16 +388,21 @@ def copy_object_back_to_original_location(self, bucket: str, key: str, storage_c
else:
# Force copy the object into standard in a single operation
copy_source = {'Bucket': bucket, 'Key': key}
copy_target = {
copy_args = {
'Bucket': bucket, 'Key': key,
'StorageClass': storage_class,
}
if version_id:
copy_source['VersionId'] = version_id
copy_target['CopySourceVersionId'] = version_id
copy_args['CopySourceVersionId'] = version_id
if tags:
copy_target['Tagging'] = tags
response = self.s3.copy_object(CopySource=copy_source, **copy_target)
copy_args['Tagging'] = tags
if self.kms_key_id:
copy_args['ServerSideEncryption'] = 'aws:kms'
copy_args['SSEKMSKeyId'] = self.kms_key_id
response = self.s3.copy_object(
**copy_args, CopySource=copy_source
)
PRINT(f'Response from boto3 copy:\n{response}')
PRINT(f'Object {bucket}/{key} copied back to its original location in S3')
return response
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "dcicutils"
version = "7.11.0"
version = "7.12.0"
description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
authors = ["4DN-DCIC Team <[email protected]>"]
license = "MIT"
Expand Down
1 change: 1 addition & 0 deletions test/test_glacier_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ def mock_health_page() -> dict:
'file_upload_bucket': 'cgap-dummy-main-application-cgap-dummy-files',
'namespace': 'cgap-dummy',
'processed_file_bucket': 'cgap-dummy-main-application-cgap-dummy-wfoutput',
's3_encrypt_key_id': 'dummy_kms_key',
}


Expand Down

0 comments on commit 0066b66

Please sign in to comment.