From 3daa71477bcbfade66327757a296bb090a382920 Mon Sep 17 00:00:00 2001 From: Peggy Date: Mon, 29 Jan 2024 09:04:47 -0800 Subject: [PATCH] Transient storage: s3 delete (#706) # Description This PR includes the following proposed change(s): - spdbt-1930(s3 file deletion); --- .../PersonalLicenceAppManager.cs | 1 + .../Document/Contract.cs | 1 + .../Document/DocumentRepository.cs | 56 ++++++++++--------- src/Spd.Utilities.FileStorage/Contract.cs | 2 + .../FileStorageService.cs | 28 ++++++++++ .../ServiceExtensions.cs | 2 +- 6 files changed, 64 insertions(+), 26 deletions(-) diff --git a/src/Spd.Manager.Licence/PersonalLicenceAppManager.cs b/src/Spd.Manager.Licence/PersonalLicenceAppManager.cs index aecca211e..57f4ab718 100644 --- a/src/Spd.Manager.Licence/PersonalLicenceAppManager.cs +++ b/src/Spd.Manager.Licence/PersonalLicenceAppManager.cs @@ -206,6 +206,7 @@ await _documentRepository.ManageAsync(new CreateDocumentCmd DocumentType2 = docType2, SubmittedByApplicantId = appResponse.ContactId, ExpiryDate = expiredDate, + ToTransientBucket = false }, ct); } } diff --git a/src/Spd.Resource.Applicants/Document/Contract.cs b/src/Spd.Resource.Applicants/Document/Contract.cs index f460c4dbb..cc2335051 100644 --- a/src/Spd.Resource.Applicants/Document/Contract.cs +++ b/src/Spd.Resource.Applicants/Document/Contract.cs @@ -45,6 +45,7 @@ public record CreateDocumentCmd : DocumentCmd public DocumentTypeEnum? DocumentType { get; set; } //tag1 public DocumentTypeEnum? DocumentType2 { get; set; } //tag2 public DateOnly? ExpiryDate { get; set; } + public bool ToTransientBucket { get; set; } = false; } public record CreateStreamDocumentCmd : CreateDocumentCmd diff --git a/src/Spd.Resource.Applicants/Document/DocumentRepository.cs b/src/Spd.Resource.Applicants/Document/DocumentRepository.cs index 3900567a5..89fbb7a8c 100644 --- a/src/Spd.Resource.Applicants/Document/DocumentRepository.cs +++ b/src/Spd.Resource.Applicants/Document/DocumentRepository.cs @@ -11,17 +11,20 @@ internal class DocumentRepository : IDocumentRepository private readonly DynamicsContext _context; private readonly IMapper _mapper; private readonly IFileStorageService _fileStorageService; + private readonly ITransientFileStorageService _transientFileStorageService; private readonly ITempFileStorageService _tempFileService; public DocumentRepository(IDynamicsContextFactory ctx, IMapper mapper, IFileStorageService fileStorageService, - ITempFileStorageService tempFileService) + ITempFileStorageService tempFileService, + ITransientFileStorageService transientFileStorageService) { _context = ctx.Create(); _mapper = mapper; _fileStorageService = fileStorageService; _tempFileService = tempFileService; + _transientFileStorageService = transientFileStorageService; } public async Task QueryAsync(DocumentQry qry, CancellationToken ct) { @@ -106,7 +109,7 @@ private async Task DocumentCreateAsync(CreateDocumentCmd cmd, Canc _context.SetLink(documenturl, nameof(documenturl.spd_SubmittedById), contact); } - await UploadFileAsync(cmd.TempFile, application.spd_applicationid, documenturl.bcgov_documenturlid, null, ct); + await UploadFileAsync(cmd.TempFile, application.spd_applicationid, documenturl.bcgov_documenturlid, null, ct, cmd.ToTransientBucket); await _context.SaveChangesAsync(ct); documenturl._spd_applicationid_value = application.spd_applicationid; return _mapper.Map(documenturl); @@ -208,7 +211,7 @@ private async Task DocumentUpdateAsync(UpdateDocumentCmd cmd, Canc return _mapper.Map(documenturl); } - private async Task UploadFileAsync(SpdTempFile tempFile, Guid? applicationId, Guid? docUrlId, bcgov_tag? tag, CancellationToken ct) + private async Task UploadFileAsync(SpdTempFile tempFile, Guid? applicationId, Guid? docUrlId, bcgov_tag? tag, CancellationToken ct, bool toTransientBucket = false) { if (applicationId == null) return; if (docUrlId == null) return; @@ -236,12 +239,19 @@ private async Task UploadFileAsync(SpdTempFile tempFile, Guid? applicationId, Gu } }; - await _fileStorageService.HandleCommand(new UploadFileCommand( + UploadFileCommand uploadFileCmd = new UploadFileCommand( Key: ((Guid)docUrlId).ToString(), Folder: $"spd_application/{applicationId}", File: file, - FileTag: fileTag - ), ct); + FileTag: fileTag); + if (toTransientBucket) + { + await _transientFileStorageService.HandleCommand(uploadFileCmd, ct); + } + else + { + await _fileStorageService.HandleCommand(uploadFileCmd, ct); + } } else { @@ -262,31 +272,27 @@ await _fileStorageService.HandleCommand(new UploadFileCommand( } }; - await _fileStorageService.HandleCommand(new UploadFileStreamCommand( - Key: ((Guid)docUrlId).ToString(), - Folder: $"spd_application/{applicationId}", - FileStream: fileStream, - FileTag: fileTag - ), ct); + UploadFileStreamCommand uploadFileCmd = new UploadFileStreamCommand( + Key: ((Guid)docUrlId).ToString(), + Folder: $"spd_application/{applicationId}", + FileStream: fileStream, + FileTag: fileTag); + if (toTransientBucket) + { + await _transientFileStorageService.HandleCommand(uploadFileCmd, ct); + } + else + { + await _fileStorageService.HandleCommand(uploadFileCmd, ct); + } } } private async Task DeleteFileAsync(Guid docUrlId, Guid applicationId, CancellationToken ct) { - //set file-classification tag to be deleted - FileTag fileTag = - new FileTag() - { - Tags = new List - { - new Utilities.FileStorage.Tag("file-classification", "deleted"), - } - }; - await _fileStorageService.HandleCommand(new UpdateTagsCommand( + await _transientFileStorageService.HandleDeleteCommand(new StorageDeleteCommand( Key: ((Guid)docUrlId).ToString(), - Folder: $"spd_application/{applicationId}", - FileTag: fileTag - ), ct); + Folder: $"spd_application/{applicationId}"), ct); } diff --git a/src/Spd.Utilities.FileStorage/Contract.cs b/src/Spd.Utilities.FileStorage/Contract.cs index dbf27948d..de353c60d 100644 --- a/src/Spd.Utilities.FileStorage/Contract.cs +++ b/src/Spd.Utilities.FileStorage/Contract.cs @@ -10,6 +10,7 @@ public interface ITransientFileStorageService { Task HandleCommand(StorageCommand cmd, CancellationToken cancellationToken); Task HandleQuery(StorageQuery query, CancellationToken cancellationToken); + Task HandleDeleteCommand(StorageDeleteCommand cmd, CancellationToken cancellationToken); } public abstract record StorageCommand(string Key, string? Folder); @@ -21,6 +22,7 @@ public record UpdateTagsCommand(string Key, string? Folder, FileTag? FileTag) : //copy the file from source to dest for the same bucket. public record CopyFileCommand(string SourceKey, string? SourceFolder, string DestKey, string? DestFolder) : StorageCommand(SourceKey, SourceFolder); + public record StorageDeleteCommand(string Key, string? Folder); public record FileTag { public IEnumerable Tags { get; set; } = Array.Empty(); diff --git a/src/Spd.Utilities.FileStorage/FileStorageService.cs b/src/Spd.Utilities.FileStorage/FileStorageService.cs index 86e465041..357e6210b 100644 --- a/src/Spd.Utilities.FileStorage/FileStorageService.cs +++ b/src/Spd.Utilities.FileStorage/FileStorageService.cs @@ -37,6 +37,10 @@ public async Task HandleQuery(StorageQuery query, Cancellat }; } + public async Task HandleDeleteCommand(StorageDeleteCommand cmd, CancellationToken cancellationToken) + { + return await DeleteStorageItem(cmd, cancellationToken); + } private async Task UploadStorageItem(UploadFileCommand cmd, CancellationToken cancellationToken) { File file = cmd.File; @@ -65,6 +69,24 @@ private async Task UploadStorageItem(UploadFileCommand cmd, Cancellation return cmd.Key; } + private async Task DeleteStorageItem(StorageDeleteCommand cmd, CancellationToken cancellationToken) + { + var folder = cmd.Folder == null ? "" : $"{cmd.Folder}/"; + var key = $"{folder}{cmd.Key}"; + + var request = new DeleteObjectRequest + { + Key = key, + BucketName = _config.Value.Bucket, + }; + + var response = await _amazonS3Client.DeleteObjectAsync(request, cancellationToken); + + //If the delete action is successful, the service sends back an HTTP 204 response. + response.EnsureNoContent(); + return cmd.Key; + } + private async Task UploadStorageItemStream(UploadFileStreamCommand cmd, CancellationToken cancellationToken) { FileStream file = cmd.FileStream; @@ -247,5 +269,11 @@ public static void EnsureSuccess(this AmazonWebServiceResponse response) if (response.HttpStatusCode != System.Net.HttpStatusCode.OK) throw new InvalidOperationException($"Operation failed with status {response.HttpStatusCode}"); } + + public static void EnsureNoContent(this AmazonWebServiceResponse response) + { + if (response.HttpStatusCode != System.Net.HttpStatusCode.NoContent) + throw new InvalidOperationException($"Delete Operation failed with status {response.HttpStatusCode}"); + } } } diff --git a/src/Spd.Utilities.FileStorage/ServiceExtensions.cs b/src/Spd.Utilities.FileStorage/ServiceExtensions.cs index 6e46c96cf..2fb42ac59 100644 --- a/src/Spd.Utilities.FileStorage/ServiceExtensions.cs +++ b/src/Spd.Utilities.FileStorage/ServiceExtensions.cs @@ -30,7 +30,7 @@ public static IServiceCollection AddFileStorageProxy(this IServiceCollection ser Options.Create(options.MainBucketSettings))); //create transient bucket - if (options.TransientBucketSettings?.Url != null && string.IsNullOrWhiteSpace(options.TransientBucketSettings?.Url.ToString())) + if (options.TransientBucketSettings?.Url != null && !string.IsNullOrWhiteSpace(options.TransientBucketSettings?.Url.ToString())) { var transientBucketConfig = new AmazonS3Config {