Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

same bucket s3 copy #685

Merged
merged 5 commits into from
Jan 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/Spd.Manager.Licence/PersonalLicenceAppManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,23 @@ public async Task<WorkerLicenceAppUpsertResponse> Handle(AnonymousWorkerLicenceA
saveCmd.ApplicationStatusEnum = ApplicationStatusEnum.PaymentPending;
var response = await _licenceAppRepository.SaveLicenceApplicationAsync(saveCmd, ct);

//todo: add file copying here.
//add photo file copying here.
if (cmd.LicenceAnonymousRequest.OriginalApplicationId == null)
throw new ArgumentException("replacement request must have original application id");
var photos = await _documentRepository.QueryAsync(
new DocumentQry(
ApplicationId: cmd.LicenceAnonymousRequest.OriginalApplicationId,
FileType: DocumentTypeEnum.Photograph),
ct);
if (photos.Items.Any())
{
foreach (var photo in photos.Items)
{
await _documentRepository.ManageAsync(
new CopyDocumentCmd(photo.DocumentUrlId, response.LicenceAppId, response.ContactId),
ct);
}
}
return new WorkerLicenceAppUpsertResponse { LicenceAppId = response.LicenceAppId };
}
#endregion
Expand Down
61 changes: 26 additions & 35 deletions src/Spd.Presentation.Dynamics/Controllers/FileStorageController.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Spd.Presentation.Dynamics.Helper;
using Spd.Presentation.Dynamics.Models;
using Spd.Utilities.FileStorage;
using Spd.Utilities.Shared;
Expand Down Expand Up @@ -66,7 +67,7 @@ public async Task<IActionResult> UploadFileAsync(
};
FileTag fileTag = new()
{
Tags = GetTagsFromStr(tags, classification)
Tags = FileStorageHelper.GetTagsFromStr(tags, classification)
};
await _storageService.HandleCommand(
new UploadFileCommand(fileId.ToString(), folder, file, fileTag),
Expand Down Expand Up @@ -116,7 +117,7 @@ public async Task<FileStreamResult> DownloadFileAsync(
HttpContext.Response.Headers.Add(SpdHeaderNames.HEADER_FILE_CLASSIFICATION,
result.FileTag.Tags.SingleOrDefault(t => t.Key == SpdHeaderNames.HEADER_FILE_CLASSIFICATION)?.Value);

string tagStr = GetStrFromTags(result.FileTag.Tags);
string tagStr = FileStorageHelper.GetStrFromTags(result.FileTag.Tags);
if (!string.IsNullOrWhiteSpace(tagStr))
HttpContext.Response.Headers.Add(SpdHeaderNames.HEADER_FILE_TAG, tagStr);
}
Expand Down Expand Up @@ -164,7 +165,7 @@ public async Task<IActionResult> UpdateTagsAsync(
{
FileTag fileTag = new()
{
Tags = GetTagsFromStr(tags, classification),
Tags = FileStorageHelper.GetTagsFromStr(tags, classification),
};
await _storageService.HandleCommand(
new UpdateTagsCommand(fileId.ToString(), folder, fileTag),
Expand All @@ -173,40 +174,30 @@ await _storageService.HandleCommand(
}
}

private Tag[] GetTagsFromStr(string? tagStr, string classification)
/// <summary>
/// copy file for one location in main bucket to another location in the same main bucket
/// </summary>
/// <param name="copyFileRequest"></param>
/// <param name="ct"></param>
/// <returns></returns>
[HttpPost]
[Route("api/files/copy-file")]
public async Task<IActionResult> CopyFileAsync(
[FromBody] CopyFileRequest copyFileRequest,
CancellationToken ct)
{
try
{
List<Tag> taglist = new() { new Tag(SpdHeaderNames.HEADER_FILE_CLASSIFICATION, classification) };
//check if file already exists
FileMetadataQueryResult queryResult = (FileMetadataQueryResult)await _storageService.HandleQuery(
new FileMetadataQuery { Key = copyFileRequest.SourceKey },
ct);
bool fileExists = queryResult != null;

if (!string.IsNullOrWhiteSpace(tagStr))
{
string[] tags = tagStr.Split(',');
foreach (string tag in tags)
{
string[] strs = tag.Split('=');
if (strs.Length != 2) throw new OutOfRangeException(HttpStatusCode.BadRequest, $"Invalid {SpdHeaderNames.HEADER_FILE_TAG} string");
taglist.Add(
new Tag(strs[0], strs[1])
);
}
}
return taglist.ToArray();
}
catch
{
throw new ApiException(HttpStatusCode.BadRequest, $"Invalid {SpdHeaderNames.HEADER_FILE_TAG} string");
}
}
if (!fileExists) { return NotFound(); }

private string GetStrFromTags(IEnumerable<Tag> tags)
{
List<string> tagStrlist = new();
foreach (Tag t in tags)
{
if (t.Key != SpdHeaderNames.HEADER_FILE_CLASSIFICATION)
tagStrlist.Add($"{t.Key}={t.Value}");
}
return string.Join(",", tagStrlist);
await _storageService.HandleCommand(
new CopyFileCommand(copyFileRequest.SourceKey, null, copyFileRequest.DestKey, null),
ct);

return StatusCode(StatusCodes.Status201Created);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Spd.Presentation.Dynamics.Helper;
using Spd.Presentation.Dynamics.Models;
using Spd.Utilities.FileStorage;
using Spd.Utilities.Shared;
using Spd.Utilities.Shared.Exceptions;
using System.ComponentModel.DataAnnotations;
using System.Net;
using File = Spd.Utilities.FileStorage.File;

namespace Spd.Presentation.Dynamics.Controllers;

/// <summary>
/// For upload and download file
/// </summary>
[Authorize]
public class TransientFileStorageController : SpdControllerBase
{
private readonly ITransientFileStorageService _storageService;
public TransientFileStorageController(ITransientFileStorageService storageService) : base()
{
_storageService = storageService;
}

/// <summary>
/// Upload or overwrite file to transient bucket
/// If the file guid exists, the file content is overwritten by the new file
/// all the tags must be sent in the requests, tags that do not exist will be removed from the file storage
/// The maximum file size would be 30M.
/// </summary>
/// <param name="request">File with selected file data.</param>
/// <param name="fileId">the GUID of the file</param>
/// <param name="classification">mandatory, must contain only alphanumeric characters</param>
/// <param name="tags">must be in the form key=value, key must contain only alphanumeric characters. Multi tags should be like: tag1=a,tag2=b</param>
/// <param name="folder">be used to allocate the file in a specific folder in storage, if not specified, defaults to / (root folder)</param>
/// <param name="ct">cancellation token, generated by dotnetcore</param>
/// <response code="200">Ok, the file was updated</response>
/// <response code="201">Created, the file was created</response>
/// <response code="400">Bad Request, classification header is missing,form data is invalid or file id is not a valid guid</response>
/// <returns>
/// </returns>
[HttpPost]
[Route("api/transient-files/{fileId}")]
public async Task<IActionResult> UploadFileAsync(
[FromForm] UploadFileRequest request,
[FromRoute] Guid fileId,
[FromHeader(Name = "file-classification")][Required] string classification,
[FromHeader(Name = "file-tag")] string? tags,
[FromHeader(Name = "file-folder")] string? folder,
CancellationToken ct)
{
//check if file already exists
FileMetadataQueryResult queryResult = (FileMetadataQueryResult)await _storageService.HandleQuery(
new FileMetadataQuery { Key = fileId.ToString(), Folder = folder },
ct);
bool fileExists = queryResult != null;

//upload file
using var ms = new MemoryStream();
await request.File.CopyToAsync(ms, ct);
File file = new()
{
FileName = request.File.FileName,
ContentType = request.File.ContentType,
Content = ms.ToArray()
};
FileTag fileTag = new()
{
Tags = FileStorageHelper.GetTagsFromStr(tags, classification)
};
await _storageService.HandleCommand(
new UploadFileCommand(fileId.ToString(), folder, file, fileTag),
ct);

return fileExists ? Ok() : StatusCode(StatusCodes.Status201Created);
}

/// <summary>
/// Download the file with fileId and folder name from to transient bucket.
/// If a file is expected to be in a folder, the client must pass the correct folder name in the request header,
/// otherwise no file will found; the default header value is the root folder
/// </summary>
/// <param name="fileId">the GUID of the file</param>
/// <param name="folder">be used to allocate the file in a specific folder in storage, if not specified, defaults to / (root folder)</param>
/// <param name="ct">cancellation token, generated by dotnetcore</param>
/// <returns></returns>
/// <response code="200">Ok</response>
/// <response code="404">Not Found</response>
/// <response code="400">Bad Request, file id is not a valid guid</response>
[HttpGet]
[Route("api/transient-files/{fileId}")]
public async Task<FileStreamResult> DownloadFileAsync(
Guid fileId,
[FromHeader(Name = "file-folder")] string? folder,
CancellationToken ct)
{
var queryResult = (FileMetadataQueryResult)await _storageService.HandleQuery(
new FileMetadataQuery { Key = fileId.ToString(), Folder = folder },
ct);

bool fileExists = queryResult != null;
if (!fileExists)
{
throw new ApiException(HttpStatusCode.NotFound, "cannot find the file");
}

FileQueryResult result = (FileQueryResult)await _storageService.HandleQuery(
new FileQuery { Key = fileId.ToString(), Folder = folder },
ct);

var content = new MemoryStream(result.File.Content);
var contentType = result.File.ContentType ?? "application/octet-stream";

if (result.FileTag != null)
{
HttpContext.Response.Headers.Add(SpdHeaderNames.HEADER_FILE_CLASSIFICATION,
result.FileTag.Tags.SingleOrDefault(t => t.Key == SpdHeaderNames.HEADER_FILE_CLASSIFICATION)?.Value);

string tagStr = FileStorageHelper.GetStrFromTags(result.FileTag.Tags);
if (!string.IsNullOrWhiteSpace(tagStr))
HttpContext.Response.Headers.Add(SpdHeaderNames.HEADER_FILE_TAG, tagStr);
}

if (!string.IsNullOrWhiteSpace(folder))
HttpContext.Response.Headers.Add(SpdHeaderNames.HEADER_FILE_FOLDER, folder);

return new FileStreamResult(content, contentType);
}

/// <summary>
/// To transient bucket, Only updates the tags passed in the header and will not affect the content of the file
/// any tags not present in the request will be deleted
/// tags must be in the form key=value, key must contain only alphanumeric characters(i.e.tag1= value1)
/// classification must contain only alphanumeric characters(i.e.confidential/internal/public)
/// </summary>
/// <param name="fileId">the GUID of the file</param>
/// <param name="classification">mandatory, must contain only alphanumeric characters</param>
/// <param name="tags">multi tags should be like: tag1=a,tag2=b</param>
/// <param name="folder">be used to allocate the file in a specific folder in storage, if not specified, defaults to / (root folder)</param>
/// <param name="ct"></param>
/// <returns></returns>
/// <response code="200">Ok</response>
/// <response code="404">Not Found</response>
/// <response code="400">Bad Request, file id is not a valid guid</response>
[HttpPost]
[Route("api/transient-files/{fileId}/tags")]
public async Task<IActionResult> UpdateTagsAsync(
[FromRoute] Guid fileId,
[FromHeader(Name = "file-classification")][Required] string classification,
[FromHeader(Name = "file-tag")] string? tags,
[FromHeader(Name = "file-folder")] string? folder,
CancellationToken ct)
{
//check if file already exists
var queryResult = (FileMetadataQueryResult)await _storageService.HandleQuery(
new FileMetadataQuery { Key = fileId.ToString(), Folder = folder },
ct);
bool fileExists = queryResult != null;
if (!fileExists)
{
return NotFound();
}
else
{
FileTag fileTag = new()
{
Tags = FileStorageHelper.GetTagsFromStr(tags, classification),
};
await _storageService.HandleCommand(
new UpdateTagsCommand(fileId.ToString(), folder, fileTag),
ct);
return Ok();
}
}

/// <summary>
/// copy file for one location in transient bucket to another location in the same transient bucket
/// </summary>
/// <param name="copyFileRequest"></param>
/// <param name="ct"></param>
/// <returns></returns>
[HttpPost]
[Route("api/transient-files/copy-file")]
public async Task<IActionResult> CopyFileAsync(
[FromBody] CopyFileRequest copyFileRequest,
CancellationToken ct)
{
//check if file already exists
FileMetadataQueryResult queryResult = (FileMetadataQueryResult)await _storageService.HandleQuery(
new FileMetadataQuery { Key = copyFileRequest.SourceKey },
ct);
bool fileExists = queryResult != null;

if (!fileExists) { return NotFound(); }

await _storageService.HandleCommand(
new CopyFileCommand(copyFileRequest.SourceKey, null, copyFileRequest.DestKey, null),
ct);

return StatusCode(StatusCodes.Status201Created);
}


}
45 changes: 45 additions & 0 deletions src/Spd.Presentation.Dynamics/Helper/FileStorageHelper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
using Spd.Utilities.FileStorage;
using Spd.Utilities.Shared.Exceptions;
using System.Net;

namespace Spd.Presentation.Dynamics.Helper;

public class FileStorageHelper
{
public static Tag[] GetTagsFromStr(string? tagStr, string classification)
{
try
{
List<Tag> taglist = new() { new Tag(SpdHeaderNames.HEADER_FILE_CLASSIFICATION, classification) };

if (!string.IsNullOrWhiteSpace(tagStr))
{
string[] tags = tagStr.Split(',');
foreach (string tag in tags)
{
string[] strs = tag.Split('=');
if (strs.Length != 2) throw new OutOfRangeException(HttpStatusCode.BadRequest, $"Invalid {SpdHeaderNames.HEADER_FILE_TAG} string");
taglist.Add(
new Tag(strs[0], strs[1])
);
}
}
return taglist.ToArray();
}
catch
{
throw new ApiException(HttpStatusCode.BadRequest, $"Invalid {SpdHeaderNames.HEADER_FILE_TAG} string");
}
}

public static string GetStrFromTags(IEnumerable<Tag> tags)
{
List<string> tagStrlist = new();
foreach (Tag t in tags)
{
if (t.Key != SpdHeaderNames.HEADER_FILE_CLASSIFICATION)
tagStrlist.Add($"{t.Key}={t.Value}");
}
return string.Join(",", tagStrlist);
}
}
8 changes: 8 additions & 0 deletions src/Spd.Presentation.Dynamics/Models/CopyFileRequest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
namespace Spd.Presentation.Dynamics.Models
{
public class CopyFileRequest
{
public string SourceKey { get; set; }
public string DestKey { get; set; }
}
}
5 changes: 0 additions & 5 deletions src/Spd.Presentation.Dynamics/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,6 @@

var builder = WebApplication.CreateBuilder(args);

//todo: remove,just testing
ILogger logger = builder.Services.BuildServiceProvider().GetRequiredService<ILogger<Program>>();
logger.LogInformation("This is a testlog");
//end

var assemblies = Directory.GetFiles(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? string.Empty, "*.dll", SearchOption.TopDirectoryOnly)
.Where(assembly =>
{
Expand Down
Loading
Loading