Skip to content

Commit

Permalink
Changed cfg property to be better understandable and file size is con…
Browse files Browse the repository at this point in the history
…verted into constant.
  • Loading branch information
milanmajchrak committed Feb 20, 2024
1 parent 39e4697 commit b467443
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ public class SyncS3BitStoreService extends S3BitStoreService {
private static final Logger log = LogManager.getLogger(SyncS3BitStoreService.class);
private boolean syncEnabled = false;

/**
* The uploading file is divided into parts and each part is uploaded separately. The size of the part is 50 MB.
*/
private static final long UPLOAD_FILE_PART_SIZE = 50 * 1024 * 1024; // 50 MB

/**
* Upload large file by parts - check the checksum of every part
*/
Expand Down Expand Up @@ -84,7 +89,7 @@ public void init() throws IOException {
syncEnabled = configurationService.getBooleanProperty("sync.storage.service.enabled", false);
}
if (!uploadByParts) {
uploadByParts = configurationService.getBooleanProperty("upload.by.parts.enabled", false);
uploadByParts = configurationService.getBooleanProperty("s3.upload.by.parts.enabled", false);
}
}

Expand Down Expand Up @@ -209,8 +214,6 @@ private void uploadByParts(String key, File scratchFile) throws IOException {
// Initiate multipart upload
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(getBucketName(), key);
String uploadId = this.s3Service.initiateMultipartUpload(initiateRequest).getUploadId();
// Upload a file in multiple parts, one part has 50MB
long partSize = 50 * 1024 * 1024; // 50 MB

// Create a list to hold the ETags for individual parts
List<PartETag> partETags = new ArrayList<>();
Expand All @@ -223,7 +226,7 @@ private void uploadByParts(String key, File scratchFile) throws IOException {
int partNumber = 1;

while (remainingBytes > 0) {
long bytesToUpload = Math.min(partSize, remainingBytes);
long bytesToUpload = Math.min(UPLOAD_FILE_PART_SIZE, remainingBytes);

// Calculate the checksum for the part
String partChecksum = calculatePartChecksum(file, fileLength - remainingBytes, bytesToUpload, digest);
Expand Down
2 changes: 1 addition & 1 deletion dspace/config/clarin-dspace.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ file.preview.enabled = false
# Synchronization is NOT enabled by default
sync.storage.service.enabled = true
# Upload large file by parts - check the checksum of every part
upload.by.parts.enabled = true
s3.upload.by.parts.enabled = true


### The build version is stored in the specific file ###
Expand Down

0 comments on commit b467443

Please sign in to comment.