Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

EVA-3227 Match metadata files info with uploaded files #15

Merged
merged 3 commits into from
Jul 16, 2024
Merged

EVA-3227 Match metadata files info with uploaded files #15

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e1672ae
match metadata files with uploaded files
nitin-ebi Jul 15, 2024
f2109d5
update file info checker
nitin-ebi Jul 15, 2024
6e36bcf
review comments
nitin-ebi Jul 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/main/java/uk/ac/ebi/eva/submission/config/AppConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import org.springframework.context.annotation.Configuration;
import org.springframework.mail.javamail.JavaMailSender;
import org.springframework.mail.javamail.JavaMailSenderImpl;
import org.springframework.web.client.RestTemplate;

@Configuration
public class AppConfig {
Expand All @@ -15,4 +16,9 @@ public JavaMailSender javaMailService() {
javaMailSender.setPort(25);
return javaMailSender;
}

@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
}
4 changes: 4 additions & 0 deletions src/main/java/uk/ac/ebi/eva/submission/controller/submissionws/SubmissionController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import uk.ac.ebi.eva.submission.controller.BaseController;
import uk.ac.ebi.eva.submission.entity.Submission;
import uk.ac.ebi.eva.submission.entity.SubmissionAccount;
import uk.ac.ebi.eva.submission.exception.MetadataFileInfoMismatchException;
import uk.ac.ebi.eva.submission.exception.RequiredFieldsMissingException;
import uk.ac.ebi.eva.submission.exception.SubmissionDoesNotExistException;
import uk.ac.ebi.eva.submission.model.SubmissionStatus;
Expand Down Expand Up @@ -93,11 +94,14 @@ public ResponseEntity<?> markSubmissionUploaded(@RequestHeader("Authorization")
}

try {
submissionService.checkMetadataFileInfoMatchesWithUploadedFiles(submissionAccount, submissionId, metadataJson);
Submission submission = this.submissionService.uploadMetadataJsonAndMarkUploaded(submissionId, metadataJson);
submissionService.sendMailNotificationForStatusUpdate(submissionAccount, submissionId, SubmissionStatus.UPLOADED, true);
return new ResponseEntity<>(stripUserDetails(submission), HttpStatus.OK);
} catch (RequiredFieldsMissingException ex) {
return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST);
} catch (MetadataFileInfoMismatchException ex) {
return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST);
}
}

Expand Down
7 changes: 7 additions & 0 deletions src/main/java/uk/ac/ebi/eva/submission/exception/MetadataFileInfoMismatchException.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package uk.ac.ebi.eva.submission.exception;

public class MetadataFileInfoMismatchException extends RuntimeException {
public MetadataFileInfoMismatchException(String message) {
super(message);
}
}
48 changes: 34 additions & 14 deletions src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,37 +16,33 @@ public class GlobusDirectoryProvisioner {

private final GlobusTokenRefreshService globusTokenRefreshService;

private final RestTemplate restTemplate;

@Value("${globus.submission.endpointId}")
private String endpointId;

private static final String GLOBUS_TRANSFER_API_BASE_URL =
"https://transfer.api.globusonline.org/v0.10/operation/endpoint";


public GlobusDirectoryProvisioner(GlobusTokenRefreshService globusTokenRefreshService) {
public GlobusDirectoryProvisioner(GlobusTokenRefreshService globusTokenRefreshService, RestTemplate restTemplate) {
this.globusTokenRefreshService = globusTokenRefreshService;
this.restTemplate = restTemplate;
}

public void createSubmissionDirectory(String directoryToCreate) {
String fileSeparator = System.getProperty("file.separator");
String[] directoriesToCreate = directoryToCreate.split(Pattern.quote(fileSeparator));
StringBuilder directoryPathSoFar = new StringBuilder();
for (String directory: directoriesToCreate) {
for (String directory : directoriesToCreate) {
directoryPathSoFar.append(directory);
createDirectory(directoryPathSoFar.toString());
directoryPathSoFar.append(fileSeparator);
}
}

private void createDirectory(String directoryToCreate) {

String accessToken = globusTokenRefreshService.getAccessToken();
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.set("Authorization", "Bearer " + accessToken);
headers.set("Accept", "application/json");
headers.set("Content-Type", "application/json");

HttpHeaders headers = getGlobusAccessHeaders();

if (this.alreadyExists(headers, directoryToCreate)) {
return;
Expand All @@ -69,17 +65,41 @@ private void createDirectory(String directoryToCreate) {
}
}

private boolean alreadyExists(HttpHeaders headers, String directoryName) {
RestTemplate restTemplate = new RestTemplate();
private boolean alreadyExists(HttpHeaders headers, String directoryName) {
HttpEntity<String> entity = new HttpEntity<>("", headers);
try {
restTemplate.exchange(
GLOBUS_TRANSFER_API_BASE_URL + "/" + endpointId + "/ls?path=" + directoryName,
HttpMethod.GET, entity, String.class);
return true;
}
catch (HttpClientErrorException ex) {
} catch (HttpClientErrorException ex) {
return false;
}
}

public String listSubmittedFiles(String submissionDirPath) {
HttpEntity<String> requestEntity = new HttpEntity<>(getGlobusAccessHeaders());
String transferApiUrl = String.format("%s/%s/ls?path=%s", GLOBUS_TRANSFER_API_BASE_URL, endpointId, submissionDirPath);
ResponseEntity<String> response = restTemplate.exchange(transferApiUrl, HttpMethod.GET, requestEntity, String.class);

if (response.getStatusCode().is2xxSuccessful()) {
System.out.printf("Directory %s listed successfully%n", submissionDirPath);
return response.getBody();
} else {
System.out.printf("Failed to retrieve directory '%s': %s", submissionDirPath, response.getStatusCode());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we not log rather than print out ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

apologies copy paste mistake.. updated

return "";
}
}

private HttpHeaders getGlobusAccessHeaders() {
String accessToken = globusTokenRefreshService.getAccessToken();
HttpHeaders headers = new HttpHeaders();
headers.set("Authorization", "Bearer " + accessToken);
headers.set("Accept", "application/json");
headers.set("Content-Type", "application/json");

return headers;
}


}
67 changes: 67 additions & 0 deletions src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
package uk.ac.ebi.eva.submission.service;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import uk.ac.ebi.eva.submission.entity.Submission;
import uk.ac.ebi.eva.submission.entity.SubmissionAccount;
import uk.ac.ebi.eva.submission.entity.SubmissionDetails;
import uk.ac.ebi.eva.submission.entity.SubmissionProcessing;
import uk.ac.ebi.eva.submission.exception.MetadataFileInfoMismatchException;
import uk.ac.ebi.eva.submission.exception.RequiredFieldsMissingException;
import uk.ac.ebi.eva.submission.exception.SubmissionDoesNotExistException;
import uk.ac.ebi.eva.submission.model.SubmissionProcessingStatus;
Expand All @@ -20,15 +24,26 @@
import uk.ac.ebi.eva.submission.util.MailSender;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

@Service
public class SubmissionService {
private static final String PROJECT = "project";
private static final String TITLE = "title";
private static final String DESCRIPTION = "description";
private static final String METADATA_FILES_TAG = "files";
private static final String METADATA_FILE_NAME = "fileName";
private static final String METADATA_FILE_SIZE = "fileSize";
private static final String GLOBUS_FILES_TAG = "DATA";
private static final String GLOBUS_FILE_NAME = "name";
private static final String GLOBUS_FILE_SIZE = "size";

private final SubmissionRepository submissionRepository;

Expand Down Expand Up @@ -82,6 +97,58 @@ public Submission initiateSubmission(SubmissionAccount submissionAccount) {
return submissionRepository.save(submission);
}

public void checkMetadataFileInfoMatchesWithUploadedFiles(SubmissionAccount submissionAccount, String submissionId, JsonNode metadataJson) {
String directoryToList = String.format("%s/%s", submissionAccount.getId(), submissionId);
String uploadedFilesInfo = globusDirectoryProvisioner.listSubmittedFiles(directoryToList);
if (uploadedFilesInfo.isEmpty()) {
throw new MetadataFileInfoMismatchException("Failed to retrieve any file info from submission directory.");
} else {
try {
ObjectMapper mapper = new ObjectMapper();
ObjectNode globusFileInfoJson = (ObjectNode) mapper.readTree(uploadedFilesInfo);
Map<String, Long> globusFileInfo = new HashMap<>();
if (globusFileInfoJson.get(GLOBUS_FILES_TAG) != null) {
globusFileInfo = StreamSupport.stream(globusFileInfoJson.get(GLOBUS_FILES_TAG).spliterator(), false)
.filter(dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText().endsWith(".vcf") || dataNode.get(GLOBUS_FILE_NAME).asText().endsWith(".vcf.gz"))
.collect(Collectors.toMap(
dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText(),
dataNode -> dataNode.get(GLOBUS_FILE_SIZE).asLong()
));
}

Map<String, Long> metadataFileInfo = StreamSupport.stream(metadataJson.get(METADATA_FILES_TAG).spliterator(), false)
.collect(Collectors.toMap(
dataNode -> dataNode.get(METADATA_FILE_NAME).asText(),
dataNode -> dataNode.get(METADATA_FILE_SIZE).asLong()
));

List<String> missingFileList = new ArrayList<>();
String fileSizeMismatchInfo = "";

for (Map.Entry<String, Long> fileEntry : metadataFileInfo.entrySet()) {
String fileName = fileEntry.getKey();
Long metadataFileSize = fileEntry.getValue();
if (globusFileInfo.containsKey(fileName)) {
Long fileSizeInGlobus = globusFileInfo.get(fileName);
if (!metadataFileSize.equals(fileSizeInGlobus)) {
fileSizeMismatchInfo += fileName + ": metadata json file size (" + metadataFileSize + ") is not equal to uploaded file size (" + fileSizeInGlobus + ")\n";
}
} else {
missingFileList.add(fileName);
}
}

if (!missingFileList.isEmpty() || !fileSizeMismatchInfo.isEmpty()) {
String missingFileMsg = missingFileList.isEmpty() ? "" : "There are some files mentioned in metadata json but not uploaded. Files : " + String.join(", ", missingFileList) + "\n";
String fileSizeMismatchMsg = fileSizeMismatchInfo.isEmpty() ? "" : "There are some files mentioned in metadata json whose size does not match with the files uploaded.\n" + fileSizeMismatchInfo;
throw new MetadataFileInfoMismatchException(missingFileMsg + fileSizeMismatchMsg);
}
} catch (JsonProcessingException ex) {
throw new MetadataFileInfoMismatchException("Error parsing fileInfo from Submission Directory");
}
}
}

public Submission uploadMetadataJsonAndMarkUploaded(String submissionId, JsonNode metadataJson) {
SubmissionDetails submissionDetails = new SubmissionDetails(submissionId);
try {
Expand Down
Loading
Loading