Skip to content

Commit

Permalink
match metadata files with uploaded files
Browse files Browse the repository at this point in the history
  • Loading branch information
@nitin-ebi
nitin-ebi committed Jul 15, 2024
1 parent 821e52d commit e1672ae
Show file tree
Hide file tree
Showing 6 changed files with 367 additions and 23 deletions.
6 changes: 6 additions & 0 deletions src/main/java/uk/ac/ebi/eva/submission/config/AppConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import org.springframework.context.annotation.Configuration;
import org.springframework.mail.javamail.JavaMailSender;
import org.springframework.mail.javamail.JavaMailSenderImpl;
import org.springframework.web.client.RestTemplate;

@Configuration
public class AppConfig {
Expand All @@ -15,4 +16,9 @@ public JavaMailSender javaMailService() {
javaMailSender.setPort(25);
return javaMailSender;
}

@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
}
4 changes: 4 additions & 0 deletions src/main/java/uk/ac/ebi/eva/submission/controller/submissionws/SubmissionController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import uk.ac.ebi.eva.submission.controller.BaseController;
import uk.ac.ebi.eva.submission.entity.Submission;
import uk.ac.ebi.eva.submission.entity.SubmissionAccount;
import uk.ac.ebi.eva.submission.exception.MetadataFileInfoMismatchException;
import uk.ac.ebi.eva.submission.exception.RequiredFieldsMissingException;
import uk.ac.ebi.eva.submission.exception.SubmissionDoesNotExistException;
import uk.ac.ebi.eva.submission.model.SubmissionStatus;
Expand Down Expand Up @@ -93,11 +94,14 @@ public ResponseEntity<?> markSubmissionUploaded(@RequestHeader("Authorization")
}

try {
submissionService.checkMetadataFileInfoMatchesWithUploadedFiles(submissionAccount, submissionId, metadataJson);
Submission submission = this.submissionService.uploadMetadataJsonAndMarkUploaded(submissionId, metadataJson);
submissionService.sendMailNotificationForStatusUpdate(submissionAccount, submissionId, SubmissionStatus.UPLOADED, true);
return new ResponseEntity<>(stripUserDetails(submission), HttpStatus.OK);
} catch (RequiredFieldsMissingException ex) {
return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST);
} catch (MetadataFileInfoMismatchException ex) {
return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST);
}
}

Expand Down
7 changes: 7 additions & 0 deletions src/main/java/uk/ac/ebi/eva/submission/exception/MetadataFileInfoMismatchException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package uk.ac.ebi.eva.submission.exception;

public class MetadataFileInfoMismatchException extends RuntimeException {
public MetadataFileInfoMismatchException(String message) {
super(message);
}
}
48 changes: 34 additions & 14 deletions src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,37 +16,33 @@ public class GlobusDirectoryProvisioner {

private final GlobusTokenRefreshService globusTokenRefreshService;

private final RestTemplate restTemplate;

@Value("${globus.submission.endpointId}")
private String endpointId;

private static final String GLOBUS_TRANSFER_API_BASE_URL =
"https://transfer.api.globusonline.org/v0.10/operation/endpoint";


public GlobusDirectoryProvisioner(GlobusTokenRefreshService globusTokenRefreshService) {
public GlobusDirectoryProvisioner(GlobusTokenRefreshService globusTokenRefreshService, RestTemplate restTemplate) {
this.globusTokenRefreshService = globusTokenRefreshService;
this.restTemplate = restTemplate;
}

public void createSubmissionDirectory(String directoryToCreate) {
String fileSeparator = System.getProperty("file.separator");
String[] directoriesToCreate = directoryToCreate.split(Pattern.quote(fileSeparator));
StringBuilder directoryPathSoFar = new StringBuilder();
for (String directory: directoriesToCreate) {
for (String directory : directoriesToCreate) {
directoryPathSoFar.append(directory);
createDirectory(directoryPathSoFar.toString());
directoryPathSoFar.append(fileSeparator);
}
}

private void createDirectory(String directoryToCreate) {

String accessToken = globusTokenRefreshService.getAccessToken();
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.set("Authorization", "Bearer " + accessToken);
headers.set("Accept", "application/json");
headers.set("Content-Type", "application/json");

HttpHeaders headers = getGlobusAccessHeaders();

if (this.alreadyExists(headers, directoryToCreate)) {
return;
Expand All @@ -69,17 +65,41 @@ private void createDirectory(String directoryToCreate) {
}
}

private boolean alreadyExists(HttpHeaders headers, String directoryName) {
RestTemplate restTemplate = new RestTemplate();
private boolean alreadyExists(HttpHeaders headers, String directoryName) {
HttpEntity<String> entity = new HttpEntity<>("", headers);
try {
restTemplate.exchange(
GLOBUS_TRANSFER_API_BASE_URL + "/" + endpointId + "/ls?path=" + directoryName,
HttpMethod.GET, entity, String.class);
return true;
}
catch (HttpClientErrorException ex) {
} catch (HttpClientErrorException ex) {
return false;
}
}

public String listSubmittedFiles(String submissionDirPath) {
HttpEntity<String> requestEntity = new HttpEntity<>(getGlobusAccessHeaders());
String transferApiUrl = String.format("%s/%s/ls?path=%s", GLOBUS_TRANSFER_API_BASE_URL, endpointId, submissionDirPath);
ResponseEntity<String> response = restTemplate.exchange(transferApiUrl, HttpMethod.GET, requestEntity, String.class);

if (response.getStatusCode().is2xxSuccessful()) {
System.out.printf("Directory %s listed successfully%n", submissionDirPath);
return response.getBody();
} else {
System.out.printf("Failed to retrieve directory '%s': %s", submissionDirPath, response.getStatusCode());
return "";
}
}

private HttpHeaders getGlobusAccessHeaders() {
String accessToken = globusTokenRefreshService.getAccessToken();
HttpHeaders headers = new HttpHeaders();
headers.set("Authorization", "Bearer " + accessToken);
headers.set("Accept", "application/json");
headers.set("Content-Type", "application/json");

return headers;
}


}
67 changes: 67 additions & 0 deletions src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
package uk.ac.ebi.eva.submission.service;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import uk.ac.ebi.eva.submission.entity.Submission;
import uk.ac.ebi.eva.submission.entity.SubmissionAccount;
import uk.ac.ebi.eva.submission.entity.SubmissionDetails;
import uk.ac.ebi.eva.submission.entity.SubmissionProcessing;
import uk.ac.ebi.eva.submission.exception.MetadataFileInfoMismatchException;
import uk.ac.ebi.eva.submission.exception.RequiredFieldsMissingException;
import uk.ac.ebi.eva.submission.exception.SubmissionDoesNotExistException;
import uk.ac.ebi.eva.submission.model.SubmissionProcessingStatus;
Expand All @@ -20,15 +24,26 @@
import uk.ac.ebi.eva.submission.util.MailSender;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

@Service
public class SubmissionService {
private static final String PROJECT = "project";
private static final String TITLE = "title";
private static final String DESCRIPTION = "description";
private static final String METADATA_FILES_TAG = "files";
private static final String METADATA_FILE_NAME = "fileName";
private static final String METADATA_FILE_SIZE = "fileSize";
private static final String GLOBUS_FILES_TAG = "DATA";
private static final String GLOBUS_FILE_NAME = "name";
private static final String GLOBUS_FILE_SIZE = "size";

private final SubmissionRepository submissionRepository;

Expand Down Expand Up @@ -82,6 +97,58 @@ public Submission initiateSubmission(SubmissionAccount submissionAccount) {
return submissionRepository.save(submission);
}

public void checkMetadataFileInfoMatchesWithUploadedFiles(SubmissionAccount submissionAccount, String submissionId, JsonNode metadataJson) {
String directoryToList = String.format("%s/%s", submissionAccount.getId(), submissionId);
String uploadedFilesInfo = globusDirectoryProvisioner.listSubmittedFiles(directoryToList);
if (uploadedFilesInfo.isEmpty()) {
throw new MetadataFileInfoMismatchException("Failed to retrieve any file info from submission directory.");
} else {
try {
ObjectMapper mapper = new ObjectMapper();
ObjectNode globusFileInfoJson = (ObjectNode) mapper.readTree(uploadedFilesInfo);
Map<String, Long> globusFileInfo = new HashMap<>();
if (globusFileInfoJson.get(GLOBUS_FILES_TAG) != null) {
globusFileInfo = StreamSupport.stream(globusFileInfoJson.get(GLOBUS_FILES_TAG).spliterator(), false)
.filter(dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText().endsWith(".vcf") || dataNode.get(GLOBUS_FILE_NAME).asText().endsWith(".vcf.gz"))
.collect(Collectors.toMap(
dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText(),
dataNode -> dataNode.get(GLOBUS_FILE_SIZE).asLong()
));
}

Map<String, Long> metadataFileInfo = StreamSupport.stream(metadataJson.get(METADATA_FILES_TAG).spliterator(), false)
.collect(Collectors.toMap(
dataNode -> dataNode.get(METADATA_FILE_NAME).asText(),
dataNode -> dataNode.get(METADATA_FILE_SIZE).asLong()
));

List<String> missingFileList = new ArrayList<>();
String fileSizeMismatchInfo = "";

for (Map.Entry<String, Long> fileEntry : metadataFileInfo.entrySet()) {
String fileName = fileEntry.getKey();
Long metadataFileSize = fileEntry.getValue();
if (globusFileInfo.containsKey(fileName)) {
Long fileSizeInGlobus = globusFileInfo.get(fileName);
if (!metadataFileSize.equals(fileSizeInGlobus)) {
fileSizeMismatchInfo += fileName + ": metadata json file size (" + metadataFileSize + ") is not equal to uploaded file size (" + fileSizeInGlobus + ")\n";
}
} else {
missingFileList.add(fileName);
}
}

if (!missingFileList.isEmpty() || !fileSizeMismatchInfo.isEmpty()) {
String missingFileMsg = missingFileList.isEmpty() ? "" : "There are some files mentioned in metadata json but not uploaded. Files : " + String.join(", ", missingFileList) + "\n";
String fileSizeMismatchMsg = fileSizeMismatchInfo.isEmpty() ? "" : "There are some files mentioned in metadata json whose size does not match with the files uploaded.\n" + fileSizeMismatchInfo;
throw new MetadataFileInfoMismatchException(missingFileMsg + fileSizeMismatchMsg);
}
} catch (JsonProcessingException ex) {
throw new MetadataFileInfoMismatchException("Error parsing fileInfo from Submission Directory");
}
}
}

public Submission uploadMetadataJsonAndMarkUploaded(String submissionId, JsonNode metadataJson) {
SubmissionDetails submissionDetails = new SubmissionDetails(submissionId);
try {
Expand Down
Loading

0 comments on commit e1672ae

Please sign in to comment.