From e1672aefb490205153194a507b01a82c134d039a Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Mon, 15 Jul 2024 02:30:53 +0100 Subject: [PATCH 1/3] match metadata files with uploaded files --- .../ebi/eva/submission/config/AppConfig.java | 6 + .../submissionws/SubmissionController.java | 4 + .../MetadataFileInfoMismatchException.java | 7 + .../service/GlobusDirectoryProvisioner.java | 48 +++- .../submission/service/SubmissionService.java | 67 +++++ .../SubmissionWSIntegrationTest.java | 258 +++++++++++++++++- 6 files changed, 367 insertions(+), 23 deletions(-) create mode 100644 src/main/java/uk/ac/ebi/eva/submission/exception/MetadataFileInfoMismatchException.java diff --git a/src/main/java/uk/ac/ebi/eva/submission/config/AppConfig.java b/src/main/java/uk/ac/ebi/eva/submission/config/AppConfig.java index 74514c4..88db069 100644 --- a/src/main/java/uk/ac/ebi/eva/submission/config/AppConfig.java +++ b/src/main/java/uk/ac/ebi/eva/submission/config/AppConfig.java @@ -4,6 +4,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.mail.javamail.JavaMailSender; import org.springframework.mail.javamail.JavaMailSenderImpl; +import org.springframework.web.client.RestTemplate; @Configuration public class AppConfig { @@ -15,4 +16,9 @@ public JavaMailSender javaMailService() { javaMailSender.setPort(25); return javaMailSender; } + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } } diff --git a/src/main/java/uk/ac/ebi/eva/submission/controller/submissionws/SubmissionController.java b/src/main/java/uk/ac/ebi/eva/submission/controller/submissionws/SubmissionController.java index e566d60..1331ca1 100644 --- a/src/main/java/uk/ac/ebi/eva/submission/controller/submissionws/SubmissionController.java +++ b/src/main/java/uk/ac/ebi/eva/submission/controller/submissionws/SubmissionController.java @@ -19,6 +19,7 @@ import uk.ac.ebi.eva.submission.controller.BaseController; import uk.ac.ebi.eva.submission.entity.Submission; import uk.ac.ebi.eva.submission.entity.SubmissionAccount; +import uk.ac.ebi.eva.submission.exception.MetadataFileInfoMismatchException; import uk.ac.ebi.eva.submission.exception.RequiredFieldsMissingException; import uk.ac.ebi.eva.submission.exception.SubmissionDoesNotExistException; import uk.ac.ebi.eva.submission.model.SubmissionStatus; @@ -93,11 +94,14 @@ public ResponseEntity markSubmissionUploaded(@RequestHeader("Authorization") } try { + submissionService.checkMetadataFileInfoMatchesWithUploadedFiles(submissionAccount, submissionId, metadataJson); Submission submission = this.submissionService.uploadMetadataJsonAndMarkUploaded(submissionId, metadataJson); submissionService.sendMailNotificationForStatusUpdate(submissionAccount, submissionId, SubmissionStatus.UPLOADED, true); return new ResponseEntity<>(stripUserDetails(submission), HttpStatus.OK); } catch (RequiredFieldsMissingException ex) { return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST); + } catch (MetadataFileInfoMismatchException ex) { + return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST); } } diff --git a/src/main/java/uk/ac/ebi/eva/submission/exception/MetadataFileInfoMismatchException.java b/src/main/java/uk/ac/ebi/eva/submission/exception/MetadataFileInfoMismatchException.java new file mode 100644 index 0000000..3b9eb61 --- /dev/null +++ b/src/main/java/uk/ac/ebi/eva/submission/exception/MetadataFileInfoMismatchException.java @@ -0,0 +1,7 @@ +package uk.ac.ebi.eva.submission.exception; + +public class MetadataFileInfoMismatchException extends RuntimeException { + public MetadataFileInfoMismatchException(String message) { + super(message); + } +} diff --git a/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java b/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java index 6eed280..a90d9db 100644 --- a/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java +++ b/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java @@ -16,6 +16,8 @@ public class GlobusDirectoryProvisioner { private final GlobusTokenRefreshService globusTokenRefreshService; + private final RestTemplate restTemplate; + @Value("${globus.submission.endpointId}") private String endpointId; @@ -23,15 +25,16 @@ public class GlobusDirectoryProvisioner { "https://transfer.api.globusonline.org/v0.10/operation/endpoint"; - public GlobusDirectoryProvisioner(GlobusTokenRefreshService globusTokenRefreshService) { + public GlobusDirectoryProvisioner(GlobusTokenRefreshService globusTokenRefreshService, RestTemplate restTemplate) { this.globusTokenRefreshService = globusTokenRefreshService; + this.restTemplate = restTemplate; } public void createSubmissionDirectory(String directoryToCreate) { String fileSeparator = System.getProperty("file.separator"); String[] directoriesToCreate = directoryToCreate.split(Pattern.quote(fileSeparator)); StringBuilder directoryPathSoFar = new StringBuilder(); - for (String directory: directoriesToCreate) { + for (String directory : directoriesToCreate) { directoryPathSoFar.append(directory); createDirectory(directoryPathSoFar.toString()); directoryPathSoFar.append(fileSeparator); @@ -39,14 +42,7 @@ public void createSubmissionDirectory(String directoryToCreate) { } private void createDirectory(String directoryToCreate) { - - String accessToken = globusTokenRefreshService.getAccessToken(); - RestTemplate restTemplate = new RestTemplate(); - HttpHeaders headers = new HttpHeaders(); - headers.set("Authorization", "Bearer " + accessToken); - headers.set("Accept", "application/json"); - headers.set("Content-Type", "application/json"); - + HttpHeaders headers = getGlobusAccessHeaders(); if (this.alreadyExists(headers, directoryToCreate)) { return; @@ -69,17 +65,41 @@ private void createDirectory(String directoryToCreate) { } } - private boolean alreadyExists(HttpHeaders headers, String directoryName) { - RestTemplate restTemplate = new RestTemplate(); + private boolean alreadyExists(HttpHeaders headers, String directoryName) { HttpEntity entity = new HttpEntity<>("", headers); try { restTemplate.exchange( GLOBUS_TRANSFER_API_BASE_URL + "/" + endpointId + "/ls?path=" + directoryName, HttpMethod.GET, entity, String.class); return true; - } - catch (HttpClientErrorException ex) { + } catch (HttpClientErrorException ex) { return false; } } + + public String listSubmittedFiles(String submissionDirPath) { + HttpEntity requestEntity = new HttpEntity<>(getGlobusAccessHeaders()); + String transferApiUrl = String.format("%s/%s/ls?path=%s", GLOBUS_TRANSFER_API_BASE_URL, endpointId, submissionDirPath); + ResponseEntity response = restTemplate.exchange(transferApiUrl, HttpMethod.GET, requestEntity, String.class); + + if (response.getStatusCode().is2xxSuccessful()) { + System.out.printf("Directory %s listed successfully%n", submissionDirPath); + return response.getBody(); + } else { + System.out.printf("Failed to retrieve directory '%s': %s", submissionDirPath, response.getStatusCode()); + return ""; + } + } + + private HttpHeaders getGlobusAccessHeaders() { + String accessToken = globusTokenRefreshService.getAccessToken(); + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", "Bearer " + accessToken); + headers.set("Accept", "application/json"); + headers.set("Content-Type", "application/json"); + + return headers; + } + + } diff --git a/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java b/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java index cdc0152..2f07f2e 100644 --- a/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java +++ b/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java @@ -1,12 +1,16 @@ package uk.ac.ebi.eva.submission.service; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import uk.ac.ebi.eva.submission.entity.Submission; import uk.ac.ebi.eva.submission.entity.SubmissionAccount; import uk.ac.ebi.eva.submission.entity.SubmissionDetails; import uk.ac.ebi.eva.submission.entity.SubmissionProcessing; +import uk.ac.ebi.eva.submission.exception.MetadataFileInfoMismatchException; import uk.ac.ebi.eva.submission.exception.RequiredFieldsMissingException; import uk.ac.ebi.eva.submission.exception.SubmissionDoesNotExistException; import uk.ac.ebi.eva.submission.model.SubmissionProcessingStatus; @@ -20,15 +24,26 @@ import uk.ac.ebi.eva.submission.util.MailSender; import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; @Service public class SubmissionService { private static final String PROJECT = "project"; private static final String TITLE = "title"; private static final String DESCRIPTION = "description"; + private static final String METADATA_FILES_TAG = "files"; + private static final String METADATA_FILE_NAME = "fileName"; + private static final String METADATA_FILE_SIZE = "fileSize"; + private static final String GLOBUS_FILES_TAG = "DATA"; + private static final String GLOBUS_FILE_NAME = "name"; + private static final String GLOBUS_FILE_SIZE = "size"; private final SubmissionRepository submissionRepository; @@ -82,6 +97,58 @@ public Submission initiateSubmission(SubmissionAccount submissionAccount) { return submissionRepository.save(submission); } + public void checkMetadataFileInfoMatchesWithUploadedFiles(SubmissionAccount submissionAccount, String submissionId, JsonNode metadataJson) { + String directoryToList = String.format("%s/%s", submissionAccount.getId(), submissionId); + String uploadedFilesInfo = globusDirectoryProvisioner.listSubmittedFiles(directoryToList); + if (uploadedFilesInfo.isEmpty()) { + throw new MetadataFileInfoMismatchException("Failed to retrieve any file info from submission directory."); + } else { + try { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode globusFileInfoJson = (ObjectNode) mapper.readTree(uploadedFilesInfo); + Map globusFileInfo = new HashMap<>(); + if (globusFileInfoJson.get(GLOBUS_FILES_TAG) != null) { + globusFileInfo = StreamSupport.stream(globusFileInfoJson.get(GLOBUS_FILES_TAG).spliterator(), false) + .filter(dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText().endsWith(".vcf") || dataNode.get(GLOBUS_FILE_NAME).asText().endsWith(".vcf.gz")) + .collect(Collectors.toMap( + dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText(), + dataNode -> dataNode.get(GLOBUS_FILE_SIZE).asLong() + )); + } + + Map metadataFileInfo = StreamSupport.stream(metadataJson.get(METADATA_FILES_TAG).spliterator(), false) + .collect(Collectors.toMap( + dataNode -> dataNode.get(METADATA_FILE_NAME).asText(), + dataNode -> dataNode.get(METADATA_FILE_SIZE).asLong() + )); + + List missingFileList = new ArrayList<>(); + String fileSizeMismatchInfo = ""; + + for (Map.Entry fileEntry : metadataFileInfo.entrySet()) { + String fileName = fileEntry.getKey(); + Long metadataFileSize = fileEntry.getValue(); + if (globusFileInfo.containsKey(fileName)) { + Long fileSizeInGlobus = globusFileInfo.get(fileName); + if (!metadataFileSize.equals(fileSizeInGlobus)) { + fileSizeMismatchInfo += fileName + ": metadata json file size (" + metadataFileSize + ") is not equal to uploaded file size (" + fileSizeInGlobus + ")\n"; + } + } else { + missingFileList.add(fileName); + } + } + + if (!missingFileList.isEmpty() || !fileSizeMismatchInfo.isEmpty()) { + String missingFileMsg = missingFileList.isEmpty() ? "" : "There are some files mentioned in metadata json but not uploaded. Files : " + String.join(", ", missingFileList) + "\n"; + String fileSizeMismatchMsg = fileSizeMismatchInfo.isEmpty() ? "" : "There are some files mentioned in metadata json whose size does not match with the files uploaded.\n" + fileSizeMismatchInfo; + throw new MetadataFileInfoMismatchException(missingFileMsg + fileSizeMismatchMsg); + } + } catch (JsonProcessingException ex) { + throw new MetadataFileInfoMismatchException("Error parsing fileInfo from Submission Directory"); + } + } + } + public Submission uploadMetadataJsonAndMarkUploaded(String submissionId, JsonNode metadataJson) { SubmissionDetails submissionDetails = new SubmissionDetails(submissionId); try { diff --git a/src/test/java/uk/ac/ebi/eva/submission/integration/SubmissionWSIntegrationTest.java b/src/test/java/uk/ac/ebi/eva/submission/integration/SubmissionWSIntegrationTest.java index 7aed856..6769f2f 100644 --- a/src/test/java/uk/ac/ebi/eva/submission/integration/SubmissionWSIntegrationTest.java +++ b/src/test/java/uk/ac/ebi/eva/submission/integration/SubmissionWSIntegrationTest.java @@ -1,6 +1,8 @@ package uk.ac.ebi.eva.submission.integration; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.hibernate.envers.AuditReader; import org.hibernate.envers.AuditReaderFactory; import org.junit.jupiter.api.Disabled; @@ -266,21 +268,63 @@ public void testSubmissionGetStatusSubmissionDoesNotExist() throws Exception { @Test @Transactional - public void testUploadMetadataJsonAndMarkUploadedd() throws Exception { + public void testUploadMetadataJsonAndMarkUploaded() throws Exception { String userToken = "webinUserToken"; SubmissionAccount submissionAccount = getWebinUserAccount(); + String submissionId = createNewSubmissionEntry(submissionAccount); + when(webinTokenService.getWebinUserAccountFromToken(anyString())).thenReturn(submissionAccount); doNothing().when(mailSender).sendEmail(anyString(), anyString(), anyString()); - String submissionId = createNewSubmissionEntry(submissionAccount); String projectTitle = "test_project_title"; String projectDescription = "test_project_description"; - String metadataJson = "{\"project\": {\"title\":\"" + projectTitle + "\",\"description\":\"" + projectDescription + "\"}}"; + + ObjectMapper mapper = new ObjectMapper(); + + // create metadata json + ObjectNode metadataRootNode = mapper.createObjectNode(); + + ObjectNode projectNode = mapper.createObjectNode(); + projectNode.put("title", projectTitle); + projectNode.put("description", projectDescription); + + ArrayNode filesArrayNode = mapper.createArrayNode(); + ObjectNode fileNode1 = mapper.createObjectNode(); + fileNode1.put("fileName", "file1.vcf"); + fileNode1.put("fileSize", 12345L); + ObjectNode fileNode2 = mapper.createObjectNode(); + fileNode2.put("fileName", "file2.vcf.gz"); + fileNode2.put("fileSize", 67890L); + + filesArrayNode.add(fileNode1); + filesArrayNode.add(fileNode2); + + metadataRootNode.put("project", projectNode); + metadataRootNode.put("files", filesArrayNode); + + // create Globus list directory result json + ObjectNode globusRootNode = mapper.createObjectNode(); + + ArrayNode dataNodeArray = mapper.createArrayNode(); + ObjectNode dataNode1 = mapper.createObjectNode(); + dataNode1.put("name", "file1.vcf"); + dataNode1.put("size", 12345L); + ObjectNode dataNode2 = mapper.createObjectNode(); + dataNode2.put("name", "file2.vcf.gz"); + dataNode2.put("size", 67890L); + + dataNodeArray.add(dataNode1); + dataNodeArray.add(dataNode2); + + globusRootNode.put("DATA", dataNodeArray); + + when(globusDirectoryProvisioner.listSubmittedFiles(submissionAccount.getId() + "/" + submissionId)).thenReturn(mapper.writeValueAsString(globusRootNode)); + HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.setBearerAuth(userToken); mvc.perform(put("/v1/submission/" + submissionId + "/uploaded") .headers(httpHeaders) - .content(metadataJson) + .content(mapper.writeValueAsString(metadataRootNode)) .contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()); @@ -298,26 +342,222 @@ public void testUploadMetadataJsonAndMarkUploadedd() throws Exception { assertThat(submissionDetails.getProjectDescription()).isEqualTo(projectDescription); assertThat(submissionDetails.getMetadataJson()).isNotNull(); assertThat(submissionDetails.getMetadataJson().get("project").get("title").asText()).isEqualTo(projectTitle); - assertThat(submissionDetails.getMetadataJson().get("project").get("title").asText()).isEqualTo(projectTitle); + assertThat(submissionDetails.getMetadataJson().get("project").get("description").asText()).isEqualTo(projectDescription); } @Test @Transactional - public void testRequiredMetadataFieldsNotProvided() throws Exception { + public void testMarkSubmissionUploadErrorGettingInfoFromGlobus() throws Exception { String userToken = "webinUserToken"; SubmissionAccount submissionAccount = getWebinUserAccount(); + String submissionId = createNewSubmissionEntry(submissionAccount); + when(webinTokenService.getWebinUserAccountFromToken(anyString())).thenReturn(submissionAccount); doNothing().when(mailSender).sendEmail(anyString(), anyString(), anyString()); + when(globusDirectoryProvisioner.listSubmittedFiles(submissionAccount.getId() + "/" + submissionId)).thenReturn(""); + + // create metadata json + ObjectMapper mapper = new ObjectMapper(); + ObjectNode metadataRootNode = mapper.createObjectNode(); + ArrayNode filesArrayNode = mapper.createArrayNode(); + ObjectNode fileNode1 = mapper.createObjectNode(); + fileNode1.put("fileName", "file1.vcf"); + fileNode1.put("fileSize", 12345L); + filesArrayNode.add(fileNode1); + metadataRootNode.put("files", filesArrayNode); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setBearerAuth(userToken); + mvc.perform(put("/v1/submission/" + submissionId + "/uploaded") + .headers(httpHeaders) + .content(mapper.writeValueAsString(metadataRootNode)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().string("Failed to retrieve any file info from submission directory.")); + } + + @Test + @Transactional + public void testMarkSubmissionUploadFileNotUploaded() throws Exception { + String userToken = "webinUserToken"; + SubmissionAccount submissionAccount = getWebinUserAccount(); String submissionId = createNewSubmissionEntry(submissionAccount); - String projectDescription = "test_project_description"; - String metadataJson = "{\"project\": {\"description\":\"" + projectDescription + "\"}}"; + + when(webinTokenService.getWebinUserAccountFromToken(anyString())).thenReturn(submissionAccount); + doNothing().when(mailSender).sendEmail(anyString(), anyString(), anyString()); + + ObjectMapper mapper = new ObjectMapper(); + + // create metadata json + ObjectNode metadataRootNode = mapper.createObjectNode(); + ArrayNode filesArrayNode = mapper.createArrayNode(); + ObjectNode fileNode1 = mapper.createObjectNode(); + fileNode1.put("fileName", "file1.vcf"); + fileNode1.put("fileSize", 12345L); + filesArrayNode.add(fileNode1); + metadataRootNode.put("files", filesArrayNode); + + // create Globus list directory result json + ObjectNode globusRootNode = mapper.createObjectNode(); + ArrayNode dataNodeArray = mapper.createArrayNode(); + ObjectNode dataNode1 = mapper.createObjectNode(); + dataNode1.put("name", "file10.vcf"); + dataNode1.put("size", 12345L); + dataNodeArray.add(dataNode1); + globusRootNode.put("DATA", dataNodeArray); + + when(globusDirectoryProvisioner.listSubmittedFiles(submissionAccount.getId() + "/" + submissionId)).thenReturn(mapper.writeValueAsString(globusRootNode)); + HttpHeaders httpHeaders = new HttpHeaders(); httpHeaders.setBearerAuth(userToken); + mvc.perform(put("/v1/submission/" + submissionId + "/uploaded") + .headers(httpHeaders) + .content(mapper.writeValueAsString(metadataRootNode)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().string("There are some files mentioned in metadata json but not uploaded. Files : file1.vcf\n")); + } + + @Test + @Transactional + public void testMarkSubmissionUploadFileSizeMismatch() throws Exception { + String userToken = "webinUserToken"; + SubmissionAccount submissionAccount = getWebinUserAccount(); + String submissionId = createNewSubmissionEntry(submissionAccount); + when(webinTokenService.getWebinUserAccountFromToken(anyString())).thenReturn(submissionAccount); + doNothing().when(mailSender).sendEmail(anyString(), anyString(), anyString()); + + ObjectMapper mapper = new ObjectMapper(); + + // create metadata json + ObjectNode metadataRootNode = mapper.createObjectNode(); + ArrayNode filesArrayNode = mapper.createArrayNode(); + ObjectNode fileNode1 = mapper.createObjectNode(); + fileNode1.put("fileName", "file1.vcf"); + fileNode1.put("fileSize", 12345L); + filesArrayNode.add(fileNode1); + metadataRootNode.put("files", filesArrayNode); + + // create Globus list directory result json + ObjectNode globusRootNode = mapper.createObjectNode(); + ArrayNode dataNodeArray = mapper.createArrayNode(); + ObjectNode dataNode1 = mapper.createObjectNode(); + dataNode1.put("name", "file1.vcf"); + dataNode1.put("size", 12346L); + dataNodeArray.add(dataNode1); + globusRootNode.put("DATA", dataNodeArray); + + when(globusDirectoryProvisioner.listSubmittedFiles(submissionAccount.getId() + "/" + submissionId)).thenReturn(mapper.writeValueAsString(globusRootNode)); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setBearerAuth(userToken); mvc.perform(put("/v1/submission/" + submissionId + "/uploaded") .headers(httpHeaders) - .content(metadataJson) + .content(mapper.writeValueAsString(metadataRootNode)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().string("There are some files mentioned in metadata json whose size does not match with the files uploaded.\n" + + "file1.vcf: metadata json file size (12345) is not equal to uploaded file size (12346)\n")); + } + + @Test + @Transactional + public void testMarkSubmissionUploadFileNotUploadedAndFileSizeMismatch() throws Exception { + String userToken = "webinUserToken"; + SubmissionAccount submissionAccount = getWebinUserAccount(); + String submissionId = createNewSubmissionEntry(submissionAccount); + + when(webinTokenService.getWebinUserAccountFromToken(anyString())).thenReturn(submissionAccount); + doNothing().when(mailSender).sendEmail(anyString(), anyString(), anyString()); + + ObjectMapper mapper = new ObjectMapper(); + + // create metadata json + ObjectNode metadataRootNode = mapper.createObjectNode(); + ArrayNode filesArrayNode = mapper.createArrayNode(); + ObjectNode fileNode1 = mapper.createObjectNode(); + fileNode1.put("fileName", "file1.vcf"); + fileNode1.put("fileSize", 12345L); + ObjectNode fileNode2 = mapper.createObjectNode(); + fileNode2.put("fileName", "file2.vcf"); + fileNode2.put("fileSize", 67890L); + filesArrayNode.add(fileNode1); + filesArrayNode.add(fileNode2); + metadataRootNode.put("files", filesArrayNode); + + // create Globus list directory result json + ObjectNode globusRootNode = mapper.createObjectNode(); + ArrayNode dataNodeArray = mapper.createArrayNode(); + ObjectNode dataNode1 = mapper.createObjectNode(); + dataNode1.put("name", "file1.vcf"); + dataNode1.put("size", 12346L); + dataNodeArray.add(dataNode1); + globusRootNode.put("DATA", dataNodeArray); + + when(globusDirectoryProvisioner.listSubmittedFiles(submissionAccount.getId() + "/" + submissionId)).thenReturn(mapper.writeValueAsString(globusRootNode)); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setBearerAuth(userToken); + mvc.perform(put("/v1/submission/" + submissionId + "/uploaded") + .headers(httpHeaders) + .content(mapper.writeValueAsString(metadataRootNode)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().string("There are some files mentioned in metadata json but not uploaded. Files : file2.vcf\n" + + "There are some files mentioned in metadata json whose size does not match with the files uploaded.\n" + + "file1.vcf: metadata json file size (12345) is not equal to uploaded file size (12346)\n")); + } + + @Test + @Transactional + public void testRequiredMetadataFieldsNotProvided() throws Exception { + String userToken = "webinUserToken"; + SubmissionAccount submissionAccount = getWebinUserAccount(); + String submissionId = createNewSubmissionEntry(submissionAccount); + + when(webinTokenService.getWebinUserAccountFromToken(anyString())).thenReturn(submissionAccount); + doNothing().when(mailSender).sendEmail(anyString(), anyString(), anyString()); + + String projectTitle = "test_project_title"; + + ObjectMapper mapper = new ObjectMapper(); + + // create metadata json + ObjectNode metadataRootNode = mapper.createObjectNode(); + + ObjectNode projectNode = mapper.createObjectNode(); + projectNode.put("title", projectTitle); + + ArrayNode filesArrayNode = mapper.createArrayNode(); + ObjectNode fileNode1 = mapper.createObjectNode(); + fileNode1.put("fileName", "file1.vcf"); + fileNode1.put("fileSize", 12345L); + filesArrayNode.add(fileNode1); + + metadataRootNode.put("project", projectNode); + metadataRootNode.put("files", filesArrayNode); + + // create globus list directory json + ObjectNode globusRootNode = mapper.createObjectNode(); + + ArrayNode dataNodeArray = mapper.createArrayNode(); + ObjectNode dataNode1 = mapper.createObjectNode(); + dataNode1.put("name", "file1.vcf"); + dataNode1.put("size", 12345L); + + dataNodeArray.add(dataNode1); + + globusRootNode.put("DATA", dataNodeArray); + + when(globusDirectoryProvisioner.listSubmittedFiles(submissionAccount.getId() + "/" + submissionId)).thenReturn(mapper.writeValueAsString(globusRootNode)); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setBearerAuth(userToken); + mvc.perform(put("/v1/submission/" + submissionId + "/uploaded") + .headers(httpHeaders) + .content(mapper.writeValueAsString(metadataRootNode)) .contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isBadRequest()) .andExpect(content().string("Required fields project title and project description " + From f2109d5095a1260a986ecdf3d69a65979e7d9716 Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Mon, 15 Jul 2024 12:58:19 +0100 Subject: [PATCH 2/3] update file info checker --- .../submission/service/SubmissionService.java | 61 ++++++++++--------- .../SubmissionWSIntegrationTest.java | 29 +++++++++ 2 files changed, 62 insertions(+), 28 deletions(-) diff --git a/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java b/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java index 2f07f2e..0b6c4b7 100644 --- a/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java +++ b/src/main/java/uk/ac/ebi/eva/submission/service/SubmissionService.java @@ -98,53 +98,58 @@ public Submission initiateSubmission(SubmissionAccount submissionAccount) { } public void checkMetadataFileInfoMatchesWithUploadedFiles(SubmissionAccount submissionAccount, String submissionId, JsonNode metadataJson) { + Map metadataFileInfo = new HashMap<>(); + if (metadataJson.get(METADATA_FILES_TAG) != null) { + metadataFileInfo = StreamSupport.stream(metadataJson.get(METADATA_FILES_TAG).spliterator(), false) + .collect(Collectors.toMap( + dataNode -> dataNode.get(METADATA_FILE_NAME).asText(), + dataNode -> dataNode.get(METADATA_FILE_SIZE).asLong() + )); + } + if (metadataFileInfo.isEmpty()) { + throw new MetadataFileInfoMismatchException("Metadata json file does not have any file info"); + } + String directoryToList = String.format("%s/%s", submissionAccount.getId(), submissionId); String uploadedFilesInfo = globusDirectoryProvisioner.listSubmittedFiles(directoryToList); if (uploadedFilesInfo.isEmpty()) { throw new MetadataFileInfoMismatchException("Failed to retrieve any file info from submission directory."); } else { + Map globusFileInfo = new HashMap<>(); try { ObjectMapper mapper = new ObjectMapper(); ObjectNode globusFileInfoJson = (ObjectNode) mapper.readTree(uploadedFilesInfo); - Map globusFileInfo = new HashMap<>(); if (globusFileInfoJson.get(GLOBUS_FILES_TAG) != null) { globusFileInfo = StreamSupport.stream(globusFileInfoJson.get(GLOBUS_FILES_TAG).spliterator(), false) - .filter(dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText().endsWith(".vcf") || dataNode.get(GLOBUS_FILE_NAME).asText().endsWith(".vcf.gz")) .collect(Collectors.toMap( dataNode -> dataNode.get(GLOBUS_FILE_NAME).asText(), dataNode -> dataNode.get(GLOBUS_FILE_SIZE).asLong() )); } + } catch (JsonProcessingException ex) { + throw new MetadataFileInfoMismatchException("Error parsing fileInfo from Submission Directory"); + } - Map metadataFileInfo = StreamSupport.stream(metadataJson.get(METADATA_FILES_TAG).spliterator(), false) - .collect(Collectors.toMap( - dataNode -> dataNode.get(METADATA_FILE_NAME).asText(), - dataNode -> dataNode.get(METADATA_FILE_SIZE).asLong() - )); - - List missingFileList = new ArrayList<>(); - String fileSizeMismatchInfo = ""; - - for (Map.Entry fileEntry : metadataFileInfo.entrySet()) { - String fileName = fileEntry.getKey(); - Long metadataFileSize = fileEntry.getValue(); - if (globusFileInfo.containsKey(fileName)) { - Long fileSizeInGlobus = globusFileInfo.get(fileName); - if (!metadataFileSize.equals(fileSizeInGlobus)) { - fileSizeMismatchInfo += fileName + ": metadata json file size (" + metadataFileSize + ") is not equal to uploaded file size (" + fileSizeInGlobus + ")\n"; - } - } else { - missingFileList.add(fileName); + List missingFileList = new ArrayList<>(); + String fileSizeMismatchInfo = ""; + + for (Map.Entry fileEntry : metadataFileInfo.entrySet()) { + String fileName = fileEntry.getKey(); + Long metadataFileSize = fileEntry.getValue(); + if (globusFileInfo.containsKey(fileName)) { + Long fileSizeInGlobus = globusFileInfo.get(fileName); + if (!metadataFileSize.equals(fileSizeInGlobus)) { + fileSizeMismatchInfo += fileName + ": metadata json file size (" + metadataFileSize + ") is not equal to uploaded file size (" + fileSizeInGlobus + ")\n"; } + } else { + missingFileList.add(fileName); } + } - if (!missingFileList.isEmpty() || !fileSizeMismatchInfo.isEmpty()) { - String missingFileMsg = missingFileList.isEmpty() ? "" : "There are some files mentioned in metadata json but not uploaded. Files : " + String.join(", ", missingFileList) + "\n"; - String fileSizeMismatchMsg = fileSizeMismatchInfo.isEmpty() ? "" : "There are some files mentioned in metadata json whose size does not match with the files uploaded.\n" + fileSizeMismatchInfo; - throw new MetadataFileInfoMismatchException(missingFileMsg + fileSizeMismatchMsg); - } - } catch (JsonProcessingException ex) { - throw new MetadataFileInfoMismatchException("Error parsing fileInfo from Submission Directory"); + if (!missingFileList.isEmpty() || !fileSizeMismatchInfo.isEmpty()) { + String missingFileMsg = missingFileList.isEmpty() ? "" : "There are some files mentioned in metadata json but not uploaded. Files : " + String.join(", ", missingFileList) + "\n"; + String fileSizeMismatchMsg = fileSizeMismatchInfo.isEmpty() ? "" : "There are some files mentioned in metadata json whose size does not match with the files uploaded.\n" + fileSizeMismatchInfo; + throw new MetadataFileInfoMismatchException(missingFileMsg + fileSizeMismatchMsg); } } } diff --git a/src/test/java/uk/ac/ebi/eva/submission/integration/SubmissionWSIntegrationTest.java b/src/test/java/uk/ac/ebi/eva/submission/integration/SubmissionWSIntegrationTest.java index 6769f2f..71d7f7a 100644 --- a/src/test/java/uk/ac/ebi/eva/submission/integration/SubmissionWSIntegrationTest.java +++ b/src/test/java/uk/ac/ebi/eva/submission/integration/SubmissionWSIntegrationTest.java @@ -345,6 +345,35 @@ public void testUploadMetadataJsonAndMarkUploaded() throws Exception { assertThat(submissionDetails.getMetadataJson().get("project").get("description").asText()).isEqualTo(projectDescription); } + @Test + @Transactional + public void testMarkSubmissionUploadNoFileInfoInMetadatajson() throws Exception { + String userToken = "webinUserToken"; + SubmissionAccount submissionAccount = getWebinUserAccount(); + String submissionId = createNewSubmissionEntry(submissionAccount); + + when(webinTokenService.getWebinUserAccountFromToken(anyString())).thenReturn(submissionAccount); + doNothing().when(mailSender).sendEmail(anyString(), anyString(), anyString()); + when(globusDirectoryProvisioner.listSubmittedFiles(submissionAccount.getId() + "/" + submissionId)).thenReturn(""); + + // create metadata json + ObjectMapper mapper = new ObjectMapper(); + ObjectNode metadataRootNode = mapper.createObjectNode(); + ArrayNode filesArrayNode = mapper.createArrayNode(); + metadataRootNode.put("files", filesArrayNode); + + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setBearerAuth(userToken); + mvc.perform(put("/v1/submission/" + submissionId + "/uploaded") + .headers(httpHeaders) + .content(mapper.writeValueAsString(metadataRootNode)) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().string("Metadata json file does not have any file info")); + } + + @Test @Transactional public void testMarkSubmissionUploadErrorGettingInfoFromGlobus() throws Exception { From 6e36bcf9f4149ac7523ea29d08105763384efbbe Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Mon, 15 Jul 2024 15:13:49 +0100 Subject: [PATCH 3/3] review comments --- .../service/GlobusDirectoryProvisioner.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java b/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java index a90d9db..ef66565 100644 --- a/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java +++ b/src/main/java/uk/ac/ebi/eva/submission/service/GlobusDirectoryProvisioner.java @@ -1,5 +1,7 @@ package uk.ac.ebi.eva.submission.service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; @@ -14,6 +16,8 @@ @Service public class GlobusDirectoryProvisioner { + private final Logger logger = LoggerFactory.getLogger(GlobusDirectoryProvisioner.class); + private final GlobusTokenRefreshService globusTokenRefreshService; private final RestTemplate restTemplate; @@ -59,9 +63,9 @@ private void createDirectory(String directoryToCreate) { // Check the response status and handle errors if necessary if (response.getStatusCode().is2xxSuccessful()) { - System.out.printf("Directory '%s' created successfully%n", directoryToCreate); + logger.info("Directory '%s' created successfully%n", directoryToCreate); } else { - System.out.printf("Failed to create directory '%s': %s", directoryToCreate, response.getStatusCode()); + logger.error("Failed to create directory '%s': %s", directoryToCreate, response.getStatusCode()); } } @@ -83,10 +87,10 @@ public String listSubmittedFiles(String submissionDirPath) { ResponseEntity response = restTemplate.exchange(transferApiUrl, HttpMethod.GET, requestEntity, String.class); if (response.getStatusCode().is2xxSuccessful()) { - System.out.printf("Directory %s listed successfully%n", submissionDirPath); + logger.info("Directory %s listed successfully%n", submissionDirPath); return response.getBody(); } else { - System.out.printf("Failed to retrieve directory '%s': %s", submissionDirPath, response.getStatusCode()); + logger.error("Failed to retrieve directory '%s': %s", submissionDirPath, response.getStatusCode()); return ""; } }