From 0d0c112dc868b90256c2c4aa2a0d8949fdd0ce20 Mon Sep 17 00:00:00 2001 From: Alexandre Flores <40147374+SugaryLump@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:56:02 +0100 Subject: [PATCH] Generate risk incidences when Siegfried outputs warnings on format identification (#3263) * SiegfriedPlugin generates risk incidences for warnings. Signed-off-by: sugarylump * Use correct file id when running Siegfried on representations Signed-off-by: sugarylump --------- Signed-off-by: sugarylump --- .../roda/core/data/common/RodaConstants.java | 6 + .../roda/core/data/v2/index/CountRequest.java | 8 + .../org/roda/core/common/PremisV3Utils.java | 48 +++++- .../characterization/SiegfriedPlugin.java | 18 ++- .../SiegfriedPluginUtils.java | 142 +++++++++++++----- .../preservation/EditFileFormatPlugin.java | 50 +++++- .../data/storage/risk/urn:siegfried:r1.json | 24 +++ .../controller/RiskIncidenceController.java | 2 +- 8 files changed, 241 insertions(+), 57 deletions(-) create mode 100644 roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:siegfried:r1.json diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java index 238fa3cb75..a45d6879ba 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java @@ -1423,6 +1423,7 @@ public enum OrchestratorType { public static final String PLUGIN_PARAMS_FORMAT = "parameter.format"; public static final String PLUGIN_PARAMS_FORMAT_VERSION = "parameter.format_version"; public static final String PLUGIN_PARAMS_PRONOM = "parameter.pronom"; + public static final String PLUGIN_PARAMS_CLEAR_INCIDENCES = "parameter.clear_incidences"; public static final String PLUGIN_CATEGORY_CONVERSION = "conversion"; public static final String PLUGIN_CATEGORY_CHARACTERIZATION = "characterization"; @@ -1573,6 +1574,9 @@ public enum OrchestratorType { public static final String RISK_INCIDENCE_FILE_PATH_COMPUTED_SEPARATOR = "/"; public static final String RISK_INCIDENCE_FILE_EXTENSION = ".json"; + /* Risk Ids */ + public static final String RISK_ID_SIEGFRIED_IDENTIFICATION_WARNING = "urn:siegfried:r1"; + /* Representation information */ public static final String REPRESENTATION_INFORMATION_ID = "id"; public static final String REPRESENTATION_INFORMATION_NAME = "name"; @@ -1703,6 +1707,7 @@ public enum OrchestratorType { /* Siegfriend payload fields */ public static final String SIEGFRIED_PAYLOAD_MATCHES = "matches"; + public static final String SIEGFRIED_PAYLOAD_MATCH_WARNING = "warning"; public static final String SIEGFRIED_PAYLOAD_MATCH_NS = "ns"; public static final String SIEGFRIED_PAYLOAD_MATCH_NS_PRONOM = "pronom"; public static final String SIEGFRIED_PAYLOAD_MATCH_MIMETYPE = "mime"; @@ -1745,6 +1750,7 @@ public enum OrchestratorType { public static final String PRESERVATION_REGISTRY_MIME = "mime"; public static final String PRESERVATION_FORMAT_NOTE_MANUAL = "manual"; + public static final String PRESERVATION_FORMAT_NOTE_SIEGFRIED_WARNING = "SIEGFRIED WARNING"; public static final String PREMIS_RELATIONSHIP_TYPE_STRUCTURAL = "structural"; public static final String PREMIS_RELATIONSHIP_SUBTYPE_HASPART = "hasPart"; diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java index 8935c3492a..a75583cd39 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/CountRequest.java @@ -49,7 +49,15 @@ public Filter getFilter() { return filter; } + public void setFilter(Filter filter) { + this.filter = filter; + } + public boolean isOnlyActive() { return onlyActive; } + + public void setOnlyActive(boolean onlyActive) { + this.onlyActive = onlyActive; + } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java b/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java index c5f0a0cfba..87e7d4dc9b 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/common/PremisV3Utils.java @@ -280,6 +280,46 @@ private static ExtensionComplexType getTechnicalMetadata(gov.loc.premis.v3.File return extensionComplexType; } + public static List getFormatNotes(ModelService model, String aipId, String representationId, + List fileDirectoryPath, String fileId, String username) { + Binary premisBin = null; + + try { + try { + premisBin = model.retrievePreservationFile(aipId, representationId, fileDirectoryPath, fileId); + } catch (NotFoundException e) { + LOGGER.debug("PREMIS object skeleton does not exist yet. Creating PREMIS object!"); + List algorithms = RodaCoreFactory.getFixityAlgorithms(); + + if (fileId == null) { + PremisSkeletonPluginUtils.createPremisSkeletonOnRepresentation(model, aipId, representationId, algorithms, + username); + } else { + File file = model.retrieveFile(aipId, representationId, fileDirectoryPath, fileId); + PremisSkeletonPluginUtils.createPremisSkeletonOnFile(model, file, algorithms, username); + } + + premisBin = model.retrievePreservationFile(aipId, representationId, fileDirectoryPath, fileId); + LOGGER.debug("PREMIS object skeleton created"); + } + gov.loc.premis.v3.File premisFile = binaryToFile(premisBin.getContent(), false); + ObjectCharacteristicsComplexType objectCharacteristics; + if (premisFile.getObjectIdentifier() != null && !premisFile.getObjectIdentifier().isEmpty()) { + objectCharacteristics = premisFile.getObjectCharacteristics().get(0); + if (objectCharacteristics.getFormat() != null && !objectCharacteristics.getFormat().isEmpty()) { + for (FormatComplexType format : objectCharacteristics.getFormat()) { + if (format.getFormatNote() != null) { + return format.getFormatNote(); + } + } + } + } + } catch (RODAException | IOException e) { + LOGGER.error("PREMIS could not be checked due to an error", e); + } + return new ArrayList<>(); + } + public static boolean formatWasManuallyModified(ModelService model, String aipId, String representationId, List fileDirectoryPath, String fileId, String username) { Binary premisBin = null; @@ -1059,14 +1099,6 @@ public static void updateFormatPreservationMetadata(ModelService model, String a PremisSkeletonPluginUtils.createPremisSkeletonOnRepresentation(model, aipId, representationId, algorithms, username); } else { - // File file; - // if (shallow) { - // file = model.retrieveFileInsideManifest(aipId, representationId, - // fileDirectoryPath, fileId); - // } else { - // file = model.retrieveFile(aipId, representationId, fileDirectoryPath, - // fileId); - // } File file = model.retrieveFile(aipId, representationId, fileDirectoryPath, fileId); PremisSkeletonPluginUtils.createPremisSkeletonOnFile(model, file, algorithms, username); } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java index eb1409bcae..ab2ce106ca 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPlugin.java @@ -150,7 +150,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic if (aip.getRepresentations() != null && !aip.getRepresentations().isEmpty()) { for (Representation representation : aip.getRepresentations()) { LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId()); - List newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, + List newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual); if (!newSources.isEmpty()) { sources.addAll(newSources); @@ -180,7 +180,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic if (aip.getRepresentations() != null && !aip.getRepresentations().isEmpty()) { for (Representation representation : aip.getRepresentations()) { LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId()); - List newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, + List newSources = SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual); if (!newSources.isEmpty()) { sources.addAll(newSources); @@ -203,7 +203,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic } } } catch (PluginException | NotFoundException | GenericException | RequestNotValidException - | AuthorizationDeniedException e) { + | AuthorizationDeniedException | AlreadyExistsException e) { LOGGER.error("Error running Siegfried {}: {}", aip.getId(), e.getMessage(), e); jobPluginInfo.incrementObjectsProcessedWithFailure(); @@ -225,7 +225,7 @@ public Report executeOnAIP(IndexService index, ModelService model, StorageServic for (Representation representation : filteredList) { try { LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), aip.getId()); - sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, representation, + sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation, cachedJob.getId(), cachedJob.getUsername(), overwriteManual)); if (sources.isEmpty()) { state = PluginState.SKIPPED; @@ -283,7 +283,8 @@ public Report executeOnRepresentation(IndexService index, ModelService model, St PluginHelper.updatePartialJobReport(this, model, reportItem, false, cachedJob); LOGGER.debug("Processing representation {} of AIP {}", representation.getId(), representation.getAipId()); try { - sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, representation, cachedJob.getId(), + sources.addAll(SiegfriedPluginUtils.runSiegfriedOnRepresentation(model, index, representation, + cachedJob.getId(), cachedJob.getUsername(), overwriteManual)); if (sources.isEmpty()) { jobPluginInfo.incrementObjectsProcessedWithSkipped(); @@ -295,7 +296,7 @@ public Report executeOnRepresentation(IndexService index, ModelService model, St model.notifyRepresentationUpdated(representation).failOnError(); } } catch (PluginException | NotFoundException | GenericException | RequestNotValidException - | AuthorizationDeniedException e) { + | AuthorizationDeniedException | AlreadyExistsException e) { LOGGER.error("Error running Siegfried {}: {}", representation.getAipId(), e.getMessage(), e); jobPluginInfo.incrementObjectsProcessedWithFailure(); @@ -331,7 +332,8 @@ public Report executeOnFile(IndexService index, ModelService model, StorageServi file.getAipId()); try { - sources.addAll(SiegfriedPluginUtils.runSiegfriedOnFile(model, file, cachedJob.getUsername(), overwriteManual)); + sources.addAll( + SiegfriedPluginUtils.runSiegfriedOnFile(model, index, file, cachedJob.getUsername(), overwriteManual)); if (sources.isEmpty()) { jobPluginInfo.incrementObjectsProcessedWithSkipped(); reportItem.setPluginState(PluginState.SKIPPED) @@ -341,7 +343,7 @@ public Report executeOnFile(IndexService index, ModelService model, StorageServi reportItem.setPluginState(PluginState.SUCCESS); } } catch (PluginException | NotFoundException | GenericException | RequestNotValidException - | AuthorizationDeniedException e) { + | AuthorizationDeniedException | AlreadyExistsException e) { LOGGER.error("Error running Siegfried on file {}: {}", file.getId(), e.getMessage(), e); jobPluginInfo.incrementObjectsProcessedWithFailure(); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPluginUtils.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPluginUtils.java index 01bcc38b20..e4d997ac13 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPluginUtils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/characterization/SiegfriedPluginUtils.java @@ -21,18 +21,27 @@ import org.roda.core.RodaCoreFactory; import org.roda.core.common.PremisV3Utils; import org.roda.core.data.common.RodaConstants; +import org.roda.core.data.exceptions.AlreadyExistsException; import org.roda.core.data.exceptions.AuthorizationDeniedException; import org.roda.core.data.exceptions.GenericException; import org.roda.core.data.exceptions.NotFoundException; import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.utils.JsonUtils; import org.roda.core.data.v2.IsRODAObject; +import org.roda.core.data.v2.index.filter.Filter; +import org.roda.core.data.v2.index.filter.FilterParameter; +import org.roda.core.data.v2.index.filter.SimpleFilterParameter; import org.roda.core.data.v2.ip.File; import org.roda.core.data.v2.ip.Representation; import org.roda.core.data.v2.ip.StoragePath; import org.roda.core.data.v2.ip.metadata.LinkingIdentifier; import org.roda.core.data.v2.jobs.Job; import org.roda.core.data.v2.jobs.PluginType; +import org.roda.core.data.v2.risks.IncidenceStatus; +import org.roda.core.data.v2.risks.RiskIncidence; +import org.roda.core.data.v2.risks.SeverityLevel; +import org.roda.core.index.IndexService; +import org.roda.core.index.utils.IterableIndexResult; import org.roda.core.model.ModelService; import org.roda.core.model.utils.ModelUtils; import org.roda.core.plugins.PluginException; @@ -142,9 +151,9 @@ public static String getVersion() { } public static List runSiegfriedOnRepresentation(ModelService model, + IndexService index, Representation representation, String jobId, String username, boolean overwriteManual) throws GenericException, - RequestNotValidException, - NotFoundException, AuthorizationDeniedException, PluginException { + RequestNotValidException, NotFoundException, AuthorizationDeniedException, PluginException, AlreadyExistsException { StoragePath representationDataPath = ModelUtils.getRepresentationDataStoragePath(representation.getAipId(), representation.getId()); StorageService storageService; @@ -154,7 +163,7 @@ public static List runSiegfriedOnRep ModelUtils.getAIPStoragePath(representation.getAipId())); try (DirectResourceAccess directAccess = tmpStorageService.getDirectAccess(representationDataPath)) { Path representationFsPath = directAccess.getPath(); - return runSiegfriedOnRepresentationOrFile(model, representation.getAipId(), representation.getId(), + return runSiegfriedOnRepresentationOrFile(model, index, representation.getAipId(), representation.getId(), new ArrayList<>(), null, representationFsPath, username, overwriteManual); } catch (IOException e) { throw new GenericException(e); @@ -171,7 +180,7 @@ public static List runSiegfriedOnRep } else { try (DirectResourceAccess directAccess = model.getStorage().getDirectAccess(representationDataPath)) { Path representationFsPath = directAccess.getPath(); - return runSiegfriedOnRepresentationOrFile(model, representation.getAipId(), representation.getId(), + return runSiegfriedOnRepresentationOrFile(model, index, representation.getAipId(), representation.getId(), new ArrayList<>(), null, representationFsPath, username, overwriteManual); } catch (IOException e) { throw new GenericException(e); @@ -179,15 +188,15 @@ public static List runSiegfriedOnRep } } - public static List runSiegfriedOnFile(ModelService model, File file, + public static List runSiegfriedOnFile(ModelService model, + IndexService index, File file, String username, boolean overwriteManual) throws GenericException, RequestNotValidException, NotFoundException, - AuthorizationDeniedException, - PluginException { + AuthorizationDeniedException, PluginException, AlreadyExistsException { StoragePath fileStoragePath = ModelUtils.getFileStoragePath(file); try (DirectResourceAccess directAccess = model.getStorage().getDirectAccess(fileStoragePath)) { Path filePath = directAccess.getPath(); - List sources = runSiegfriedOnRepresentationOrFile(model, file.getAipId(), + List sources = runSiegfriedOnRepresentationOrFile(model, index, file.getAipId(), file.getRepresentationId(), file.getPath(), file.getId(), filePath, username, overwriteManual); model.notifyFileUpdated(file).failOnError(); return sources; @@ -197,10 +206,11 @@ public static List runSiegfriedOnFil } private static List runSiegfriedOnRepresentationOrFile(ModelService model, - String aipId, String representationId, List fileDirectoryPath, String fileId, Path path, String username, + IndexService index, String aipId, String representationId, List fileDirectoryPath, String fileId, Path path, + String username, Boolean overwriteManual) - throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException, - PluginException { + throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException, PluginException, + AlreadyExistsException { List sources = new ArrayList<>(); if (FSUtils.exists(path)) { @@ -225,37 +235,95 @@ private static List runSiegfriedOnRe jsonFilePath.remove(jsonFilePath.size() - 1); - if (!PremisV3Utils.formatWasManuallyModified(model, aipId, representationId, jsonFilePath, jsonFileId, username) - || overwriteManual) { - ContentPayload payload = new StringContentPayload(file.toString()); - model.createOrUpdateOtherMetadata(aipId, representationId, jsonFilePath, jsonFileId, - SiegfriedPlugin.FILE_SUFFIX, RodaConstants.OTHER_METADATA_TYPE_SIEGFRIED, payload, username, false); - - sources.add(PluginHelper.getLinkingIdentifier(aipId, representationId, jsonFilePath, jsonFileId, - RodaConstants.PRESERVATION_LINKING_OBJECT_SOURCE)); - - // Update PREMIS files - final JsonNode matches = file.get("matches"); - for (JsonNode match : matches) { - String format = null; - String version = null; - String pronom = null; - String mime = null; - - if ("pronom".equalsIgnoreCase(match.get("ns").textValue())) { - format = match.get("format").textValue(); - version = match.get("version").textValue(); - pronom = match.get("id").textValue(); - mime = match.get("mime").textValue(); + JsonNode matches = file.get(RodaConstants.SIEGFRIED_PAYLOAD_MATCHES); + if (matches != null) { + if (!PremisV3Utils.formatWasManuallyModified(model, aipId, representationId, jsonFilePath, jsonFileId, + username) || overwriteManual) { + ContentPayload payload = new StringContentPayload(file.toString()); + model.createOrUpdateOtherMetadata(aipId, representationId, jsonFilePath, jsonFileId, + SiegfriedPlugin.FILE_SUFFIX, RodaConstants.OTHER_METADATA_TYPE_SIEGFRIED, payload, username, false); + + sources.add(PluginHelper.getLinkingIdentifier(aipId, representationId, jsonFilePath, jsonFileId, + RodaConstants.PRESERVATION_LINKING_OBJECT_SOURCE)); + + // Update PREMIS files + for (JsonNode match : matches) { + String format = null; + String version = null; + String pronom = null; + String mime = null; + + if ("pronom".equalsIgnoreCase(match.get("ns").textValue())) { + format = match.get("format").textValue(); + version = match.get("version").textValue(); + pronom = match.get("id").textValue(); + mime = match.get("mime").textValue(); + } + + JsonNode warning = match.get(RodaConstants.SIEGFRIED_PAYLOAD_MATCH_WARNING); + List notes = new ArrayList<>(); + if (StringUtils.isNotBlank(warning.textValue())) { + notes.add(RodaConstants.PRESERVATION_FORMAT_NOTE_SIEGFRIED_WARNING + ": " + warning.textValue()); + updateFileRiskIncidences(model, index, aipId, representationId, jsonFileId, jsonFilePath, + warning.textValue()); + } + PremisV3Utils.updateFormatPreservationMetadata(model, aipId, representationId, jsonFilePath, jsonFileId, + format, version, pronom, mime, notes, username, true); } - - PremisV3Utils.updateFormatPreservationMetadata(model, aipId, representationId, jsonFilePath, jsonFileId, - format, version, pronom, mime, new ArrayList<>(), username, true); } } } } - return sources; } + + private static void updateFileRiskIncidences(ModelService model, IndexService index, String aipId, + String representationId, String fileId, List filePath, String warning) throws RequestNotValidException, + GenericException, AuthorizationDeniedException, AlreadyExistsException, NotFoundException { + // Mitigate previous incidences + for (RiskIncidence incidence : getPreviousSiegfriedIncidences(model, index, fileId)) { + incidence.setStatus(IncidenceStatus.MITIGATED); + model.updateRiskIncidence(incidence, true); + } + // Create a new incidence + RiskIncidence riskIncidence = new RiskIncidence(); + if (fileId != null) { + riskIncidence.setFileId(fileId); + riskIncidence.setFilePath(filePath); + riskIncidence.setObjectClass(File.class.getName()); + } else { + riskIncidence.setObjectClass(Representation.class.getName()); + } + riskIncidence.setRiskId(RodaConstants.RISK_ID_SIEGFRIED_IDENTIFICATION_WARNING); + riskIncidence.setDetectedBy(SiegfriedPlugin.getStaticName()); + riskIncidence.setByPlugin(true); + riskIncidence.setStatus(IncidenceStatus.UNMITIGATED); + riskIncidence.setRepresentationId(representationId); + riskIncidence.setAipId(aipId); + riskIncidence.setDescription(warning); + riskIncidence.setFileId(fileId); + riskIncidence.setSeverity(SeverityLevel.LOW); + model.createRiskIncidence(riskIncidence, true); + } + + public static List getPreviousSiegfriedIncidences(ModelService model, IndexService index, + String fileId) throws RequestNotValidException, GenericException, AuthorizationDeniedException, NotFoundException { + List riskIncidences = new ArrayList<>(); + Filter filter = new Filter(); + List filterParameters = new ArrayList<>(); + filterParameters.add(new SimpleFilterParameter("riskId", RodaConstants.RISK_ID_SIEGFRIED_IDENTIFICATION_WARNING)); + filterParameters.add(new SimpleFilterParameter("fileId", fileId)); + filterParameters.add(new SimpleFilterParameter("status", IncidenceStatus.UNMITIGATED.name())); + filter.add(filterParameters); + try (IterableIndexResult results = index.findAll(RiskIncidence.class, filter, true, + Arrays.asList("id", "status"))) { + for (RiskIncidence incidence : results) { + RiskIncidence modelIncidence = model.retrieveRiskIncidence(incidence.getId()); + riskIncidences.add(modelIncidence); + } + } catch (IOException e) { + LOGGER.error("Error finding file id {}'s associated risk incidences", fileId, e); + } + return riskIncidences; + } } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/preservation/EditFileFormatPlugin.java b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/preservation/EditFileFormatPlugin.java index 1d7e166031..c5ce399da3 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/preservation/EditFileFormatPlugin.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/plugins/base/preservation/EditFileFormatPlugin.java @@ -28,6 +28,8 @@ import org.roda.core.data.v2.jobs.PluginState; import org.roda.core.data.v2.jobs.PluginType; import org.roda.core.data.v2.jobs.Report; +import org.roda.core.data.v2.risks.IncidenceStatus; +import org.roda.core.data.v2.risks.RiskIncidence; import org.roda.core.data.v2.validation.ValidationException; import org.roda.core.index.IndexService; import org.roda.core.model.ModelService; @@ -37,6 +39,7 @@ import org.roda.core.plugins.PluginException; import org.roda.core.plugins.PluginHelper; import org.roda.core.plugins.RODAObjectsProcessingLogic; +import org.roda.core.plugins.base.characterization.SiegfriedPluginUtils; import org.roda.core.plugins.orchestrate.JobPluginInfo; import org.roda.core.storage.DirectResourceAccess; import org.roda.core.storage.StorageService; @@ -69,12 +72,20 @@ public class EditFileFormatPlugin extends AbstractPlugin { PluginParameter .getBuilder(RodaConstants.PLUGIN_PARAMS_PRONOM, "PRONOM", PluginParameter.PluginParameterType.STRING) .withDescription("The PRONOM identifier to set for all selected files.").build()); + pluginParameters.put(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES, + PluginParameter + .getBuilder(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES, "Clear Siegfried risk incidences", + PluginParameter.PluginParameterType.BOOLEAN) + .withDescription( + "Have this plugin clear risk incidences caused by Siegfried file format identification warnings.") + .build()); } private String mimetype; private String format; private String formatVersion; private String pronom; + private boolean clearIncidences; public static String getStaticName() { return "Edit File Format"; @@ -116,6 +127,7 @@ public List getParameters() { parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_FORMAT)); parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_FORMAT_VERSION)); parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_PRONOM)); + parameters.add(pluginParameters.get(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES)); return parameters; } @@ -126,6 +138,11 @@ public void setParameterValues(Map parameters) throws InvalidPar format = parameters.get(RodaConstants.PLUGIN_PARAMS_FORMAT); formatVersion = parameters.get(RodaConstants.PLUGIN_PARAMS_FORMAT_VERSION); pronom = parameters.get(RodaConstants.PLUGIN_PARAMS_PRONOM); + if (parameters.containsKey(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES)) { + clearIncidences = Boolean.parseBoolean(parameters.get(RodaConstants.PLUGIN_PARAMS_CLEAR_INCIDENCES)); + } else { + clearIncidences = false; + } } @Override @@ -154,7 +171,7 @@ private void processFiles(IndexService index, ModelService model, StorageService } else { List sources = new ArrayList<>(); try { - sources.add(setFileFormatMetadata(model, file, cachedJob.getId(), cachedJob.getUsername())); + sources.add(setFileFormatMetadata(model, index, file, cachedJob.getId(), cachedJob.getUsername())); jobPluginInfo.incrementObjectsProcessedWithSuccess(); reportItem.setPluginState(PluginState.SUCCESS); } catch (RequestNotValidException | NotFoundException | AuthorizationDeniedException | GenericException @@ -200,7 +217,8 @@ private Report validateParameters() { return reportItem; } - private LinkingIdentifier setFileFormatMetadata(ModelService model, File file, String jobId, String username) + private LinkingIdentifier setFileFormatMetadata(ModelService model, IndexService index, File file, String jobId, + String username) throws GenericException, RequestNotValidException, NotFoundException, AuthorizationDeniedException, PluginException { StoragePath fileDataPath = ModelUtils.getFileStoragePath(file); @@ -228,9 +246,11 @@ private LinkingIdentifier setFileFormatMetadata(ModelService model, File file, S String updatedPronomIdentifier = pronom; String updatedMimetype = mimetype; + List notes = mitigatePreviousIncidencesAndCreateNotes(model, index, file.getAipId(), + file.getRepresentationId(), file.getId(), jsonFilePath, username); PremisV3Utils.updateFormatPreservationMetadata(model, file.getAipId(), file.getRepresentationId(), jsonFilePath, jsonFileId, updatedFormat, updatedFormatVersion, updatedPronomIdentifier, updatedMimetype, - Arrays.asList(RodaConstants.PRESERVATION_FORMAT_NOTE_MANUAL), username, true); + notes, username, true); source = PluginHelper.getLinkingIdentifier(file.getAipId(), file.getRepresentationId(), jsonFilePath, jsonFileId, RodaConstants.PRESERVATION_LINKING_OBJECT_SOURCE); @@ -241,6 +261,30 @@ private LinkingIdentifier setFileFormatMetadata(ModelService model, File file, S return source; } + private List mitigatePreviousIncidencesAndCreateNotes(ModelService model, IndexService index, String aipId, + String representationId, String fileId, List filePath, String username) + throws AuthorizationDeniedException, RequestNotValidException, NotFoundException, GenericException { + List notes = new ArrayList<>(); + notes.add(RodaConstants.PRESERVATION_FORMAT_NOTE_MANUAL); + List siegfriedRiskIncidences = SiegfriedPluginUtils.getPreviousSiegfriedIncidences(model, index, + fileId); + if (!siegfriedRiskIncidences.isEmpty()) { + if (clearIncidences) { + for (RiskIncidence incidence : siegfriedRiskIncidences) { + incidence.setStatus(IncidenceStatus.MITIGATED); + model.updateRiskIncidence(incidence, true); + } + } else { + for (String note : PremisV3Utils.getFormatNotes(model, aipId, representationId, filePath, fileId, username)) { + if (note.contains(RodaConstants.PRESERVATION_FORMAT_NOTE_SIEGFRIED_WARNING)) { + notes.add(note); + } + } + } + } + return notes; + } + @Override public RodaConstants.PreservationEventType getPreservationEventType() { return RodaConstants.PreservationEventType.UPDATE; diff --git a/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:siegfried:r1.json b/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:siegfried:r1.json new file mode 100644 index 0000000000..ae93a12aa2 --- /dev/null +++ b/roda-core/roda-core/src/main/resources/default/data/storage/risk/urn:siegfried:r1.json @@ -0,0 +1,24 @@ +{ + "categories": [ + "Content characterization" + ], + "id": "urn:siegfried:r1", + "name": "File format identification issue", + "description": "File format identification process has identified possible issues while identifying file formats.", + "notes": "The file format identification process may have reported warnings during the matching process of the file format identifier signatures. These are not strictly errors but may still warrant further investigation. Common warnings include being unable to detect the file format or a mismatch between the identified file format via identifier signatures and the file name extension.", + "preMitigationNotes": "\n- The repository cannot reliably confirm the technical metadata of the affected files", + "mitigationOwner": "Preservation", + "mitigationStrategy": "Manually review and/or edit the technical metadata of affected files in order to verify the relevance of the produced warnings.", + "preMitigationProbability": 4, + "preMitigationImpact": 3, + "preMitigationSeverity": 12, + "preMitigationSeverityLevel": "MODERATE", + "identifiedBy": "Risk automatically detected by the File Format Identification plugin \"roda.core.plugins.plugins.base.characterization.SiegfriedPlugin\".", + "mitigationOwnerType": "", + "createdBy": "admin", + "createdOn": 1721257200000, + "updatedBy": "admin", + "updatedOn": 1721257200000, + "identifiedOn": "1721257200000" +} + \ No newline at end of file diff --git a/roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/controller/RiskIncidenceController.java b/roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/controller/RiskIncidenceController.java index 8b52957ac4..4ca0a254e8 100644 --- a/roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/controller/RiskIncidenceController.java +++ b/roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/controller/RiskIncidenceController.java @@ -73,7 +73,7 @@ public IndexResult find(@RequestBody FindRequest findRequest, Str } @Override - public LongResponse count(CountRequest countRequest) { + public LongResponse count(@RequestBody CountRequest countRequest) { RequestContext requestContext = RequestUtils.parseHTTPRequest(request); if (UserUtility.hasPermissions(requestContext.getUser(), RodaConstants.PERMISSION_METHOD_FIND_RISK_INCIDENCE)) { return new LongResponse(indexService.count(RiskIncidence.class, countRequest, requestContext));