Merge branch 'master' of https://github.com/folio-org/mod-data-export…

…-worker into ualibweb-squashed

src/main/java/org/folio/dew/batch/acquisitions/edifact/CompositePOConverter.java

@@ -28,7 +28,7 @@ public void convertPOtoEdifact(EDIStreamWriter writer, CompositePurchaseOrder co
     int messageSegmentCount = 0;
 
     messageSegmentCount++;
-    writePOHeader(ediFileConfig.getFileId(), writer);
+    writePOHeader(compPO, writer);
 
     String rushOrderQualifier = compPO.getCompositePoLines().stream().filter(line -> line.getRush() != null).anyMatch(CompositePoLine::getRush) ? RUSH_ORDER : NOT_RUSH_ORDER;
     messageSegmentCount++;
@@ -78,9 +78,9 @@ public void convertPOtoEdifact(EDIStreamWriter writer, CompositePurchaseOrder co
   }
 
   // Order header = Start of order; EDIFACT message type - There would be a new UNH for each FOLIO PO in the file
-  private void writePOHeader(String fileId, EDIStreamWriter writer) throws EDIStreamException {
+  private void writePOHeader(CompositePurchaseOrder compPO, EDIStreamWriter writer) throws EDIStreamException {
     writer.writeStartSegment("UNH")
-      .writeElement(fileId)
+      .writeElement(compPO.getPoNumber())
       .writeStartElement()
       .writeComponent("ORDERS")
       .writeComponent("D")

src/main/java/org/folio/dew/batch/bulkedit/jobs/BulkEditInstanceListProcessor.java

@@ -0,0 +1,33 @@
+package org.folio.dew.batch.bulkedit.jobs;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.folio.dew.domain.dto.InstanceCollection;
+import org.folio.dew.domain.dto.InstanceFormat;
+import org.folio.dew.error.BulkEditException;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+import static org.folio.dew.utils.Constants.NO_MATCH_FOUND_MESSAGE;
+
+@Component
+@StepScope
+@RequiredArgsConstructor
+@Log4j2
+public class BulkEditInstanceListProcessor implements ItemProcessor<InstanceCollection, List<InstanceFormat>> {
+  private final BulkEditInstanceProcessor bulkEditInstanceProcessor;
+
+  @Override
+  public List<InstanceFormat> process(InstanceCollection instances) {
+    if (instances.getInstances().isEmpty()) {
+      log.error(NO_MATCH_FOUND_MESSAGE);
+      throw new BulkEditException(NO_MATCH_FOUND_MESSAGE);
+    }
+    return instances.getInstances().stream()
+      .map(bulkEditInstanceProcessor::process)
+      .toList();
+  }
+}

src/main/java/org/folio/dew/batch/bulkedit/jobs/BulkEditInstanceProcessor.java

@@ -0,0 +1,121 @@
+package org.folio.dew.batch.bulkedit.jobs;
+
+import static org.apache.commons.lang3.ObjectUtils.isEmpty;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.apache.commons.io.FilenameUtils;
+import org.folio.dew.domain.dto.*;
+import org.folio.dew.domain.dto.FormatOfInstance;
+import org.folio.dew.domain.dto.Instance;
+import org.folio.dew.domain.dto.InstanceContributorsInner;
+import org.folio.dew.domain.dto.InstanceSeriesInner;
+import org.folio.dew.service.InstanceReferenceService;
+import org.folio.dew.service.SpecialCharacterEscaper;
+import org.jetbrains.annotations.NotNull;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.commons.lang3.StringUtils.EMPTY;
+import static org.folio.dew.utils.Constants.ARRAY_DELIMITER_SPACED;
+import static org.folio.dew.utils.Constants.ITEM_DELIMITER;
+
+@Component
+@StepScope
+@RequiredArgsConstructor
+@Log4j2
+public class BulkEditInstanceProcessor implements ItemProcessor<Instance, InstanceFormat> {
+
+  private final InstanceReferenceService  instanceReferenceService;
+  private final SpecialCharacterEscaper escaper;
+
+
+  @Value("#{jobParameters['identifierType']}")
+  private String identifierType;
+  @Value("#{jobParameters['jobId']}")
+  private String jobId;
+  @Value("#{jobParameters['fileName']}")
+  private String fileName;
+
+  @Override
+  public InstanceFormat process(@NotNull Instance instance) {
+    var errorServiceArgs = new ErrorServiceArgs(jobId, getIdentifier(instance, identifierType), FilenameUtils.getName(fileName));
+
+    var instanceFormat = InstanceFormat.builder()
+      .id(instance.getId())
+      .discoverySuppress(isEmpty(instance.getVersion()) ? EMPTY : Boolean.toString(instance.getDiscoverySuppress()))
+      .staffSuppress(isEmpty(instance.getStaffSuppress()) ? EMPTY : Boolean.toString(instance.getStaffSuppress()))
+      .previouslyHeld(isEmpty(instance.getPreviouslyHeld()) ? EMPTY : Boolean.toString(instance.getPreviouslyHeld()))
+      .hrid(instance.getHrid())
+      .source(instance.getSource())
+      .catalogedDate(instance.getCatalogedDate())
+      .statusId(instanceReferenceService.getInstanceStatusNameById(instance.getStatusId(), errorServiceArgs))
+      .modeOfIssuanceId(instanceReferenceService.getModeOfIssuanceNameById(instance.getModeOfIssuanceId(), errorServiceArgs))
+      .administrativeNotes(isEmpty(instance.getAdministrativeNotes()) ? EMPTY : String.join(ITEM_DELIMITER, escaper.escape(instance.getAdministrativeNotes())))
+      .title(instance.getTitle())
+      .indexTitle(instance.getIndexTitle())
+      .series(fetchSeries(instance.getSeries()))
+      .contributors(fetchContributorNames(instance.getContributors()))
+      .editions(isEmpty(instance.getEditions()) ? EMPTY : String.join(ITEM_DELIMITER, escaper.escape(new ArrayList<>(instance.getEditions()))))
+      .physicalDescriptions(isEmpty(instance.getPhysicalDescriptions()) ? EMPTY : String.join(ITEM_DELIMITER, escaper.escape(instance.getPhysicalDescriptions())))
+      .instanceTypeId(instanceReferenceService.getInstanceTypeNameById(instance.getInstanceTypeId(), errorServiceArgs))
+      .natureOfContentTermIds(fetchNatureOfContentTerms(instance.getNatureOfContentTermIds(), errorServiceArgs))
+      .instanceFormatIds(fetchInstanceFormats(instance.getInstanceFormats(), errorServiceArgs))
+      .languages(isEmpty(instance.getLanguages()) ? EMPTY : String.join(ITEM_DELIMITER, escaper.escape(instance.getLanguages())))
+      .publicationFrequency(isEmpty(instance.getPublicationFrequency()) ? EMPTY : String.join(ITEM_DELIMITER, escaper.escape(new ArrayList<>(instance.getPublicationFrequency()))))
+      .publicationRange(isEmpty(instance.getPublicationRange()) ? EMPTY : String.join(ITEM_DELIMITER, escaper.escape(new ArrayList<>(instance.getPublicationRange()))))
+      .build();
+
+
+    return instanceFormat.withOriginal(instance);
+  }
+
+  private String fetchInstanceFormats(List<FormatOfInstance> instanceFormats, ErrorServiceArgs errorServiceArgs) {
+    return isEmpty(instanceFormats) ? EMPTY :
+      instanceFormats.stream()
+        .map(iFormat -> instanceReferenceService.getFormatOfInstanceNameById(iFormat.getId(), errorServiceArgs))
+        .map(iFormatName -> String.join(ITEM_DELIMITER, escaper.escape(iFormatName)))
+        .collect(Collectors.joining(ITEM_DELIMITER));
+  }
+
+  private String fetchNatureOfContentTerms(Set<String> natureOfContentTermIds, ErrorServiceArgs errorServiceArgs) {
+    return isEmpty(natureOfContentTermIds) ? EMPTY :
+      natureOfContentTermIds.stream()
+        .map(natId -> instanceReferenceService.getNatureOfContentTermNameById(natId, errorServiceArgs))
+        .map(natName -> String.join(ITEM_DELIMITER, escaper.escape(natName)))
+        .collect(Collectors.joining(ITEM_DELIMITER));
+  }
+
+  private String fetchContributorNames(List<InstanceContributorsInner> contributors) {
+    return isEmpty(contributors) ? EMPTY :
+      contributors.stream()
+        .map(c -> String.join(ARRAY_DELIMITER_SPACED, escaper.escape(c.getName())))
+        .collect(Collectors.joining(ARRAY_DELIMITER_SPACED));
+  }
+
+  private String fetchSeries(Set<InstanceSeriesInner> series) {
+    return isEmpty(series) ? EMPTY :
+        series.stream()
+        .map(instanceSeriesInner -> String.join(ITEM_DELIMITER, escaper.escape(instanceSeriesInner.getValue())))
+        .collect(Collectors.joining(ITEM_DELIMITER));
+  }
+
+
+  private String getIdentifier(Instance instance, String identifierType) {
+    try {
+      return switch (org.folio.dew.domain.dto.IdentifierType.fromValue(identifierType)) {
+        case HRID -> instance.getHrid();
+        default -> instance.getId();
+      };
+    } catch (IllegalArgumentException e) {
+      return instance.getId();
+    }
+  }
+}

src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/BulkEditInstanceIdentifiersJobConfig.java

@@ -0,0 +1,91 @@
+package org.folio.dew.batch.bulkedit.jobs.processidentifiers;
+
+
+import lombok.RequiredArgsConstructor;
+import org.folio.dew.batch.CsvListFileWriter;
+import org.folio.dew.batch.JsonListFileWriter;
+import org.folio.dew.batch.JobCompletionNotificationListener;
+import org.folio.dew.batch.bulkedit.jobs.BulkEditInstanceListProcessor;
+import org.folio.dew.domain.dto.ExportType;
+import org.folio.dew.domain.dto.ItemIdentifier;
+import org.folio.dew.domain.dto.InstanceFormat;
+import org.folio.dew.error.BulkEditException;
+import org.folio.dew.error.BulkEditSkipListener;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.core.job.builder.JobBuilder;
+import org.springframework.batch.core.launch.support.RunIdIncrementer;
+import org.springframework.batch.core.repository.JobRepository;
+import org.springframework.batch.core.step.builder.StepBuilder;
+import org.springframework.batch.item.file.FlatFileItemReader;
+import org.springframework.batch.item.support.CompositeItemProcessor;
+import org.springframework.batch.item.support.CompositeItemWriter;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.folio.dew.domain.dto.EntityType.INSTANCE;
+import static org.folio.dew.domain.dto.JobParameterNames.TEMP_LOCAL_FILE_PATH;
+import static org.folio.dew.utils.Constants.CHUNKS;
+import static org.folio.dew.utils.Constants.JOB_NAME_POSTFIX_SEPARATOR;
+
+@Configuration
+@RequiredArgsConstructor
+public class BulkEditInstanceIdentifiersJobConfig {
+
+  private final BulkEditInstanceListProcessor bulkEditInstanceListProcessor;
+  private final InstanceFetcher instanceFetcher;
+  private final BulkEditSkipListener bulkEditSkipListener;
+
+  @Bean
+  public Job bulkEditProcessInstanceIdentifiersJob(JobCompletionNotificationListener listener, Step bulkEditInstanceStep,
+                                                   JobRepository jobRepository) {
+    return new JobBuilder(ExportType.BULK_EDIT_IDENTIFIERS + JOB_NAME_POSTFIX_SEPARATOR + INSTANCE.getValue(), jobRepository)
+      .incrementer(new RunIdIncrementer())
+      .listener(listener)
+      .flow(bulkEditInstanceStep)
+      .end()
+      .build();
+  }
+
+  @Bean
+  public Step bulkEditInstanceStep(FlatFileItemReader<ItemIdentifier> csvItemIdentifierReader,
+                               CompositeItemWriter<List<InstanceFormat>> compositeInstanceListWriter,
+                               ListIdentifiersWriteListener<InstanceFormat> listIdentifiersWriteListener, JobRepository jobRepository,
+                               PlatformTransactionManager transactionManager) {
+    return new StepBuilder("bulkEditInstanceStep", jobRepository)
+      .<ItemIdentifier, List<InstanceFormat>> chunk(CHUNKS, transactionManager)
+      .reader(csvItemIdentifierReader)
+      .processor(identifierInstanceProcessor())
+      .faultTolerant()
+      .skipLimit(1_000_000)
+      .processorNonTransactional() // Required to avoid repeating BulkEditItemProcessor#process after skip.
+      .skip(BulkEditException.class)
+      .listener(bulkEditSkipListener)
+      .writer(compositeInstanceListWriter)
+      .listener(listIdentifiersWriteListener)
+      .build();
+  }
+
+  @Bean
+  public CompositeItemProcessor<ItemIdentifier, List<InstanceFormat>> identifierInstanceProcessor() {
+    var processor = new CompositeItemProcessor<ItemIdentifier, List<InstanceFormat>>();
+    processor.setDelegates(Arrays.asList(instanceFetcher, bulkEditInstanceListProcessor));
+    return processor;
+  }
+
+  @Bean
+  @StepScope
+  public CompositeItemWriter<List<InstanceFormat>> compositeInstanceListWriter(@Value("#{jobParameters['" + TEMP_LOCAL_FILE_PATH + "']}") String outputFileName) {
+    var writer = new CompositeItemWriter<List<InstanceFormat>>();
+    writer.setDelegates(Arrays.asList(new CsvListFileWriter<>(outputFileName, InstanceFormat.getInstanceColumnHeaders(), InstanceFormat.getInstanceFieldsArray(), (field, i) -> field),
+      new JsonListFileWriter<>(new FileSystemResource(outputFileName + ".json"))));
+    return writer;
+  }
+}

src/main/java/org/folio/dew/batch/bulkedit/jobs/processidentifiers/InstanceFetcher.java

@@ -0,0 +1,51 @@
+package org.folio.dew.batch.bulkedit.jobs.processidentifiers;
+
+import feign.codec.DecodeException;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.folio.dew.client.InventoryInstancesClient;
+import org.folio.dew.domain.dto.IdentifierType;
+import org.folio.dew.domain.dto.InstanceCollection;
+import org.folio.dew.domain.dto.ItemIdentifier;
+import org.folio.dew.error.BulkEditException;
+import org.folio.dew.utils.ExceptionHelper;
+import org.jetbrains.annotations.NotNull;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.folio.dew.domain.dto.IdentifierType.HOLDINGS_RECORD_ID;
+import static org.folio.dew.utils.BulkEditProcessorHelper.getMatchPattern;
+import static org.folio.dew.utils.BulkEditProcessorHelper.resolveIdentifier;
+
+@Component
+@StepScope
+@RequiredArgsConstructor
+@Log4j2
+public class InstanceFetcher implements ItemProcessor<ItemIdentifier, InstanceCollection> {
+  private final InventoryInstancesClient inventoryInstancesClient;
+
+  @Value("#{jobParameters['identifierType']}")
+  private String identifierType;
+
+  private final Set<ItemIdentifier> identifiersToCheckDuplication = new HashSet<>();
+
+  @Override
+  public InstanceCollection process(@NotNull ItemIdentifier itemIdentifier) throws BulkEditException {
+    if (identifiersToCheckDuplication.contains(itemIdentifier)) {
+      throw new BulkEditException("Duplicate entry");
+    }
+    identifiersToCheckDuplication.add(itemIdentifier);
+    var limit = HOLDINGS_RECORD_ID == IdentifierType.fromValue(identifierType) ? Integer.MAX_VALUE : 1;
+    var idType = resolveIdentifier(identifierType);
+    try {
+      return inventoryInstancesClient.getInstanceByQuery(String.format(getMatchPattern(identifierType), idType, itemIdentifier.getItemId()), limit);
+    } catch (DecodeException e) {
+      throw new BulkEditException(ExceptionHelper.fetchMessage(e));
+    }
+  }
+}

src/main/java/org/folio/dew/client/InstanceFormatsClient.java

@@ -0,0 +1,14 @@
+package org.folio.dew.client;
+
+import org.folio.dew.config.feign.FeignClientConfiguration;
+import org.folio.dew.domain.dto.FormatOfInstance;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+
+@FeignClient(name = "instance-formats", configuration = FeignClientConfiguration.class)
+public interface InstanceFormatsClient {
+  @GetMapping(value = "/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
+  FormatOfInstance getById(@PathVariable String id);
+}

src/main/java/org/folio/dew/client/InstanceModeOfIssuanceClient.java

@@ -0,0 +1,14 @@
+package org.folio.dew.client;
+
+import org.folio.dew.config.feign.FeignClientConfiguration;
+import org.folio.dew.domain.dto.IssuanceMode;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+
+@FeignClient(name = "modes-of-issuance", configuration = FeignClientConfiguration.class)
+public interface InstanceModeOfIssuanceClient {
+  @GetMapping(value = "/{modeOfIssuanceId}", produces = MediaType.APPLICATION_JSON_VALUE)
+  IssuanceMode getById(@PathVariable String modeOfIssuanceId);
+}

src/main/java/org/folio/dew/client/InstanceStatusesClient.java

@@ -0,0 +1,14 @@
+package org.folio.dew.client;
+
+import org.folio.dew.config.feign.FeignClientConfiguration;
+import org.folio.dew.domain.dto.InstanceStatus;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+
+@FeignClient(name = "instance-statuses", configuration = FeignClientConfiguration.class)
+public interface InstanceStatusesClient {
+  @GetMapping(value = "/{instanceStatusId}", produces = MediaType.APPLICATION_JSON_VALUE)
+  InstanceStatus getById(@PathVariable String instanceStatusId);
+}

src/main/java/org/folio/dew/client/InstanceTypesClient.java

@@ -0,0 +1,13 @@
+package org.folio.dew.client;
+
+import org.folio.dew.config.feign.FeignClientConfiguration;
+import org.folio.dew.domain.dto.InstanceType;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+@FeignClient(name = "instance-types", configuration = FeignClientConfiguration.class)
+public interface InstanceTypesClient {
+  @GetMapping(value = "/{id}", produces = MediaType.APPLICATION_JSON_VALUE)
+  InstanceType getById(@PathVariable String id);
+}