Skip to content

Commit

Permalink
[MODEXPW-529] Create CSV file for claims (#609)
Browse files Browse the repository at this point in the history
* [MODEXPW-529] Rename and restructure converters and mapper

* [MODEXPW-529] Rename converters

* [MODEXPW-529] Add csv mapper and converter

* [MODEXPW-529] Use new mapper together with old one in tasklets

* [MODEXPW-529] Simplify existing acq tests to speed up builds

* [MODEXPW-529] Simplify edifact mapper test

* [MODEXPW-529] Add CsvMapperTest (WIP)

* [MODEXPW-529] Fix failing tests

* [MODEXPW-529] Move tests to appropriate packages

* [MODEXPW-529] Finish CsvMapper test, todo 2 field extractions

* [MODEXPW-529] Fix sonar issues

* [MODEXPW-529] Add title extraction logic

* [MODEXPW-529] Add quantity calculation logic

* [MODEXPW-529] Update pointer

* [MODEXPW-529] Fix sonar issue

* [MODEXPW-529] Improve grouping logic

* [MODEXPW-529] Update pointer and rename export config
  • Loading branch information
Saba-Zedginidze-EPAM authored Dec 19, 2024
1 parent ee9b9f1 commit 882cc76
Show file tree
Hide file tree
Showing 41 changed files with 665 additions and 236 deletions.
2 changes: 1 addition & 1 deletion folio-export-common
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package org.folio.dew.batch.acquisitions.edifact.config;

import org.folio.dew.batch.acquisitions.edifact.mapper.CsvMapper;
import org.folio.dew.batch.acquisitions.edifact.mapper.ExportResourceMapper;
import org.folio.dew.batch.acquisitions.edifact.mapper.converter.CompOrderEdiConverter;
import org.folio.dew.batch.acquisitions.edifact.mapper.converter.CompPoLineEdiConverter;
import org.folio.dew.batch.acquisitions.edifact.mapper.EdifactMapper;
import org.folio.dew.batch.acquisitions.edifact.services.ConfigurationService;
import org.folio.dew.batch.acquisitions.edifact.services.ExpenseClassService;
import org.folio.dew.batch.acquisitions.edifact.services.HoldingService;
import org.folio.dew.batch.acquisitions.edifact.services.IdentifierTypeService;
import org.folio.dew.batch.acquisitions.edifact.services.LocationService;
import org.folio.dew.batch.acquisitions.edifact.services.MaterialTypeService;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;

@Configuration
@ComponentScan({ "org.folio.dew.batch.acquisitions.edifact" })
public class AcquisitionExportConfig {

@Bean
CompPoLineEdiConverter compositePOLineConverter(IdentifierTypeService identifierTypeService, MaterialTypeService materialTypeService,
ExpenseClassService expenseClassService, LocationService locationService, HoldingService holdingService) {
return new CompPoLineEdiConverter(identifierTypeService, materialTypeService, expenseClassService, locationService, holdingService);
}

@Bean
CompOrderEdiConverter compositePurchaseOrderConverter(CompPoLineEdiConverter compPoLineEdiConverter, ConfigurationService configurationService) {
return new CompOrderEdiConverter(compPoLineEdiConverter, configurationService);
}

@Bean
ExportResourceMapper edifactMapper(CompOrderEdiConverter compOrderEdiConverter) {
return new EdifactMapper(compOrderEdiConverter);
}

@Bean
ExportResourceMapper csvMapper(OrdersService ordersService) {
return new CsvMapper(ordersService);
}

}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
import java.util.Map;

import org.apache.commons.collections4.CollectionUtils;
import org.folio.dew.batch.acquisitions.edifact.PurchaseOrdersToEdifactMapper;
import org.folio.dew.batch.acquisitions.edifact.mapper.ExportResourceMapper;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.folio.dew.domain.dto.Piece;
import org.folio.dew.domain.dto.VendorEdiOrdersExportConfig;
import org.folio.dew.domain.dto.acquisitions.edifact.EdifactExportHolder;
import org.folio.dew.domain.dto.acquisitions.edifact.ExportHolder;
import org.folio.dew.error.NotFoundException;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.scope.context.ChunkContext;
Expand All @@ -23,28 +23,41 @@
public class MapToEdifactClaimsTasklet extends MapToEdifactTasklet {

public static final String CLAIM_PIECE_IDS = "claimPieceIds";
private final ExportResourceMapper edifactMapper;
private final ExportResourceMapper csvMapper;

public MapToEdifactClaimsTasklet(ObjectMapper ediObjectMapper, OrdersService ordersService,
PurchaseOrdersToEdifactMapper purchaseOrdersToEdifactMapper) {
super(ediObjectMapper, ordersService, purchaseOrdersToEdifactMapper);
ExportResourceMapper edifactMapper, ExportResourceMapper csvMapper) {
super(ediObjectMapper, ordersService);
this.edifactMapper = edifactMapper;
this.csvMapper = csvMapper;
}

@Override
protected ExportResourceMapper getExportResourceMapper(VendorEdiOrdersExportConfig ediOrdersExportConfig) {
return switch (ediOrdersExportConfig.getFileFormat()) {
case EDI -> edifactMapper;
case CSV -> csvMapper;
};
}

@Override
protected List<String> getExportConfigMissingFields(VendorEdiOrdersExportConfig ediOrdersExportConfig) {
return CollectionUtils.isEmpty(ediOrdersExportConfig.getClaimPieceIds())
? List.of(CLAIM_PIECE_IDS)
: List.of();
}

@Override
protected EdifactExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig, Map<String, Object> jobParameters) {
protected ExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig, Map<String, Object> jobParameters) {
var pieces = ordersService.getPiecesByIdsAndReceivingStatus(ediExportConfig.getClaimPieceIds(), Piece.ReceivingStatusEnum.LATE);
if (pieces.isEmpty()) {
throw new NotFoundException(Piece.class);
}

var poLineQuery = convertIdsToCqlQuery(pieces.stream().map(Piece::getPoLineId).toList());
var compOrders = getCompositeOrders(poLineQuery);
return new EdifactExportHolder(compOrders, pieces);
return new ExportHolder(compOrders, pieces);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@
import java.util.stream.Collectors;

import org.apache.commons.collections4.CollectionUtils;
import org.folio.dew.batch.acquisitions.edifact.PurchaseOrdersToEdifactMapper;
import org.folio.dew.batch.acquisitions.edifact.mapper.ExportResourceMapper;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.folio.dew.client.DataExportSpringClient;
import org.folio.dew.domain.dto.ExportConfigCollection;
import org.folio.dew.domain.dto.ExportType;
import org.folio.dew.domain.dto.VendorEdiOrdersExportConfig;
import org.folio.dew.domain.dto.acquisitions.edifact.EdifactExportHolder;
import org.folio.dew.domain.dto.acquisitions.edifact.ExportHolder;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.stereotype.Component;
Expand All @@ -34,23 +34,31 @@
public class MapToEdifactOrdersTasklet extends MapToEdifactTasklet {

private final DataExportSpringClient dataExportSpringClient;
private final ExportResourceMapper edifactMapper;

public MapToEdifactOrdersTasklet(ObjectMapper ediObjectMapper, OrdersService ordersService,
DataExportSpringClient dataExportSpringClient,
PurchaseOrdersToEdifactMapper purchaseOrdersToEdifactMapper) {
super(ediObjectMapper, ordersService, purchaseOrdersToEdifactMapper);
ExportResourceMapper edifactMapper) {
super(ediObjectMapper, ordersService);
this.edifactMapper = edifactMapper;
this.dataExportSpringClient = dataExportSpringClient;
}

@Override
protected List<String> getExportConfigMissingFields(VendorEdiOrdersExportConfig ediOrdersExportConfig) {
return List.of();
}

@Override
protected EdifactExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig, Map<String, Object> jobParameters) {
protected ExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig, Map<String, Object> jobParameters) {
var poLineQuery = getPoLineQuery(ediExportConfig);
var compOrders = getCompositeOrders(poLineQuery);
return new EdifactExportHolder(compOrders, List.of());
return new ExportHolder(compOrders, List.of());
}

@Override
protected ExportResourceMapper getExportResourceMapper(VendorEdiOrdersExportConfig ediOrdersExportConfig) {
return edifactMapper;
}

protected String getPoLineQuery(VendorEdiOrdersExportConfig ediConfig) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,17 @@

import org.apache.commons.lang3.StringUtils;
import org.folio.dew.batch.ExecutionContextUtils;
import org.folio.dew.batch.acquisitions.edifact.PurchaseOrdersToEdifactMapper;
import org.folio.dew.batch.acquisitions.edifact.exceptions.CompositeOrderMappingException;
import org.folio.dew.batch.acquisitions.edifact.exceptions.EdifactException;
import org.folio.dew.batch.acquisitions.edifact.mapper.ExportResourceMapper;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.folio.dew.domain.dto.CompositePoLine;
import org.folio.dew.domain.dto.CompositePurchaseOrder;
import org.folio.dew.domain.dto.JobParameterNames;
import org.folio.dew.domain.dto.PoLine;
import org.folio.dew.domain.dto.PurchaseOrder;
import org.folio.dew.domain.dto.VendorEdiOrdersExportConfig;
import org.folio.dew.domain.dto.acquisitions.edifact.EdifactExportHolder;
import org.folio.dew.domain.dto.acquisitions.edifact.ExportHolder;
import org.folio.dew.error.NotFoundException;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
Expand All @@ -41,7 +41,6 @@ public abstract class MapToEdifactTasklet implements Tasklet {

private final ObjectMapper ediObjectMapper;
protected final OrdersService ordersService;
private final PurchaseOrdersToEdifactMapper purchaseOrdersToEdifactMapper;

@Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
Expand All @@ -54,7 +53,7 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon
persistPoLineIds(chunkContext, holder.orders());

String jobName = jobParameters.get(JobParameterNames.JOB_NAME).toString();
var edifactStringResult = purchaseOrdersToEdifactMapper.convertOrdersToEdifact(holder.orders(), holder.pieces(), ediExportConfig, jobName);
var edifactStringResult = getExportResourceMapper(ediExportConfig).convertForExport(holder.orders(), holder.pieces(), ediExportConfig, jobName);

// save edifact file content in memory
ExecutionContextUtils.addToJobExecutionContext(chunkContext.getStepContext().getStepExecution(), "edifactOrderAsString", edifactStringResult, "");
Expand Down Expand Up @@ -122,9 +121,11 @@ private <T> T convertTo(Object value, Class<T> c) {
}
}

protected abstract ExportResourceMapper getExportResourceMapper(VendorEdiOrdersExportConfig ediOrdersExportConfig);

protected abstract List<String> getExportConfigMissingFields(VendorEdiOrdersExportConfig ediOrdersExportConfig);

protected abstract EdifactExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig,
Map<String, Object> jobParameters) throws JsonProcessingException, EDIStreamException;
protected abstract ExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig,
Map<String, Object> jobParameters) throws JsonProcessingException, EDIStreamException;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package org.folio.dew.batch.acquisitions.edifact.mapper;

import static java.util.stream.Collectors.groupingBy;
import static org.folio.dew.utils.Constants.LINE_BREAK;

import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.apache.commons.lang3.tuple.Pair;
import org.folio.dew.batch.acquisitions.edifact.mapper.converter.ClaimCsvConverter;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.folio.dew.domain.dto.CompositePoLine;
import org.folio.dew.domain.dto.CompositePurchaseOrder;
import org.folio.dew.domain.dto.Piece;
import org.folio.dew.domain.dto.VendorEdiOrdersExportConfig;
import org.folio.dew.domain.dto.acquisitions.edifact.ClaimCsvEntry;

public class CsvMapper implements ExportResourceMapper {

private final OrdersService ordersService;

public CsvMapper(OrdersService ordersService) {
this.ordersService = ordersService;
}

@Override
public String convertForExport(List<CompositePurchaseOrder> compPOs, List<Piece> pieces, VendorEdiOrdersExportConfig ediExportConfig, String jobName) {
var claimCsvConverter = new ClaimCsvConverter();
var csvResult = new StringBuilder(claimCsvConverter.getCsvHeaders()).append(LINE_BREAK);
getClaimEntries(compPOs, pieces).stream()
.map(claimCsvConverter::convertEntryToCsv)
.map(line -> line.concat(LINE_BREAK))
.forEachOrdered(csvResult::append);
return csvResult.toString();
}

private List<ClaimCsvEntry> getClaimEntries(List<CompositePurchaseOrder> orders, List<Piece> pieces) {
// Map each PoLine ID to its corresponding Pieces
var poLineIdToPieces = pieces.stream().collect(groupingBy(Piece::getPoLineId));
// Map each PoLine ID to its corresponding PoLine
var poLineIdToPoLine = orders.stream().flatMap(order -> order.getCompositePoLines().stream())
.collect(Collectors.toMap(CompositePoLine::getId, Function.identity()));
// Map each Piece ID to its corresponding Title
var pieceIdToTitle = poLineIdToPieces.entrySet().stream()
.flatMap(entry -> entry.getValue().stream()
.map(piece -> Pair.of(piece.getId(), getTitleById(poLineIdToPoLine.get(entry.getKey()), piece)))) // Pair of Piece ID and Title
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight));

// Key extractor for grouping pieces by: Po Line Number, Display Summary, Chronology, Enumeration, and Title
Function<Piece, ClaimCsvEntry> keyExtractor = piece ->
new ClaimCsvEntry(poLineIdToPoLine.get(piece.getPoLineId()), piece, pieceIdToTitle.get(piece.getId()), 0);

// Group pieces by the previously defined key (Overridden equals and hashCode methods in ClaimCsvEntry)
// Only a single piece from each group is used, as they share all necessary attributes
Map<ClaimCsvEntry, Long> claimedPieces = pieces.stream()
.collect(Collectors.groupingBy(keyExtractor, Collectors.counting()));

// Return a list of ClaimCsvEntry objects, each representing a group of claimed pieces
return claimedPieces.entrySet().stream()
.map(entry -> entry.getKey().withQuantity(entry.getValue()))
.sorted(Comparator.comparing(o -> o.compositePoLine().getPoLineNumber()))
.toList();
}

private String getTitleById(CompositePoLine poLine, Piece piece) {
return Boolean.TRUE.equals(poLine.getIsPackage())
? ordersService.getTitleById(piece.getTitleId()).getTitle()
: poLine.getTitleOrPackage();
}

}
Loading

0 comments on commit 882cc76

Please sign in to comment.