From 18c64b0c06350142fdb537fe2de0f9045572ca12 Mon Sep 17 00:00:00 2001 From: Vitaliy Velikodniy Date: Sat, 30 Sep 2023 11:13:20 +0300 Subject: [PATCH] add ignoring errors modes --- README.md | 11 ++ build.gradle | 2 +- .../vitaliy/fixedlength/FixedLength.java | 180 +++++++++++------- src/test/java/ParserTest.java | 40 +++- 4 files changed, 161 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 4892a1e..5b871df 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,17 @@ List parse = new FixedLength() .registerLineType(Employee.class); ``` +### Ignoring errors + +If there is errors on your line format there are two modes that you could skip these errors if you want to: + +* `skipErroneousLines` — line with error will not be added to result. +* `skipErroneousFields` — fields with errors will be `null`. + +In both cases warnings will be raised in logs. + +By default, exception will be raised for entire process. + ### Cases to use In the case if you have 2 different records in one line and there is a split index you can add a method in your entity that should return index of the next record and mark it with annotation `SplitLineAfter`. diff --git a/build.gradle b/build.gradle index da021f0..c07b455 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ plugins { } group 'name.velikodniy.vitaliy' -version '0.8' +version '0.9' repositories { mavenLocal() diff --git a/src/main/java/name/velikodniy/vitaliy/fixedlength/FixedLength.java b/src/main/java/name/velikodniy/vitaliy/fixedlength/FixedLength.java index 2be9449..9b9a7c2 100644 --- a/src/main/java/name/velikodniy/vitaliy/fixedlength/FixedLength.java +++ b/src/main/java/name/velikodniy/vitaliy/fixedlength/FixedLength.java @@ -1,6 +1,5 @@ package name.velikodniy.vitaliy.fixedlength; -import java.util.Comparator; import name.velikodniy.vitaliy.fixedlength.annotation.FixedField; import name.velikodniy.vitaliy.fixedlength.annotation.FixedLine; import name.velikodniy.vitaliy.fixedlength.annotation.SplitLineAfter; @@ -12,15 +11,17 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Scanner; +import java.util.Comparator; +import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Scanner; import java.util.Spliterator; import java.util.Spliterators; -import java.util.Map; -import java.util.List; -import java.util.Arrays; -import java.util.ArrayList; +import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -30,6 +31,8 @@ public class FixedLength { + private static final Logger LOGGER = Logger.getLogger(FixedLength.class.getName()); + private static final Map< Class, Class> @@ -37,6 +40,8 @@ public class FixedLength { = Formatter.getDefaultFormatters(); private final List> lineTypes = new ArrayList<>(); private boolean skipUnknownLines = true; + private boolean skipErroneousFields = false; + private boolean skipErroneousLines = false; private Charset charset = Charset.defaultCharset(); private String delimiterString = "\n"; private Pattern delimiter = Pattern.compile(delimiterString); @@ -90,6 +95,16 @@ public FixedLength stopSkipUnknownLines() { return this; } + public FixedLength skipErroneousFields() { + skipErroneousFields = true; + return this; + } + + public FixedLength skipErroneousLines() { + skipErroneousLines = true; + return this; + } + public FixedLength registerLineTypes(final List> lineClasses) { lineTypes.addAll( lineClasses.stream() @@ -156,8 +171,8 @@ private T lineToObject(FixedFormatRecord fixedFormatRecord) { } catch (NoSuchMethodException e) { throw new FixedLengthException("No empty constructor in class", e); } catch (IllegalAccessException - | InstantiationException - | InvocationTargetException e) { + | InstantiationException + | InvocationTargetException e) { throw new FixedLengthException( "Unable to instantiate " + clazz.getName(), e ); @@ -176,21 +191,37 @@ private T lineToObject(FixedFormatRecord fixedFormatRecord) { endOfFieldIndex ), fieldAnnotation.padding()); if (acceptFieldContent(str, fieldAnnotation)) { - field.setAccessible(true); - - try { - field.set( - lineAsObject, - Formatter.instance(FORMATTERS, field.getType()).asObject(str, fieldAnnotation) - ); - } catch (IllegalAccessException e) { - throw new FixedLengthException("Access to field failed", e); - } + fillField(field, lineAsObject, str, fieldAnnotation); } } return lineAsObject; } + private void fillField(Field field, T lineAsObject, String str, FixedField fieldAnnotation) { + field.setAccessible(true); + + try { + field.set( + lineAsObject, + Formatter.instance(FORMATTERS, field.getType()).asObject(str, fieldAnnotation) + ); + } catch (IllegalAccessException e) { + throw new FixedLengthException("Access to field failed", e); + } catch (Exception e) { + if (e instanceof FixedLengthException) { + throw e; + } + if (!skipErroneousFields) { + throw e; + } + LOGGER.warning(String.format( + "Skipping field of type %s with error in value %s", + field.getType(), + str + )); + } + } + private boolean acceptFieldContent(String content, FixedField fieldAnnotation) { if (content == null) { return false; @@ -199,38 +230,49 @@ private boolean acceptFieldContent(String content, FixedField fieldAnnotation) { return false; } if (fieldAnnotation.ignore().isEmpty()) { - // No ignore cotent defined, accepting + // No ignore content defined, accepting return true; } - // Ignore cotent defined: accepting if not matching ignore regular expression + // Ignore content defined: accepting if not matching ignore regular expression Pattern pattern = Pattern.compile(fieldAnnotation.ignore()); return !pattern.matcher(content).matches(); } private List lineToObjects(FixedFormatRecord fixedFormatRecord) { - T lineAsObject = this.lineToObject(fixedFormatRecord); - Method splitMethod = fixedFormatRecord.fixedFormatLine.splitAfterMethod; - if (splitMethod == null) { - return Collections.singletonList(lineAsObject); - } - int splitIndex; try { - splitIndex = (Integer) splitMethod.invoke(lineAsObject); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new FixedLengthException("Access to method failed", e); - } - if (splitIndex <= 0 || splitIndex >= fixedFormatRecord.rawLine.length()) { - return Collections.singletonList(lineAsObject); - } - String subRawLine = fixedFormatRecord.rawLine.substring(splitIndex); - FixedFormatRecord subRecord = this.fixedFormatLine(subRawLine); - if (subRecord == null) { - return Collections.singletonList(lineAsObject); + T lineAsObject = this.lineToObject(fixedFormatRecord); + Method splitMethod = fixedFormatRecord.fixedFormatLine.splitAfterMethod; + if (splitMethod == null) { + return Collections.singletonList(lineAsObject); + } + int splitIndex; + try { + splitIndex = (Integer) splitMethod.invoke(lineAsObject); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new FixedLengthException("Access to method failed", e); + } + if (splitIndex <= 0 || splitIndex >= fixedFormatRecord.rawLine.length()) { + return Collections.singletonList(lineAsObject); + } + String subRawLine = fixedFormatRecord.rawLine.substring(splitIndex); + FixedFormatRecord subRecord = this.fixedFormatLine(subRawLine); + if (subRecord == null) { + return Collections.singletonList(lineAsObject); + } + List lineAsObjects = new ArrayList<>(); + lineAsObjects.add(lineAsObject); + lineAsObjects.addAll(lineToObjects(subRecord)); + return lineAsObjects; + } catch (Exception e) { + if (e instanceof FixedLengthException) { + throw e; + } + if (!skipErroneousLines) { + throw e; + } + LOGGER.warning("Skipping line with error"); + return Collections.emptyList(); } - List lineAsObjects = new ArrayList<>(); - lineAsObjects.add(lineAsObject); - lineAsObjects.addAll(lineToObjects(subRecord)); - return lineAsObjects; } public List parse(InputStream stream) throws FixedLengthException { @@ -265,34 +307,34 @@ public String format(List lines) { for (T line : lines) { Arrays.stream(line.getClass().getDeclaredFields()) - .filter( - f -> - f.getAnnotation(FixedField.class) != null - ) - .sorted(Comparator.comparingInt(f -> f.getAnnotation(FixedField.class).offset())) - .forEach(f -> { - FixedField fixedFieldAnnotation = f.getAnnotation(FixedField.class); - - Formatter formatter = (Formatter) Formatter.instance(FORMATTERS, f.getType()); - - f.setAccessible(true); - - T value; - try { - value = (T) f.get(line); - } catch (IllegalAccessException e) { - throw new FixedLengthException(e.getMessage(), e); - } - - if (value != null) { - builder.append( - fixedFieldAnnotation.align().make( - formatter.asString(value, fixedFieldAnnotation), - fixedFieldAnnotation.length(), - fixedFieldAnnotation.padding()) - ); - } - }); + .filter( + f -> + f.getAnnotation(FixedField.class) != null + ) + .sorted(Comparator.comparingInt(f -> f.getAnnotation(FixedField.class).offset())) + .forEach(f -> { + FixedField fixedFieldAnnotation = f.getAnnotation(FixedField.class); + + Formatter formatter = (Formatter) Formatter.instance(FORMATTERS, f.getType()); + + f.setAccessible(true); + + T value; + try { + value = (T) f.get(line); + } catch (IllegalAccessException e) { + throw new FixedLengthException(e.getMessage(), e); + } + + if (value != null) { + builder.append( + fixedFieldAnnotation.align().make( + formatter.asString(value, fixedFieldAnnotation), + fixedFieldAnnotation.length(), + fixedFieldAnnotation.padding()) + ); + } + }); if (lines.size() != currentLine++) { builder.append(this.delimiterString); diff --git a/src/test/java/ParserTest.java b/src/test/java/ParserTest.java index 349c413..9109e2a 100644 --- a/src/test/java/ParserTest.java +++ b/src/test/java/ParserTest.java @@ -6,13 +6,13 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.time.LocalDate; +import java.time.format.DateTimeParseException; import java.util.List; import java.util.regex.Pattern; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.*; class ParserTest { @@ -20,6 +20,10 @@ class ParserTest { "Joe1 Smith Developer 07500010012009\n" + "Joe3 Smith Developer "; + String singleTypeWithErrorExample = + "Joe1 Smith Developer 07500010012009\n" + + "Joe1 Smith Developer 07500013012009"; + String mixedTypesExample = "EmplJoe1 Smith Developer 07500010012009\n" + "CatSnowball 20200103\n" + @@ -51,6 +55,38 @@ void testParseOneLineType() throws FixedLengthException { assertEquals(2, parse.size()); } + @Test + @DisplayName("Parse as input stream with throwing exception when format erroneous fields") + void testParseThrowsExceptionOnInvalidFormat() throws FixedLengthException { + assertThrows(DateTimeParseException.class, () -> + new FixedLength() + .registerLineType(Employee.class) + .parse(new ByteArrayInputStream(singleTypeWithErrorExample.getBytes()))); + } + + @Test + @DisplayName("Parse as input stream with skipping format erroneous fields") + void testParseWithSkippingErroneousFields() throws FixedLengthException { + List parse = new FixedLength() + .registerLineType(Employee.class) + .skipErroneousFields() + .parse(new ByteArrayInputStream(singleTypeWithErrorExample.getBytes())); + + assertEquals(2, parse.size()); + assertNull(((Employee) parse.get(1)).hireDate); + } + + @Test + @DisplayName("Parse as input stream with skipping format erroneous lines") + void testParseWithSkippingErroneousLines() throws FixedLengthException { + List parse = new FixedLength() + .registerLineType(Employee.class) + .skipErroneousLines() + .parse(new ByteArrayInputStream(singleTypeWithErrorExample.getBytes())); + + assertEquals(1, parse.size()); + } + @Test @DisplayName("Parse as input stream with default charset and one line type") void testParseOneLineTypeUS_ACII() throws FixedLengthException {