Merge pull request #80 from chris9692/master

Derived fields can have int not integer because avro supports only int
linkedin · Nov 29, 2022 · cd1177a · cd1177a
2 parents 2c1911d + cf59856
commit cd1177a
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 5 deletions.
diff --git a/cdi-core/src/main/java/com/linkedin/cdi/extractor/JsonExtractor.java b/cdi-core/src/main/java/com/linkedin/cdi/extractor/JsonExtractor.java
@@ -326,7 +326,7 @@ private JsonObject addDerivedFields(JsonObject row) {
         case KEY_WORD_BOOLEAN:
           row.addProperty(name, Boolean.parseBoolean(strValue));
           break;
-        case KEY_WORD_INTEGER:
+        case KEY_WORD_INT:
           row.addProperty(name, Integer.parseInt(strValue));
           break;
         case KEY_WORD_NUMBER:

diff --git a/cdi-core/src/main/java/com/linkedin/cdi/extractor/MultistageExtractor.java b/cdi-core/src/main/java/com/linkedin/cdi/extractor/MultistageExtractor.java
@@ -76,7 +76,7 @@ public class MultistageExtractor<S, D> implements Extractor<S, D> {
   protected final static String PXD = "P\\d+D";
   protected final static String CONTENT_TYPE_KEY = "Content-Type";
   protected final static List<String> SUPPORTED_DERIVED_FIELD_TYPES =
-      Arrays.asList(KEY_WORD_EPOC, KEY_WORD_STRING, KEY_WORD_REGEXP, KEY_WORD_BOOLEAN, KEY_WORD_INTEGER, KEY_WORD_NUMBER);
+      Arrays.asList(KEY_WORD_EPOC, KEY_WORD_STRING, KEY_WORD_REGEXP, KEY_WORD_BOOLEAN, KEY_WORD_INT, KEY_WORD_NUMBER);
   protected static final String COMMA_STR = ",";
   protected WorkUnitStatus workUnitStatus = WorkUnitStatus.builder().build();
   protected WorkUnitState state = null;
@@ -482,7 +482,7 @@ protected JsonArray addDerivedFieldsToAltSchema() {
           dataType.addProperty(KEY_WORD_TYPE, "long");
           break;
         case KEY_WORD_STRING:
-        case KEY_WORD_INTEGER:
+        case KEY_WORD_INT:
         case KEY_WORD_NUMBER:
         case KEY_WORD_BOOLEAN:
           dataType.addProperty(KEY_WORD_TYPE, entry.getValue().get(KEY_WORD_TYPE));

diff --git a/cdi-core/src/test/java/com/linkedin/cdi/extractor/JsonExtractorTest.java b/cdi-core/src/test/java/com/linkedin/cdi/extractor/JsonExtractorTest.java
@@ -356,7 +356,7 @@ public void testAddDerivedFields() throws Exception {
         "dateTimeString",
         ImmutableMap.of("type", "string", "source", "{{dateTimeString}}"),
         "someInteger",
-        ImmutableMap.of("type", "integer", "source", "{{someInteger}}"),
+        ImmutableMap.of("type", "int", "source", "{{someInteger}}"),
         "someEpoc",
         ImmutableMap.of("type", "epoc", "source", "{{someEpoc}}"),
         "someNumber",

diff --git a/docs/parameters/ms.derived.fields.md b/docs/parameters/ms.derived.fields.md
@@ -34,7 +34,7 @@ DIL supports 6 types of derived fields:
 - `epoc`: it provides a time value field in the form of epoch (millisecond level). 
 - `regexp`: it provides a string value field by extracting the value from another field using a Java REGEXP pattern
 - `string`: it provides a string value field by taking from another field, which can be a nested field, without transformation
-- `integer`: it provides a integer value field by taking from another integer field, which can be a nested field, without transformation
+- `int`: it provides a integer value field by taking from another integer field, which can be a nested field, without transformation
 - `number`: it provides a number value field by taking from another number field, which can be a nested field, without transformation
 - `boolean`: it provides a boolean value field by taking from another boolean field, which can be a nested field, without transformation