Skip to content

Commit

Permalink
remove source information on Pure parsing to optimize storage size (#464
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@akphi
akphi authored Jul 21, 2022
1 parent ef4a56d commit 265d10f
Show file tree
Hide file tree
Showing 18 changed files with 207 additions and 1,363 deletions.
4 changes: 4 additions & 0 deletions legend-sdlc-entity-maven-plugin/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
Expand Down
11 changes: 8 additions & 3 deletions ...ity-maven-plugin/src/test/java/org/finos/legend/sdlc/entities/PureDomainDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package org.finos.legend.sdlc.entities;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.eclipse.collections.impl.utility.Iterate;
import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser;
Expand All @@ -38,7 +39,7 @@
public class PureDomainDeserializer implements EntityTextSerializer
{
private final PureGrammarParser pureParser = PureGrammarParser.newInstance();
private final ObjectMapper jsonMapper = PureProtocolObjectMapperFactory.getNewObjectMapper();
private final ObjectMapper jsonMapper = PureProtocolObjectMapperFactory.getNewObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL);

@Override
public String getName()
Expand Down Expand Up @@ -83,7 +84,11 @@ public Entity deserialize(String content) throws IOException
PureModelContextData pureModelContextData;
try
{
pureModelContextData = this.pureParser.parseModel(content);
pureModelContextData = this.pureParser.parseModel(
content,
// NOTE: remove source information to optimize model size for storage
false
);
}
catch (EngineException e)
{
Expand All @@ -97,7 +102,7 @@ public Entity deserialize(String content) throws IOException
PackageableElement element = elements.get((elements.get(0) instanceof SectionIndex) ? 1 : 0);
String classifierPath = getClassifierPath(element);
String intermediateJson = this.jsonMapper.writeValueAsString(element);
Map<String, Object> entityContent = this.jsonMapper.readValue(intermediateJson, this.jsonMapper.getTypeFactory().constructMapType(Map.class, String.class, Object.class));
Map<String, Object> entityContent = this.jsonMapper.readValue(intermediateJson, this.jsonMapper.getTypeFactory().constructMapType(Map.class, String.class, Object.class));
return Entity.newEntity(element.getPath(), classifierPath, entityContent);
}

Expand Down
43 changes: 2 additions & 41 deletions ...n/src/test/resources/simple-json-model/entities/model/domain/associations/Employment.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,64 +7,25 @@
"package": "model::domain::associations",
"properties": [
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 0,
"upperBound": null
"lowerBound": 0
},
"name": "employer",
"propertyTypeSourceInformation": {
"endColumn": 31,
"endLine": 3,
"sourceId": "",
"startColumn": 13,
"startLine": 3
},
"sourceInformation": {
"endColumn": 35,
"endLine": 3,
"sourceId": "",
"startColumn": 3,
"startLine": 3
},
"stereotypes": [],
"taggedValues": [],
"type": "model::domain::Firm"
},
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 0,
"upperBound": null
"lowerBound": 0
},
"name": "employees",
"propertyTypeSourceInformation": {
"endColumn": 34,
"endLine": 4,
"sourceId": "",
"startColumn": 14,
"startLine": 4
},
"sourceInformation": {
"endColumn": 38,
"endLine": 4,
"sourceId": "",
"startColumn": 3,
"startLine": 4
},
"stereotypes": [],
"taggedValues": [],
"type": "model::domain::Person"
}
],
"qualifiedProperties": [],
"sourceInformation": {
"endColumn": 1,
"endLine": 5,
"sourceId": "",
"startColumn": 1,
"startLine": 1
},
"stereotypes": [],
"taggedValues": []
}
Expand Down
37 changes: 0 additions & 37 deletions ...en-plugin/src/test/resources/simple-json-model/entities/model/domain/classes/Address.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,64 +8,27 @@
"package": "model::domain::classes",
"properties": [
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 1,
"upperBound": 1
},
"name": "type",
"propertyTypeSourceInformation": {
"endColumn": 41,
"endLine": 3,
"sourceId": "",
"startColumn": 9,
"startLine": 3
},
"sourceInformation": {
"endColumn": 45,
"endLine": 3,
"sourceId": "",
"startColumn": 3,
"startLine": 3
},
"stereotypes": [],
"taggedValues": [],
"type": "model::domain::enums::AddressType"
},
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 1,
"upperBound": 1
},
"name": "address",
"propertyTypeSourceInformation": {
"endColumn": 17,
"endLine": 4,
"sourceId": "",
"startColumn": 12,
"startLine": 4
},
"sourceInformation": {
"endColumn": 21,
"endLine": 4,
"sourceId": "",
"startColumn": 3,
"startLine": 4
},
"stereotypes": [],
"taggedValues": [],
"type": "String"
}
],
"qualifiedProperties": [],
"sourceInformation": {
"endColumn": 1,
"endLine": 5,
"sourceId": "",
"startColumn": 1,
"startLine": 1
},
"stereotypes": [],
"superTypes": [],
"taggedValues": []
Expand Down
25 changes: 1 addition & 24 deletions ...c/test/resources/simple-json-model/entities/model/domain/classes/EntityWithAddresses.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,39 +8,16 @@
"package": "model::domain::classes",
"properties": [
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 0,
"upperBound": null
"lowerBound": 0
},
"name": "addresses",
"propertyTypeSourceInformation": {
"endColumn": 44,
"endLine": 3,
"sourceId": "",
"startColumn": 14,
"startLine": 3
},
"sourceInformation": {
"endColumn": 48,
"endLine": 3,
"sourceId": "",
"startColumn": 3,
"startLine": 3
},
"stereotypes": [],
"taggedValues": [],
"type": "model::domain::classes::Address"
}
],
"qualifiedProperties": [],
"sourceInformation": {
"endColumn": 1,
"endLine": 4,
"sourceId": "",
"startColumn": 1,
"startLine": 1
},
"stereotypes": [],
"superTypes": [],
"taggedValues": []
Expand Down
52 changes: 0 additions & 52 deletions ...maven-plugin/src/test/resources/simple-json-model/entities/model/domain/classes/Firm.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,89 +8,37 @@
"package": "model::domain::classes",
"properties": [
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 1,
"upperBound": 1
},
"name": "legalName",
"propertyTypeSourceInformation": {
"endColumn": 19,
"endLine": 3,
"sourceId": "",
"startColumn": 14,
"startLine": 3
},
"sourceInformation": {
"endColumn": 23,
"endLine": 3,
"sourceId": "",
"startColumn": 3,
"startLine": 3
},
"stereotypes": [],
"taggedValues": [],
"type": "String"
},
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 0,
"upperBound": 1
},
"name": "commonName",
"propertyTypeSourceInformation": {
"endColumn": 20,
"endLine": 4,
"sourceId": "",
"startColumn": 15,
"startLine": 4
},
"sourceInformation": {
"endColumn": 27,
"endLine": 4,
"sourceId": "",
"startColumn": 3,
"startLine": 4
},
"stereotypes": [],
"taggedValues": [],
"type": "String"
},
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 0,
"upperBound": 1
},
"name": "founded",
"propertyTypeSourceInformation": {
"endColumn": 21,
"endLine": 5,
"sourceId": "",
"startColumn": 12,
"startLine": 5
},
"sourceInformation": {
"endColumn": 28,
"endLine": 5,
"sourceId": "",
"startColumn": 3,
"startLine": 5
},
"stereotypes": [],
"taggedValues": [],
"type": "StrictDate"
}
],
"qualifiedProperties": [],
"sourceInformation": {
"endColumn": 1,
"endLine": 6,
"sourceId": "",
"startColumn": 1,
"startLine": 1
},
"stereotypes": [],
"superTypes": [
"model::domain::classes::EntityWithAddresses"
Expand Down
37 changes: 0 additions & 37 deletions ...ven-plugin/src/test/resources/simple-json-model/entities/model/domain/classes/Person.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,64 +8,27 @@
"package": "model::domain::classes",
"properties": [
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 1,
"upperBound": 1
},
"name": "firstName",
"propertyTypeSourceInformation": {
"endColumn": 19,
"endLine": 3,
"sourceId": "",
"startColumn": 14,
"startLine": 3
},
"sourceInformation": {
"endColumn": 23,
"endLine": 3,
"sourceId": "",
"startColumn": 3,
"startLine": 3
},
"stereotypes": [],
"taggedValues": [],
"type": "String"
},
{
"defaultValue": null,
"multiplicity": {
"lowerBound": 1,
"upperBound": 1
},
"name": "lastName",
"propertyTypeSourceInformation": {
"endColumn": 18,
"endLine": 4,
"sourceId": "",
"startColumn": 13,
"startLine": 4
},
"sourceInformation": {
"endColumn": 22,
"endLine": 4,
"sourceId": "",
"startColumn": 3,
"startLine": 4
},
"stereotypes": [],
"taggedValues": [],
"type": "String"
}
],
"qualifiedProperties": [],
"sourceInformation": {
"endColumn": 1,
"endLine": 5,
"sourceId": "",
"startColumn": 1,
"startLine": 1
},
"stereotypes": [],
"superTypes": [
"model::domain::classes::EntityWithAddresses"
Expand Down
Loading

0 comments on commit 265d10f

Please sign in to comment.