A comprehensive error handling strategy of pulling entities -- part II

finos · Aug 22, 2023 · 184c8b1 · 184c8b1
1 parent 07e86c0
commit 184c8b1
Show file tree

Hide file tree

Showing 20 changed files with 535 additions and 0 deletions.
diff --git a/legend-sdlc-model/src/main/java/org/finos/legend/sdlc/domain/model/entity/InvalidEntity.java b/legend-sdlc-model/src/main/java/org/finos/legend/sdlc/domain/model/entity/InvalidEntity.java
@@ -0,0 +1,56 @@
+// Copyright 2022 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.sdlc.domain.model.entity;
+
+public interface InvalidEntity
+{
+    String getPath();
+
+    String getErrorMessage();
+
+    String getStackTrace();
+
+    String getSourceCode();
+
+    static InvalidEntity newInvalidEntity(String path, String errorMessage, String stackTrace,  String sourceCode)
+    {
+        return new InvalidEntity()
+        {
+            @Override
+            public String getPath()
+            {
+                return path;
+            }
+
+            @Override
+            public String getErrorMessage()
+            {
+                return errorMessage;
+            }
+
+            @Override
+            public String getStackTrace()
+            {
+                return stackTrace;
+            }
+
+            @Override
+            public String getSourceCode()
+            {
+                return sourceCode;
+            }
+        };
+    }
+}
diff --git a/...ver/src/main/java/org/finos/legend/sdlc/server/domain/api/entity/EntityAccessContext.java b/...ver/src/main/java/org/finos/legend/sdlc/server/domain/api/entity/EntityAccessContext.java
@@ -15,6 +15,7 @@
 package org.finos.legend.sdlc.server.domain.api.entity;
 
 import org.finos.legend.sdlc.domain.model.entity.Entity;
+import org.finos.legend.sdlc.domain.model.entity.InvalidEntity;
 
 import java.util.List;
 import java.util.Map;
@@ -31,5 +32,9 @@ default List<Entity> getEntities(Predicate<String> entityPathPredicate, Predicat
 
     List<Entity> getEntities(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate, boolean excludeInvalid);
 
+    InvalidEntity getInvalidEntity(String path);
+
+    List<InvalidEntity> getInvalidEntities();
+
     List<String> getEntityPaths(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate);
 }
diff --git a/...nd-sdlc-server/src/main/java/org/finos/legend/sdlc/server/gitlab/api/GitLabEntityApi.java b/...nd-sdlc-server/src/main/java/org/finos/legend/sdlc/server/gitlab/api/GitLabEntityApi.java
@@ -20,6 +20,7 @@
 import org.eclipse.collections.api.map.MutableMap;
 import org.eclipse.collections.impl.utility.ListIterate;
 import org.finos.legend.sdlc.domain.model.entity.Entity;
+import org.finos.legend.sdlc.domain.model.entity.InvalidEntity;
 import org.finos.legend.sdlc.domain.model.entity.change.EntityChange;
 import org.finos.legend.sdlc.domain.model.entity.change.EntityChangeType;
 import org.finos.legend.sdlc.domain.model.revision.Revision;
@@ -223,6 +224,50 @@ public List<Entity> getEntities(Predicate<String> entityPathPredicate, Predicate
             }
         }
 
+        @Override
+        public InvalidEntity getInvalidEntity(String path)
+        {
+            ProjectFileAccessProvider.FileAccessContext fileAccessContext = getFileAccessContext(getProjectFileAccessProvider());
+            ProjectStructure projectStructure = ProjectStructure.getProjectStructure(fileAccessContext);
+            for (ProjectStructure.EntitySourceDirectory sourceDirectory : projectStructure.getEntitySourceDirectories())
+            {
+                String filePath = sourceDirectory.entityPathToFilePath(path);
+                ProjectFileAccessProvider.ProjectFile file = fileAccessContext.getFile(filePath);
+                if (file != null)
+                {
+                    try
+                    {
+                        Entity localEntity = sourceDirectory.deserialize(file);
+                        if (!Objects.equals(localEntity.getPath(), path))
+                        {
+                            throw new RuntimeException("Expected entity path " + path + ", found " + localEntity.getPath());
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        return InvalidEntity.newInvalidEntity(path, e.getMessage(), e.getStackTrace().toString(), file.getContentAsString());
+                    }
+                }
+            }
+            throw new LegendSDLCServerException("Unknown invalid entity " + path + " for " + getInfoForException(), Status.NOT_FOUND);
+        }
+
+        @Override
+        public List<InvalidEntity> getInvalidEntities()
+        {
+            try (Stream<EntityProjectFile> stream = getEntityProjectFiles(getFileAccessContext(getProjectFileAccessProvider())))
+            {
+                return stream.map(EntityProjectFile::getInvalidEntity).filter(Objects::nonNull).collect(Collectors.toList());
+            }
+            catch (Exception e)
+            {
+                throw buildException(e,
+                        () -> "User " + getCurrentUser() + " is not allowed to get entities for " + getInfoForException(),
+                        () -> "Unknown entities for " + getInfoForException(),
+                        () -> "Failed to get entities for " + getInfoForException());
+            }
+        }
+
         @Override
         public List<String> getEntityPaths(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate)
         {
@@ -764,5 +809,25 @@ synchronized Entity getEntity()
             }
             return this.entity;
         }
+
+        synchronized InvalidEntity getInvalidEntity()
+        {
+            if (this.entity == null)
+            {
+                try
+                {
+                    Entity localEntity = this.sourceDirectory.deserialize(this.file);
+                    if (!Objects.equals(localEntity.getPath(), getEntityPath()))
+                    {
+                        throw new RuntimeException("Expected entity path " + getEntityPath() + ", found " + localEntity.getPath());
+                    }
+                }
+                catch (Exception e)
+                {
+                    return InvalidEntity.newInvalidEntity(getEntityPath(), e.getMessage(), e.getStackTrace().toString(), this.file.getContentAsString());
+                }
+            }
+            return null;
+        }
     }
 }
diff --git a/...dlc-server/src/main/java/org/finos/legend/sdlc/server/resources/EntityAccessResource.java b/...dlc-server/src/main/java/org/finos/legend/sdlc/server/resources/EntityAccessResource.java
@@ -16,6 +16,7 @@
 
 import org.eclipse.collections.api.factory.Maps;
 import org.finos.legend.sdlc.domain.model.entity.Entity;
+import org.finos.legend.sdlc.domain.model.entity.InvalidEntity;
 import org.finos.legend.sdlc.server.domain.api.entity.EntityAccessContext;
 
 import java.util.Collection;
@@ -45,6 +46,11 @@ protected List<Entity> getEntities(EntityAccessContext entityAccessContext, Set<
         return entityAccessContext.getEntities(entityPathPredicate, classifierPathPredicate, contentPredicate, excludeInvalidEntities);
     }
 
+    protected List<InvalidEntity> getInvalidEntities(EntityAccessContext entityAccessContext)
+    {
+        return entityAccessContext.getInvalidEntities();
+    }
+
     private Predicate<String> getEntityPathPredicate(Set<String> packages, boolean includeSubPackages, String nameRegex)
     {
         Pattern namePattern = compileRegex(nameRegex);

diff --git a/...gend/sdlc/server/resources/backup/project/group/GroupBackupWorkspaceEntitiesResource.java b/...gend/sdlc/server/resources/backup/project/group/GroupBackupWorkspaceEntitiesResource.java
@@ -18,6 +18,7 @@
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
 import org.finos.legend.sdlc.domain.model.entity.Entity;
+import org.finos.legend.sdlc.domain.model.entity.InvalidEntity;
 import org.finos.legend.sdlc.server.domain.api.entity.EntityApi;
 import org.finos.legend.sdlc.server.resources.EntityAccessResource;
 
@@ -84,4 +85,26 @@ public Entity getEntityByPath(@PathParam("projectId") String projectId, @PathPar
                 () -> this.entityApi.getBackupGroupWorkspaceEntityAccessContext(projectId, workspaceId).getEntity(path)
         );
     }
+
+    @GET
+    @Path("invalidEntities")
+    @ApiOperation("Get invalid entities of the backup group workspace")
+    public List<InvalidEntity> getInvalidEntities(@PathParam("projectId") String projectId, @PathParam("workspaceId") String workspaceId)
+    {
+        return executeWithLogging(
+                "getting invalid entities in backup group workspace " + workspaceId + " for project " + projectId,
+                () -> getInvalidEntities(this.entityApi.getBackupGroupWorkspaceEntityAccessContext(projectId, workspaceId))
+        );
+    }
+
+    @GET
+    @Path("invalidEntities/{path}")
+    @ApiOperation("Get an invalid entity of the backup group workspace by its path")
+    public InvalidEntity getInvalidEntityByPath(@PathParam("projectId") String projectId, @PathParam("workspaceId") String workspaceId, @PathParam("path") String path)
+    {
+        return executeWithLogging(
+                "getting entity " + path + " in backup group workspace " + workspaceId + " for project " + projectId,
+                () -> this.entityApi.getBackupGroupWorkspaceEntityAccessContext(projectId, workspaceId).getInvalidEntity(path)
+        );
+    }
 }
diff --git a/...c/server/resources/backup/project/group/GroupBackupWorkspaceRevisionEntitiesResource.java b/...c/server/resources/backup/project/group/GroupBackupWorkspaceRevisionEntitiesResource.java
@@ -18,6 +18,7 @@
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
 import org.finos.legend.sdlc.domain.model.entity.Entity;
+import org.finos.legend.sdlc.domain.model.entity.InvalidEntity;
 import org.finos.legend.sdlc.server.domain.api.entity.EntityApi;
 import org.finos.legend.sdlc.server.resources.EntityAccessResource;
 
@@ -86,4 +87,29 @@ public Entity getEntityByPath(@PathParam("projectId") String projectId, @PathPar
                 () -> this.entityApi.getBackupGroupWorkspaceRevisionEntityAccessContext(projectId, workspaceId, revisionId).getEntity(path)
         );
     }
+
+    @GET
+    @Path("invalidEntities")
+    @ApiOperation("Get invalid entities of the backup group workspace at the revision")
+    public List<InvalidEntity> getInvalidEntities(@PathParam("projectId") String projectId,
+                                                  @PathParam("workspaceId") String workspaceId,
+                                                  @PathParam("revisionId")
+                                                  @ApiParam("Including aliases: head, latest, current, base") String revisionId)
+    {
+        return executeWithLogging(
+                "getting invalid entities in revision " + revisionId + " of backup group workspace " + workspaceId + " for project " + projectId,
+                () -> getInvalidEntities(this.entityApi.getBackupGroupWorkspaceRevisionEntityAccessContext(projectId, workspaceId, revisionId))
+        );
+    }
+
+    @GET
+    @Path("invalidEntities/{path}")
+    @ApiOperation("Get an invalid entity of the backup group workspace at the revision by its path")
+    public InvalidEntity getInvalidEntityByPath(@PathParam("projectId") String projectId, @PathParam("workspaceId") String workspaceId, @PathParam("revisionId") String revisionId, @PathParam("path") String path)
+    {
+        return executeWithLogging(
+                "getting invalid entity " + path + " in revision " + revisionId + " of backup group workspace " + workspaceId + " for project " + projectId,
+                () -> this.entityApi.getBackupGroupWorkspaceRevisionEntityAccessContext(projectId, workspaceId, revisionId).getInvalidEntity(path)
+        );
+    }
 }
diff --git a/...nos/legend/sdlc/server/resources/backup/project/user/BackupWorkspaceEntitiesResource.java b/...nos/legend/sdlc/server/resources/backup/project/user/BackupWorkspaceEntitiesResource.java
@@ -18,6 +18,7 @@
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
 import org.finos.legend.sdlc.domain.model.entity.Entity;
+import org.finos.legend.sdlc.domain.model.entity.InvalidEntity;
 import org.finos.legend.sdlc.server.domain.api.entity.EntityApi;
 import org.finos.legend.sdlc.server.resources.EntityAccessResource;
 
@@ -84,4 +85,26 @@ public Entity getEntityByPath(@PathParam("projectId") String projectId, @PathPar
                 () -> this.entityApi.getBackupUserWorkspaceEntityAccessContext(projectId, workspaceId).getEntity(path)
         );
     }
+
+    @GET
+    @Path("invalidEntities")
+    @ApiOperation("Get invalid entities of the backup user workspace")
+    public List<InvalidEntity> getInvalidEntities(@PathParam("projectId") String projectId, @PathParam("workspaceId") String workspaceId)
+    {
+        return executeWithLogging(
+                "getting invalid entities in backup user workspace " + workspaceId + " for project " + projectId,
+                () -> getInvalidEntities(this.entityApi.getBackupUserWorkspaceEntityAccessContext(projectId, workspaceId))
+        );
+    }
+
+    @GET
+    @Path("invalidEntities/{path}")
+    @ApiOperation("Get an invalid entity of the backup user workspace by its path")
+    public InvalidEntity getInvalidEntityByPath(@PathParam("projectId") String projectId, @PathParam("workspaceId") String workspaceId, @PathParam("path") String path)
+    {
+        return executeWithLogging(
+                "getting invalid entity " + path + " in backup user workspace " + workspaceId + " for project " + projectId,
+                () -> this.entityApi.getBackupUserWorkspaceEntityAccessContext(projectId, workspaceId).getInvalidEntity(path)
+        );
+    }
 }
diff --git a/...nd/sdlc/server/resources/backup/project/user/BackupWorkspaceRevisionEntitiesResource.java b/...nd/sdlc/server/resources/backup/project/user/BackupWorkspaceRevisionEntitiesResource.java
@@ -18,6 +18,7 @@
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
 import org.finos.legend.sdlc.domain.model.entity.Entity;
+import org.finos.legend.sdlc.domain.model.entity.InvalidEntity;
 import org.finos.legend.sdlc.server.domain.api.entity.EntityApi;
 import org.finos.legend.sdlc.server.resources.EntityAccessResource;
 
@@ -85,4 +86,29 @@ public Entity getEntityByPath(@PathParam("projectId") String projectId, @PathPar
                 () -> this.entityApi.getBackupUserWorkspaceRevisionEntityAccessContext(projectId, workspaceId, revisionId).getEntity(path)
         );
     }
+
+    @GET
+    @Path("invalidEntities")
+    @ApiOperation("Get invalid entities of the backup user workspace at the revision")
+    public List<InvalidEntity> getInvalidEntities(@PathParam("projectId") String projectId,
+                                                  @PathParam("workspaceId") String workspaceId,
+                                                  @PathParam("revisionId")
+                                                  @ApiParam("Including aliases: head, latest, current, base") String revisionId)
+    {
+        return executeWithLogging(
+                "getting invalid entities in revision " + revisionId + " of backup user workspace " + workspaceId + " for project " + projectId,
+                () -> getInvalidEntities(this.entityApi.getBackupUserWorkspaceRevisionEntityAccessContext(projectId, workspaceId, revisionId))
+        );
+    }
+
+    @GET
+    @Path("invalidEntities/{path}")
+    @ApiOperation("Get an invalid entity of the backup user workspace at the revision by its path")
+    public InvalidEntity getInvalidEntityByPath(@PathParam("projectId") String projectId, @PathParam("workspaceId") String workspaceId, @PathParam("revisionId") String revisionId, @PathParam("path") String path)
+    {
+        return executeWithLogging(
+                "getting invalid entity " + path + " in revision " + revisionId + " of backup user workspace " + workspaceId + " for project " + projectId,
+                () -> this.entityApi.getBackupUserWorkspaceRevisionEntityAccessContext(projectId, workspaceId, revisionId).getInvalidEntity(path)
+        );
+    }
 }
diff --git a/...es/conflictResolution/project/group/GroupConflictResolutionWorkspaceEntitiesResource.java b/...es/conflictResolution/project/group/GroupConflictResolutionWorkspaceEntitiesResource.java
@@ -18,6 +18,7 @@
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
 import org.finos.legend.sdlc.domain.model.entity.Entity;
+import org.finos.legend.sdlc.domain.model.entity.InvalidEntity;
 import org.finos.legend.sdlc.server.domain.api.entity.EntityApi;
 import org.finos.legend.sdlc.server.resources.EntityAccessResource;
 
@@ -84,4 +85,26 @@ public Entity getEntityByPath(@PathParam("projectId") String projectId, @PathPar
                 () -> this.entityApi.getGroupWorkspaceWithConflictResolutionEntityAccessContext(projectId, workspaceId).getEntity(path)
         );
     }
+
+    @GET
+    @Path("invalidEntities")
+    @ApiOperation("Get invalid entities of the group workspace with conflict resolution")
+    public List<InvalidEntity> getInvalidEntities(@PathParam("projectId") String projectId, @PathParam("workspaceId") String workspaceId)
+    {
+        return executeWithLogging(
+                "getting invalid entities in group workspace with conflict resolution " + workspaceId + " for project " + projectId,
+                () -> getInvalidEntities(this.entityApi.getGroupWorkspaceWithConflictResolutionEntityAccessContext(projectId, workspaceId))
+        );
+    }
+
+    @GET
+    @Path("invalidEntities/{path}")
+    @ApiOperation("Get an invalid entity of the group workspace with conflict resolution by its path")
+    public InvalidEntity getInvalidEntityByPath(@PathParam("projectId") String projectId, @PathParam("workspaceId") String workspaceId, @PathParam("path") String path)
+    {
+        return executeWithLogging(
+                "getting invalid entity " + path + " in group workspace with conflict resolution " + workspaceId + " for project " + projectId,
+                () -> this.entityApi.getGroupWorkspaceWithConflictResolutionEntityAccessContext(projectId, workspaceId).getInvalidEntity(path)
+        );
+    }
 }