[Optimization]optimization resource manager (#3154)

DataLinkDC · Feb 16, 2024 · d54f19d · d54f19d
1 parent cfdc97f
commit d54f19d
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 24 deletions.
diff --git a/dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java
@@ -64,7 +64,7 @@ public class ResourceServiceImpl extends ServiceImpl<ResourcesMapper, Resources>
     public boolean syncRemoteDirectoryStructure() {
         Resources rootResource = getOne(new LambdaQueryWrapper<Resources>().eq(Resources::getPid, -1));
         if (rootResource == null) {
-            throw new BusException("root directory is not exists!! please check database");
+            throw new BusException(Status.RESOURCE_ROOT_DIR_NOT_EXIST);
         }
         List<Resources> local = list();
         Map<Integer, Resources> localMap =
@@ -96,7 +96,7 @@ public TreeNodeDTO createFolder(Integer pid, String fileName, String desc) {
         long count = count(
                 new LambdaQueryWrapper<Resources>().eq(Resources::getPid, pid).eq(Resources::getFileName, fileName));
         if (count > 0) {
-            throw new BusException("folder is exists!");
+            throw new BusException(Status.RESOURCE_FOLDER_EXISTS);
         }
         String path = "/" + fileName;
         Resources resources = new Resources();
@@ -145,7 +145,7 @@ public void rename(Integer id, String fileName, String desc) {
                 .eq(Resources::getPid, byId.getPid())
                 .eq(Resources::getFileName, fileName)
                 .ne(Resources::getId, id));
-        Assert.isFalse(count > 0, () -> new BusException("folder is exists!"));
+        Assert.isFalse(count > 0, () -> new BusException(Status.RESOURCE_FOLDER_EXISTS));
         List<String> split = StrUtil.split(sourceFullName, "/");
         split.remove(split.size() - 1);
         split.add(fileName);
@@ -254,6 +254,7 @@ public void uploadFile(Integer pid, String desc, File file) {
      * @param pResource    pResource
      * @param size         size
      */
+    @Transactional(rollbackFor = Exception.class)
     private void upload(
             Integer pid, String desc, Consumer<String> uploadAction, String fileName, Resources pResource, long size) {
         Resources currentUploadResource = getOne(

diff --git a/...y-client/dinky-client-base/src/main/java/org/dinky/resource/impl/HdfsResourceManager.java b/...y-client/dinky-client-base/src/main/java/org/dinky/resource/impl/HdfsResourceManager.java
@@ -19,6 +19,7 @@
 
 package org.dinky.resource.impl;
 
+import org.dinky.data.enums.Status;
 import org.dinky.data.exception.BusException;
 import org.dinky.data.model.ResourcesVO;
 import org.dinky.resource.BaseResourceManager;
@@ -47,7 +48,7 @@ public void remove(String path) {
         try {
             getHdfs().delete(new Path(getFilePath(path)), true);
         } catch (IOException e) {
-            throw BusException.valueOf("file.delete.failed", e);
+            throw new BusException(Status.RESOURCE_FILE_DELETE_FAILED, e);
         }
     }
 
@@ -56,7 +57,7 @@ public void rename(String path, String newPath) {
         try {
             getHdfs().rename(new Path(getFilePath(path)), new Path(getFilePath(newPath)));
         } catch (IOException e) {
-            throw BusException.valueOf("file.rename.failed", e);
+            throw new BusException(Status.RESOURCE_FILE_RENAME_FAILED, e);
         }
     }
 
@@ -68,7 +69,7 @@ public void putFile(String path, InputStream fileStream) {
             stream.flush();
             stream.close();
         } catch (IOException e) {
-            throw BusException.valueOf("file.upload.failed", e);
+            throw new BusException(Status.RESOURCE_FILE_UPLOAD_FAILED, e);
         }
     }
 
@@ -80,7 +81,7 @@ public void putFile(String path, File file) {
             stream.flush();
             stream.close();
         } catch (IOException e) {
-            throw BusException.valueOf("file.upload.failed", e);
+            throw new BusException(Status.RESOURCE_FILE_UPLOAD_FAILED, e);
         }
     }
 
@@ -122,7 +123,7 @@ public List<ResourcesVO> getFullDirectoryStructure(int rootId) {
                     .fileName(file.getPath().getName())
                     .isDirectory(file.isDirectory())
                     .type(0)
-                    .size(file.getBlockSize())
+                    .size(file.getLen())
                     .build();
 
             resList.add(resources);
@@ -135,13 +136,13 @@ public InputStream readFile(String path) {
         try {
             return getHdfs().open(new Path(getFilePath(path)));
         } catch (IOException e) {
-            throw BusException.valueOf("file.read.failed", e);
+            throw new BusException(Status.RESOURCE_FILE_READ_FAILED, e);
         }
     }
 
     public FileSystem getHdfs() {
         if (hdfs == null && instances.getResourcesEnable().getValue()) {
-            throw BusException.valueOf("Resource configuration error, HDFS is not enabled");
+            throw new BusException(Status.RESOURCE_HDFS_CONFIGURATION_ERROR);
         }
         return hdfs;
     }
@@ -154,15 +155,15 @@ public void checkHdfsFile(String path) {
         try {
             getHdfs().exists(new Path(path));
         } catch (IOException e) {
-            throw BusException.valueOf("hdfs.dir.or.file.not.exist", e);
+            throw new BusException(Status.RESOURCE_DIR_OR_FILE_NOT_EXIST, e);
         }
     }
 
     public FileStatus[] listHdfsFilePaths(String path) {
         try {
             return getHdfs().listStatus(new Path(path));
         } catch (IOException e) {
-            throw BusException.valueOf("file.path.visit.failed", e);
+            throw new BusException(Status.RESOURCE_FILE_PATH_VISIT_FAILED, e);
         }
     }
 

diff --git a/...-client/dinky-client-base/src/main/java/org/dinky/resource/impl/LocalResourceManager.java b/...-client/dinky-client-base/src/main/java/org/dinky/resource/impl/LocalResourceManager.java
@@ -19,6 +19,7 @@
 
 package org.dinky.resource.impl;
 
+import org.dinky.data.enums.Status;
 import org.dinky.data.exception.BusException;
 import org.dinky.data.model.ResourcesVO;
 import org.dinky.resource.BaseResourceManager;
@@ -51,8 +52,7 @@ public void remove(String path) {
                 throw new BusException("remove file failed,reason unknown");
             }
         } catch (IORuntimeException e) {
-            log.error("remove file failed", e);
-            throw new BusException(e.getMessage());
+            throw new BusException(Status.RESOURCE_FILE_DELETE_FAILED, e);
         }
     }
 
@@ -62,8 +62,7 @@ public void rename(String path, String newPath) {
             String newName = FileUtil.getName(newPath);
             FileUtil.rename(new File(getFilePath(path)), newName, true);
         } catch (Exception e) {
-            log.error("rename file failed", e);
-            throw new BusException(e.getMessage());
+            throw new BusException(Status.RESOURCE_FILE_RENAME_FAILED, e);
         }
     }
 
@@ -72,8 +71,7 @@ public void putFile(String path, InputStream fileStream) {
         try {
             FileUtil.writeFromStream(fileStream, getFilePath(path));
         } catch (Exception e) {
-            log.error("putFile file failed", e);
-            throw new BusException(e.getMessage());
+            throw new BusException(Status.RESOURCE_FILE_UPLOAD_FAILED, e);
         }
     }
 
@@ -121,7 +119,7 @@ public List<ResourcesVO> getFullDirectoryStructure(int rootId) {
                     .filter(Objects::nonNull)
                     .collect(Collectors.toList());
         } catch (IOException e) {
-            throw new RuntimeException(e);
+            throw new BusException(Status.RESOURCE_FILE_PATH_VISIT_FAILED, e);
         }
     }
 

diff --git a/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/OssResourceManager.java b/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/OssResourceManager.java
@@ -19,8 +19,8 @@
 
 package org.dinky.resource.impl;
 
+import org.dinky.data.enums.Status;
 import org.dinky.data.exception.BusException;
-import org.dinky.data.exception.DinkyException;
 import org.dinky.data.model.ResourcesVO;
 import org.dinky.oss.OssTemplate;
 import org.dinky.resource.BaseResourceManager;
@@ -66,7 +66,7 @@ public void putFile(String path, InputStream fileStream) {
         try {
             getOssTemplate().putObject(getOssTemplate().getBucketName(), getFilePath(path), fileStream);
         } catch (Exception e) {
-            throw new DinkyException(e);
+            throw new BusException(Status.RESOURCE_FILE_UPLOAD_FAILED, e);
         }
     }
 
@@ -76,7 +76,7 @@ public void putFile(String path, File file) {
             getOssTemplate()
                     .putObject(getOssTemplate().getBucketName(), getFilePath(path), FileUtil.getInputStream(file));
         } catch (Exception e) {
-            throw new DinkyException(e);
+            throw new BusException(Status.RESOURCE_FILE_UPLOAD_FAILED, e);
         }
     }
 
@@ -136,7 +136,7 @@ public InputStream readFile(String path) {
 
     public OssTemplate getOssTemplate() {
         if (ossTemplate == null && instances.getResourcesEnable().getValue()) {
-            throw BusException.valueOf("Resource configuration error, OSS is not enabled");
+            throw new BusException(Status.RESOURCE_OSS_CONFIGURATION_ERROR);
         }
         return ossTemplate;
     }

diff --git a/dinky-common/src/main/java/org/dinky/data/enums/Status.java b/dinky-common/src/main/java/org/dinky/data/enums/Status.java
@@ -296,6 +296,15 @@ public enum Status {
      */
     ROOT_DIR_NOT_ALLOW_DELETE(9031, "resource.root.dir.not.allow.delete"),
     RESOURCE_DIR_OR_FILE_NOT_EXIST(9032, "resource.dir.or.file.not.exist"),
+    RESOURCE_FILE_PATH_VISIT_FAILED(9033, "file.path.visit.failed"),
+    RESOURCE_HDFS_CONFIGURATION_ERROR(9034, "resource.hdfs.configuration.error"),
+    RESOURCE_FILE_UPLOAD_FAILED(9035, "file.upload.failed"),
+    RESOURCE_FILE_RENAME_FAILED(9036, "file.rename.failed"),
+    RESOURCE_FILE_DELETE_FAILED(9037, "file.delete.failed"),
+    RESOURCE_FILE_READ_FAILED(9038, "file.read.failed"),
+    RESOURCE_OSS_CONFIGURATION_ERROR(9039, "resource.oss.configuration.error"),
+    RESOURCE_ROOT_DIR_NOT_EXIST(9040, "resource.root.dir.not.exist"),
+    RESOURCE_FOLDER_EXISTS(9041, "resource.folder.exists"),
 
     /**
      * global exception

diff --git a/dinky-common/src/main/resources/i18n/messages_en_US.properties b/dinky-common/src/main/resources/i18n/messages_en_US.properties
@@ -158,6 +158,9 @@ role.binding.row.permission=Role Already Binding Row Permission , Can Not Delete
 unknown.i18n=Unknown i18n information, please check. . .
 
 file.upload.failed=File upload failed, reason: {0}
+file.rename.failed=File rename failed, reason: {0}
+file.delete.failed=File delete failed, reason: {0}
+file.read.failed=File read failed, reason: {0}
 
 # dinky-alert
 alert.rule.jobFail=Job Failure
@@ -263,7 +266,7 @@ sys.resource.settings.hdfs.fs.defaultFS.note=fs.defaultFS configuration items, s
 sys.resource.settings.hdfs.core.site=core-site.xml
 sys.resource.settings.hdfs.core.site.note=core-site.xml configuration file content,High availability required
 sys.resource.settings.hdfs.hdfs.site=hdfs-site.xml
-sys.resource.settings.hdfs.hdfs.site.note=hdfs-site.xml configuration file content\uFF0CHigh availability required
+sys.resource.settings.hdfs.hdfs.site.note=hdfs-site.xml configuration file content,High availability required
 
 
 #Dinky Gateway
@@ -284,3 +287,8 @@ process.register.exits=The current task is executing, do not submit it repeatedl
 # resource
 resource.root.dir.not.allow.delete=The root directory is not allowed to be deleted
 resource.dir.or.file.not.exist=The directory or file does not exist
+file.path.visit.failed=File path access failed, please check whether the path is correct/exist on the corresponding storage system
+resource.hdfs.configuration.error=Resource configuration error, HDFS is not enabled
+resource.oss.configuration.error=Resource configuration error, OSS is not enabled
+resource.root.dir.not.exist=The root directory does not exist, please check the database
+resource.folder.exists=The folder already exists
diff --git a/dinky-common/src/main/resources/i18n/messages_zh_CN.properties b/dinky-common/src/main/resources/i18n/messages_zh_CN.properties
@@ -158,6 +158,9 @@ role.binding.row.permission=该角色已绑定行权限，无法删除
 unknown.i18n=未知 i18n 信息,请检查. . .
 
 file.upload.failed=文件上传失败, 原因: {0}
+file.rename.failed=文件重命名失败, 原因: {0}
+file.delete.failed=文件删除失败, 原因: {0}
+file.read.failed=文件读取失败, 原因: {0}
 
 daemon.task.config.not.exist=线程任务配置不能为空
 daemon.task.not.support=不支持线程任务类型：
@@ -285,3 +288,8 @@ process.register.exits=当前任务正在执行，请勿重复提交，如有问
 # resource
 resource.root.dir.not.allow.delete=根目录不允许删除
 resource.dir.or.file.not.exist=资源目录或文件不存在
+file.path.visit.failed=文件路径访问失败,请检查路径在对应存储系统上是否正确/是否存在
+resource.hdfs.configuration.error=资源配置错误，未启用HDFS
+resource.oss.configuration.error=资源配置错误，未启用OSS
+resource.root.dir.not.exist=根目录不存在，请检查数据库
+resource.folder.exists=文件夹已存在