Feature dev query hdfs info

dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/HdfsResourceManager.java

@@ -24,16 +24,20 @@
 import org.dinky.resource.BaseResourceManager;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.List;
 
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.util.ArrayUtil;
+import cn.hutool.core.util.StrUtil;
 
 public class HdfsResourceManager implements BaseResourceManager {
     FileSystem hdfs;
@@ -87,7 +91,43 @@ public String getFileContent(String path) {
 
     @Override
     public List<ResourcesVO> getFullDirectoryStructure(int rootId) {
-        throw new RuntimeException("Sync HDFS Not implemented!");
+        String basePath = new Path(getBasePath()).toUri().getPath();
+        checkHdfsFile(basePath);
+
+        List<FileStatus> filePathsList = new ArrayList<>();
+        listAllHdfsFilePaths(basePath, filePathsList);
+
+        List<ResourcesVO> resList = new ArrayList<>();
+
+        for (FileStatus file : filePathsList) {
+            Path parentPath = file.getPath().getParent();
+            if (parentPath == null) {
+                continue;
+            }
+            int parentId = 1;
+
+            // Determine whether it is the root directory
+            final String parentSimplePath = parentPath.toUri().getPath();
+            if (!parentSimplePath.equals(basePath)) {
+                String path = parentSimplePath.replace(basePath, "");
+                parentId = path.isEmpty() ? rootId : path.hashCode();
+            }
+
+            String self = StrUtil.replaceFirst(file.getPath().toUri().getPath(), basePath, "");
+
+            ResourcesVO resources = ResourcesVO.builder()
+                    .id(self.hashCode())
+                    .pid(parentId)
+                    .fullName(self)
+                    .fileName(file.getPath().getName())
+                    .isDirectory(file.isDirectory())
+                    .type(0)
+                    .size(file.getBlockSize())
+                    .build();
+
+            resList.add(resources);
+        }
+        return resList;
     }
 
     @Override
@@ -109,4 +149,36 @@ public FileSystem getHdfs() {
     public synchronized void setHdfs(FileSystem hdfs) {
         this.hdfs = hdfs;
     }
+
+    public void checkHdfsFile(String path) {
+        try {
+            getHdfs().exists(new Path(path));
+        } catch (IOException e) {
+            throw BusException.valueOf("hdfs.dir.or.file.not.exist", e);
+        }
+    }
+
+    public FileStatus[] listHdfsFilePaths(String path) {
+        try {
+            return getHdfs().listStatus(new Path(path));
+        } catch (IOException e) {
+            throw BusException.valueOf("file.path.visit.failed", e);
+        }
+    }
+
+    public void listAllHdfsFilePaths(String path, List<FileStatus> fileStatusList) {
+
+        FileStatus[] paths = listHdfsFilePaths(path);
+
+        if (ArrayUtil.isEmpty(paths)) {
+            return;
+        }
+
+        for (FileStatus file : paths) {
+            fileStatusList.add(file);
+            if (file.isDirectory()) {
+                listAllHdfsFilePaths(file.getPath().toString(), fileStatusList);
+            }
+        }
+    }
 }

dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/LocalResourceManager.java

@@ -38,6 +38,7 @@
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IORuntimeException;
 import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.util.StrUtil;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -89,24 +90,27 @@ public String getFileContent(String path) {
 
     @Override
     public List<ResourcesVO> getFullDirectoryStructure(int rootId) {
-        String basePath = getBasePath();
+        String basePath = FileUtil.file(getBasePath()).getPath();
+
         try (Stream<Path> paths = Files.walk(Paths.get(basePath))) {
             return paths.map(path -> {
                         if (path.compareTo(Paths.get(basePath)) == 0) {
                             // 跳过根目录 | skip root directory
                             return null;
                         }
                         Path parentPath = path.getParent();
-                        String parent = "";
-                        if (parentPath != null) {
-                            parent = parentPath.toString().replace(basePath, "");
+                        String self = StrUtil.replaceFirst(path.toString(), basePath, "");
+                        int parentId = 1;
+
+                        // Determine whether it is the root directory
+                        if (!parentPath.toUri().getPath().equals(basePath)) {
+                            String parent = parentPath.toString().replace(basePath, "");
+                            parentId = parent.isEmpty() ? rootId : parent.hashCode();
                         }
-                        String self = path.toString().replace(basePath, "");
-                        int pid = parent.isEmpty() ? rootId : parent.hashCode();
                         File file = new File(path.toString());
                         return ResourcesVO.builder()
                                 .id(self.hashCode())
-                                .pid(pid)
+                                .pid(parentId)
                                 .fullName(self)
                                 .fileName(file.getName())
                                 .isDirectory(file.isDirectory())

dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/OssResourceManager.java

@@ -38,6 +38,7 @@
 
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.util.StrUtil;
 
 public class OssResourceManager implements BaseResourceManager {
     OssTemplate ossTemplate;
@@ -87,6 +88,11 @@ public String getFileContent(String path) {
     @Override
     public List<ResourcesVO> getFullDirectoryStructure(int rootId) {
         String basePath = getBasePath();
+        if (StrUtil.isNotBlank(basePath)) {
+            if (basePath.charAt(0) == '/') {
+                basePath = basePath.substring(1);
+            }
+        }
 
         List<S3ObjectSummary> listBucketObjects =
                 getOssTemplate().listBucketObjects(getOssTemplate().getBucketName(), basePath);

pom.xml

@@ -50,7 +50,7 @@
 
     <properties>
         <assertj.version>3.21.0</assertj.version>
-        <aws-java-sdk-s3.version>1.12.589</aws-java-sdk-s3.version>
+        <aws-java-sdk-s3.version>1.12.645</aws-java-sdk-s3.version>
         <build-helper-maven-plugin.version>1.7</build-helper-maven-plugin.version>
         <clickhouse.version>0.2.6</clickhouse.version>
         <commons-email>1.5</commons-email>