diff --git a/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/HdfsResourceManager.java b/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/HdfsResourceManager.java index 9d8bfcc385..b0ac70702c 100644 --- a/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/HdfsResourceManager.java +++ b/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/HdfsResourceManager.java @@ -24,16 +24,20 @@ import org.dinky.resource.BaseResourceManager; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.List; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; +import cn.hutool.core.util.ArrayUtil; +import cn.hutool.core.util.StrUtil; public class HdfsResourceManager implements BaseResourceManager { FileSystem hdfs; @@ -87,7 +91,43 @@ public String getFileContent(String path) { @Override public List getFullDirectoryStructure(int rootId) { - throw new RuntimeException("Sync HDFS Not implemented!"); + String basePath = new Path(getBasePath()).toUri().getPath(); + checkHdfsFile(basePath); + + List filePathsList = new ArrayList<>(); + listAllHdfsFilePaths(basePath, filePathsList); + + List resList = new ArrayList<>(); + + for (FileStatus file : filePathsList) { + Path parentPath = file.getPath().getParent(); + if (parentPath == null) { + continue; + } + int parentId = 1; + + // Determine whether it is the root directory + final String parentSimplePath = parentPath.toUri().getPath(); + if (!parentSimplePath.equals(basePath)) { + String path = parentSimplePath.replace(basePath, ""); + parentId = path.isEmpty() ? rootId : path.hashCode(); + } + + String self = StrUtil.replaceFirst(file.getPath().toUri().getPath(), basePath, ""); + + ResourcesVO resources = ResourcesVO.builder() + .id(self.hashCode()) + .pid(parentId) + .fullName(self) + .fileName(file.getPath().getName()) + .isDirectory(file.isDirectory()) + .type(0) + .size(file.getBlockSize()) + .build(); + + resList.add(resources); + } + return resList; } @Override @@ -109,4 +149,36 @@ public FileSystem getHdfs() { public synchronized void setHdfs(FileSystem hdfs) { this.hdfs = hdfs; } + + public void checkHdfsFile(String path) { + try { + getHdfs().exists(new Path(path)); + } catch (IOException e) { + throw BusException.valueOf("hdfs.dir.or.file.not.exist", e); + } + } + + public FileStatus[] listHdfsFilePaths(String path) { + try { + return getHdfs().listStatus(new Path(path)); + } catch (IOException e) { + throw BusException.valueOf("file.path.visit.failed", e); + } + } + + public void listAllHdfsFilePaths(String path, List fileStatusList) { + + FileStatus[] paths = listHdfsFilePaths(path); + + if (ArrayUtil.isEmpty(paths)) { + return; + } + + for (FileStatus file : paths) { + fileStatusList.add(file); + if (file.isDirectory()) { + listAllHdfsFilePaths(file.getPath().toString(), fileStatusList); + } + } + } } diff --git a/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/LocalResourceManager.java b/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/LocalResourceManager.java index 421ef369f5..b7caaab030 100644 --- a/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/LocalResourceManager.java +++ b/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/LocalResourceManager.java @@ -38,6 +38,7 @@ import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IORuntimeException; import cn.hutool.core.io.IoUtil; +import cn.hutool.core.util.StrUtil; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -89,7 +90,8 @@ public String getFileContent(String path) { @Override public List getFullDirectoryStructure(int rootId) { - String basePath = getBasePath(); + String basePath = FileUtil.file(getBasePath()).getPath(); + try (Stream paths = Files.walk(Paths.get(basePath))) { return paths.map(path -> { if (path.compareTo(Paths.get(basePath)) == 0) { @@ -97,16 +99,18 @@ public List getFullDirectoryStructure(int rootId) { return null; } Path parentPath = path.getParent(); - String parent = ""; - if (parentPath != null) { - parent = parentPath.toString().replace(basePath, ""); + String self = StrUtil.replaceFirst(path.toString(), basePath, ""); + int parentId = 1; + + // Determine whether it is the root directory + if (!parentPath.toUri().getPath().equals(basePath)) { + String parent = parentPath.toString().replace(basePath, ""); + parentId = parent.isEmpty() ? rootId : parent.hashCode(); } - String self = path.toString().replace(basePath, ""); - int pid = parent.isEmpty() ? rootId : parent.hashCode(); File file = new File(path.toString()); return ResourcesVO.builder() .id(self.hashCode()) - .pid(pid) + .pid(parentId) .fullName(self) .fileName(file.getName()) .isDirectory(file.isDirectory()) diff --git a/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/OssResourceManager.java b/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/OssResourceManager.java index e41c958e77..84421a330d 100644 --- a/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/OssResourceManager.java +++ b/dinky-client/dinky-client-base/src/main/java/org/dinky/resource/impl/OssResourceManager.java @@ -38,6 +38,7 @@ import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; +import cn.hutool.core.util.StrUtil; public class OssResourceManager implements BaseResourceManager { OssTemplate ossTemplate; @@ -87,6 +88,11 @@ public String getFileContent(String path) { @Override public List getFullDirectoryStructure(int rootId) { String basePath = getBasePath(); + if (StrUtil.isNotBlank(basePath)) { + if (basePath.charAt(0) == '/') { + basePath = basePath.substring(1); + } + } List listBucketObjects = getOssTemplate().listBucketObjects(getOssTemplate().getBucketName(), basePath); diff --git a/pom.xml b/pom.xml index bee301c3c8..a6af110350 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ 3.21.0 - 1.12.589 + 1.12.645 1.7 0.2.6 1.5