From 16a1c283f29e455e0374b03ef202995f821e6d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Miguel=20Moreno?= Date: Thu, 28 Mar 2024 17:10:59 +0100 Subject: [PATCH] Fixed file changes detection without inotify - Created FileSystemWatcher class - Updated ImageStorage class > Fixes #114 --- .../plugin/storage/FileSystemWatcher.java | 326 ++++++++++++++++++ .../bukkit/plugin/storage/ImageStorage.java | 218 ++---------- 2 files changed, 359 insertions(+), 185 deletions(-) create mode 100644 src/main/java/io/josemmo/bukkit/plugin/storage/FileSystemWatcher.java diff --git a/src/main/java/io/josemmo/bukkit/plugin/storage/FileSystemWatcher.java b/src/main/java/io/josemmo/bukkit/plugin/storage/FileSystemWatcher.java new file mode 100644 index 0000000..09ba535 --- /dev/null +++ b/src/main/java/io/josemmo/bukkit/plugin/storage/FileSystemWatcher.java @@ -0,0 +1,326 @@ +package io.josemmo.bukkit.plugin.storage; + +import com.sun.nio.file.ExtendedWatchEventModifier; +import io.josemmo.bukkit.plugin.utils.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Service for detecting file events inside a given directory. + *

+ * It supports recursive storage (e.g., nested directories) and watches for file system changes in realtime + * when supported by the OS. + */ +public abstract class FileSystemWatcher { + private static final int MAX_DEPTH = 32; + private static final int POLLING_INTERVAL = 4000; + private static final String PROBE_FILENAME = ".inotify_test"; + private static final Logger LOGGER = Logger.getLogger("FileSystemWatcher"); + protected final Path basePath; + /** Map of existing directories with the files they contain and their last modification timestamps */ + private final SortedMap> fileTree = new TreeMap<>(); + private @Nullable Thread watcherThread; + + /** + * Class constructor + * @param basePath Base path + */ + public FileSystemWatcher(@NotNull Path basePath) { + this.basePath = basePath; + } + + /** + * Start watcher + * @throws RuntimeException if failed to start + */ + protected void start() throws RuntimeException { + // Prevent initializing more than once + if (watcherThread != null) { + throw new RuntimeException("File system watcher is already running"); + } + + // Perform initial scan + scan(); + + // Start watching files + watcherThread = new WatcherThread(); + watcherThread.start(); + } + + /** + * Stop watcher + */ + protected void stop() { + if (watcherThread != null) { + watcherThread.interrupt(); + watcherThread = null; + } + } + + /** + * On file created + * @param path File path + */ + protected abstract void onFileCreated(@NotNull Path path); + + /** + * On file modified + * @param path File path + */ + protected abstract void onFileModified(@NotNull Path path); + + /** + * On file deleted + * @param path File path + */ + protected abstract void onFileDeleted(@NotNull Path path); + + /** + * Scan base directory recursively + */ + private void scan() { + synchronized (fileTree) { + // Assume all directories and files have been deleted, will discard existing items afterward + Set deletedDirectories = new HashSet<>(fileTree.keySet()); + Set deletedFiles = fileTree.values().stream() + .map(Map::keySet) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + + // Traverse file tree + Set options = EnumSet.noneOf(FileVisitOption.class); + try { + Files.walkFileTree(basePath, options, MAX_DEPTH, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) { + fileTree.putIfAbsent(path, new HashMap<>()); + deletedDirectories.remove(path); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { + Map subtree = fileTree.get(path.getParent()); + Long oldModifiedAt = subtree.get(path); + long newModifiedAt = attrs.lastModifiedTime().toMillis(); + if (oldModifiedAt == null) { + subtree.put(path, newModifiedAt); + onFileCreated(path); + } else if (newModifiedAt > oldModifiedAt) { + subtree.put(path, newModifiedAt); + onFileModified(path); + } + deletedFiles.remove(path); + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + LOGGER.severe("Failed to list files in directory", e); + } + + // Process deleted files and directories + for (Path path : deletedFiles) { + fileTree.get(path.getParent()).remove(path); + onFileDeleted(path); + } + for (Path path : deletedDirectories) { + fileTree.remove(path); + } + } + } + + private class WatcherThread extends Thread { + private final boolean IS_WINDOWS = System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("win"); + + @Override + public void run() { + if (isInotifySupported()) { + runWithFileSystemEvents(); + } else { + LOGGER.warning("Device does not support inotify, detection of file changes will be slower"); + runWithPolling(); + } + } + + /** + * Is inotify supported + * @return Whether inotify is supported + */ + @SuppressWarnings("ResultOfMethodCallIgnored") + private boolean isInotifySupported() { + try (WatchService watchService = FileSystems.getDefault().newWatchService()) { + // Start listening for events + basePath.register( + watchService, + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_DELETE, + StandardWatchEventKinds.ENTRY_MODIFY + ); + + // Create and delete probe directory + File testFile = basePath.resolve(PROBE_FILENAME).toFile(); + testFile.mkdir(); + testFile.delete(); + + // Check that at least one event was emitted + WatchKey watchKey = watchService.poll(); + return (watchKey != null && !watchKey.pollEvents().isEmpty()); + } catch (IOException __) { + return false; + } + } + + /** + * Run with polling + */ + @SuppressWarnings({"InfiniteLoopStatement", "BusyWait"}) + private void runWithPolling() { + try { + while (true) { + Thread.sleep(POLLING_INTERVAL); + scan(); + } + } catch (InterruptedException __) { + // Silently ignore exception, this is expected when service shuts down + } + } + + /** + * Run with file system events + */ + private void runWithFileSystemEvents() { + try (WatchService watchService = FileSystems.getDefault().newWatchService()) { + // Register initial directories + synchronized (fileTree) { + for (Path path : fileTree.keySet()) { + registerDirectory(watchService, path); + } + } + + // Listen for events + WatchKey key; + try { + while ((key = watchService.take()) != null) { + for (WatchEvent event : key.pollEvents()) { + WatchEvent.Kind kind = event.kind(); + Path keyPath = (Path) key.watchable(); + Path path = keyPath.resolve((Path) event.context()); + handleWatchEvent(watchService, path, kind); + } + key.reset(); + } + } catch (ClosedWatchServiceException | InterruptedException __) { + // Silently ignore exception, this is expected when service shuts down + } + } catch (IOException e) { + LOGGER.severe("Unexpected error at watch service", e); + } + } + + /** + * Handle watch event + * @param watchService Watch service + * @param path File or directory path + * @param kind Event kind + */ + private void handleWatchEvent(@NotNull WatchService watchService, @NotNull Path path, @NotNull WatchEvent.Kind kind) { + synchronized (fileTree) { + Map subtree = fileTree.computeIfAbsent(path.getParent(), k -> new HashMap<>()); + + // Handle deletion of files and directories + if (kind == StandardWatchEventKinds.ENTRY_DELETE) { + if (subtree.containsKey(path)) { + subtree.remove(path); + onFileDeleted(path); + } else { + unregisterDirectory(path); + } + return; + } + + // Handle creation of directories + if (path.toFile().isDirectory()) { + if (kind == StandardWatchEventKinds.ENTRY_CREATE) { + registerDirectory(watchService, path); + } + return; + } + + // Handle creation and modification of files + // NOTE: in Windows, some file creation events are reported as modifications + Long oldModifiedAt = subtree.get(path); + long newModifiedAt = path.toFile().lastModified(); + if (oldModifiedAt == null) { + subtree.put(path, newModifiedAt); + onFileCreated(path); + } else if (newModifiedAt > oldModifiedAt) { + subtree.put(path, newModifiedAt); + onFileModified(path); + } + } + } + + /** + * Register directory + * @param watchService Watch service + * @param path Directory path + */ + private void registerDirectory(@NotNull WatchService watchService, @NotNull Path path) { + // Windows supports listing to events in the entire file tree, + // in that case only allow registering the listener on the base path + if (IS_WINDOWS && !path.equals(basePath)) { + return; + } + + // Start watching directory for events + try { + WatchEvent.Kind[] events = new WatchEvent.Kind[]{ + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_DELETE, + StandardWatchEventKinds.ENTRY_MODIFY + }; + WatchEvent.Modifier[] modifiers = IS_WINDOWS ? + new WatchEvent.Modifier[]{ExtendedWatchEventModifier.FILE_TREE} : + new WatchEvent.Modifier[0]; + path.register(watchService, events, modifiers); + LOGGER.fine("Started watching directory at \"" + path + "\""); + } catch (IOException e) { + LOGGER.severe("Failed to register directory", e); + } + } + + /** + * Unregister directory + * @param path Directory path + */ + private void unregisterDirectory(@NotNull Path path) { + synchronized (fileTree) { + if (!fileTree.containsKey(path)) { + // Already unregistered, can skip work + return; + } + boolean foundFirst = false; + Iterator>> iter = fileTree.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + if (entry.getKey().startsWith(path)) { + for (Path childPath : entry.getValue().keySet()) { + onFileDeleted(childPath); + } + foundFirst = true; + iter.remove(); + } else if (foundFirst) { + // We can break early because set is alphabetically sorted by key + break; + } + } + } + } + } +} diff --git a/src/main/java/io/josemmo/bukkit/plugin/storage/ImageStorage.java b/src/main/java/io/josemmo/bukkit/plugin/storage/ImageStorage.java index e6085d4..13cc947 100644 --- a/src/main/java/io/josemmo/bukkit/plugin/storage/ImageStorage.java +++ b/src/main/java/io/josemmo/bukkit/plugin/storage/ImageStorage.java @@ -1,14 +1,11 @@ package io.josemmo.bukkit.plugin.storage; -import com.sun.nio.file.ExtendedWatchEventModifier; import io.josemmo.bukkit.plugin.utils.Logger; import io.josemmo.bukkit.plugin.utils.Permissions; import org.bukkit.command.CommandSender; import org.bukkit.entity.Player; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.File; -import java.io.IOException; import java.nio.file.*; import java.util.*; import java.util.regex.Matcher; @@ -16,23 +13,18 @@ import java.util.regex.PatternSyntaxException; /** - * A service whose purpose is to keep track of all available image files in a given directory. - * It supports recursive storage (e.g., nested directories) and watches for file system changes in realtime. + * Service for keeping track of image images. *

* All files are indexed based on their filename. - * Due to recursion, filenames can contain forward slashes (i.e., "/") and act as relative paths to the base + * Due to recursion, filenames can contain forward slashes (i.e., "/") and act as relative paths to the base * directory. */ -public class ImageStorage { - private static final boolean IS_WINDOWS = System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("win"); +public class ImageStorage extends FileSystemWatcher { private static final Logger LOGGER = Logger.getLogger("ImageStorage"); /** Map of registered files indexed by filename */ private final SortedMap files = new TreeMap<>(); - private final Path basePath; private final Path cachePath; private final String allowedPaths; - private @Nullable WatchService watchService; - private @Nullable Thread watchServiceThread; /** * Class constructor @@ -41,7 +33,7 @@ public class ImageStorage { * @param allowedPaths Allowed paths pattern */ public ImageStorage(@NotNull Path basePath, @NotNull Path cachePath, @NotNull String allowedPaths) { - this.basePath = basePath; + super(basePath); this.cachePath = cachePath; this.allowedPaths = allowedPaths; } @@ -64,15 +56,10 @@ public ImageStorage(@NotNull Path basePath, @NotNull Path cachePath, @NotNull St /** * Start service - * @throws IOException if failed to start watch service - * @throws RuntimeException if already running + * @throws RuntimeException if failed to start watch service */ - public void start() throws IOException, RuntimeException { - // Prevent initializing more than once - if (watchService != null || watchServiceThread != null) { - throw new RuntimeException("Service is already running"); - } - + @Override + public void start() throws RuntimeException { // Create base directories if necessary if (basePath.toFile().mkdirs()) { LOGGER.info("Created images directory as it did not exist"); @@ -81,33 +68,17 @@ public void start() throws IOException, RuntimeException { LOGGER.info("Created cache directory as it did not exist"); } - // Start watching files - watchService = FileSystems.getDefault().newWatchService(); - watchServiceThread = new WatcherThread(); - watchServiceThread.start(); - registerDirectory(basePath, true); + // Start file system watcher + super.start(); LOGGER.fine("Found " + files.size() + " file(s) in images directory"); } /** * Stop service */ + @Override public void stop() { - // Interrupt watch service thread - if (watchServiceThread != null) { - watchServiceThread.interrupt(); - watchServiceThread = null; - } - - // Close watch service - if (watchService != null) { - try { - watchService.close(); - } catch (IOException e) { - LOGGER.warning("Failed to close watch service", e); - } - watchService = null; - } + super.stop(); } /** @@ -140,7 +111,7 @@ public synchronized int size() { * @return Whether sender is allowed to access path */ public boolean isPathAllowed(@NotNull Path path, @NotNull CommandSender sender) { - return isPathAllowed(getFilename(path), sender); + return isPathAllowed(pathToFilename(path), sender); } /** @@ -191,58 +162,20 @@ public boolean isPathAllowed(@NotNull String path, @NotNull CommandSender sender } /** - * Register directory - * @param path Path to directory - * @param isBase Whether is base directory or not + * Convert path to filename + * @param path File path + * @return Relative path used for indexing */ - private synchronized void registerDirectory(@NotNull Path path, boolean isBase) { - // Validate path - if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { - LOGGER.warning("Cannot list files in \"" + path + "\" as it is not a valid directory"); - return; - } - - // Do initial directory listing - for (File child : Objects.requireNonNull(path.toFile().listFiles())) { - if (child.isDirectory()) { - registerDirectory(child.toPath(), false); - } else { - registerFile(child.toPath()); - } - } - - // Start watching for files changes - if (!IS_WINDOWS || isBase) { - try { - WatchEvent.Kind[] events = new WatchEvent.Kind[]{ - StandardWatchEventKinds.ENTRY_CREATE, - StandardWatchEventKinds.ENTRY_DELETE, - StandardWatchEventKinds.ENTRY_MODIFY - }; - WatchEvent.Modifier[] modifiers = IS_WINDOWS ? - new WatchEvent.Modifier[]{ExtendedWatchEventModifier.FILE_TREE} : - new WatchEvent.Modifier[0]; - path.register(Objects.requireNonNull(watchService), events, modifiers); - LOGGER.fine("Started watching directory at \"" + path + "\""); - } catch (IOException | NullPointerException e) { - LOGGER.severe("Failed to register directory", e); - } - } + private @NotNull String pathToFilename(@NotNull Path path) { + return basePath.relativize(path).toString().replaceAll("\\\\", "/"); } /** - * Register file - * @param path Path to file + * On file created + * @param path File path */ - private synchronized void registerFile(@NotNull Path path) { - // Validate path - if (!Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) { - LOGGER.warning("Cannot register \"" + path + "\" as it is not a valid file"); - return; - } - - // Add file to map - String filename = getFilename(path); + protected synchronized void onFileCreated(@NotNull Path path) { + String filename = pathToFilename(path); ImageFile imageFile = new ImageFile(filename, path); if (files.putIfAbsent(filename, imageFile) == null) { LOGGER.fine("Registered file \"" + filename + "\""); @@ -250,113 +183,28 @@ private synchronized void registerFile(@NotNull Path path) { } /** - * Unregister directory - * @param filename Filename to directory + * On file modified + * @param path File path */ - private synchronized void unregisterDirectory(@NotNull String filename) { - boolean foundFirst = false; - Iterator> iter = files.entrySet().iterator(); - while (iter.hasNext()) { - String entryKey = iter.next().getKey(); - if (entryKey.startsWith(filename+"/")) { - foundFirst = true; - iter.remove(); - LOGGER.fine("Unregistered file \"" + entryKey + "\""); - } else if (foundFirst) { - // We can break early because set is alphabetically sorted by key - break; - } - } - } - - /** - * Unregister file - * @param filename Filename to file - */ - private synchronized void unregisterFile(@NotNull String filename) { - ImageFile imageFile = files.remove(filename); + protected synchronized void onFileModified(@NotNull Path path) { + String filename = pathToFilename(path); + ImageFile imageFile = files.get(filename); if (imageFile != null) { imageFile.invalidate(); - LOGGER.fine("Unregistered file \"" + filename + "\""); + LOGGER.fine("Invalidated file \"" + filename + "\""); } } /** - * Invalidate file - * @param filename Filename to file + * On file deleted + * @param path File path */ - private synchronized void invalidateFile(@NotNull String filename) { - ImageFile imageFile = files.get(filename); + protected synchronized void onFileDeleted(@NotNull Path path) { + String filename = pathToFilename(path); + ImageFile imageFile = files.remove(filename); if (imageFile != null) { imageFile.invalidate(); - } - } - - /** - * Handle watch event - * @param path Path to file or directory - * @param kind Event kind - */ - private synchronized void handleWatchEvent(@NotNull Path path, WatchEvent.Kind kind) { - // Check whether file currently exists in file system (for CREATE and UPDATE events) - // or is registered in the file list (for DELETE event) - String filename = getFilename(path); - boolean isFile = path.toFile().isFile() || files.containsKey(filename); - - // Handle creation event - if (kind == StandardWatchEventKinds.ENTRY_CREATE) { - if (isFile) { - registerFile(path); - } else { - registerDirectory(path, false); - } - return; - } - - // Handle deletion event - if (kind == StandardWatchEventKinds.ENTRY_DELETE) { - if (isFile) { - unregisterFile(filename); - } else { - unregisterDirectory(filename); - } - return; - } - - // Handle modification event - if (kind == StandardWatchEventKinds.ENTRY_MODIFY && isFile) { - invalidateFile(filename); - } - } - - /** - * Get filename from path - * @param path Path to file - * @return Relative path used for indexing - */ - private @NotNull String getFilename(@NotNull Path path) { - return basePath.relativize(path).toString().replaceAll("\\\\", "/"); - } - - private class WatcherThread extends Thread { - @Override - public void run() { - try { - WatchKey key; - while ((key = Objects.requireNonNull(watchService).take()) != null) { - for (WatchEvent event : key.pollEvents()) { - WatchEvent.Kind kind = event.kind(); - Path keyPath = (Path) key.watchable(); - Path path = keyPath.resolve((Path) event.context()); - handleWatchEvent(path, kind); - } - key.reset(); - } - } catch (ClosedWatchServiceException | InterruptedException __) { - // Silently ignore exception, this is expected when service shuts down - } catch (NullPointerException e) { - LOGGER.severe("Watch service was stopped before watcher thread", e); - } + LOGGER.fine("Unregistered file \"" + filename + "\""); } } }