From 41e3d950120e53807d35a05de79e7c761c3b3d11 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 20 Sep 2023 15:25:19 +0900 Subject: [PATCH] WatchImporter: Treat locked files as open --- ui/src/bamboo/task/WatchImporter.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ui/src/bamboo/task/WatchImporter.java b/ui/src/bamboo/task/WatchImporter.java index 1e9234e2..20f78459 100644 --- a/ui/src/bamboo/task/WatchImporter.java +++ b/ui/src/bamboo/task/WatchImporter.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.channels.FileChannel; import java.nio.file.*; import java.util.HashMap; import java.util.List; @@ -123,11 +124,26 @@ void handleOpenWarc(Config.Watch watch, Path path) throws IOException { private void handleClosedWarc(Config.Watch watch, Path path) throws IOException { log.finest("handleClosedWarc(" + path + ")"); + long size = Files.size(path); + if (size == 0) { + return; // ignore empty files + } + + // Pywb flocks open files instead of renaming them, so check for a file lock + try (FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { + var lock = channel.tryLock(); + if (lock == null) { + log.finest("WARC has file lock, treating as still open: " + path); + handleOpenWarc(watch, path); + return; + } + lock.release(); + } + String filename = path.getFileName().toString(); Warc warc = warcs.getOrNullByFilename(filename); Crawl crawl = crawls.get(watch.crawlId); - long size = Files.size(path); String digest = Scrub.calculateDigest("SHA-256", path); log.info("Moving now-closed WARC " + path);