fetch tool: --output-file option

src/org/netpreserve/jwarc/WarcCompression.java

@@ -5,6 +5,16 @@
 
 package org.netpreserve.jwarc;
 
+import java.nio.file.Path;
+
 public enum WarcCompression {
- NONE, GZIP
+ NONE, GZIP;
+
+ static WarcCompression forPath(Path path) {
+ if (path.getFileName().toString().endsWith(".gz")) {
+ return GZIP;
+ } else {
+ return NONE;
+ }
+ }
 }

src/org/netpreserve/jwarc/WarcWriter.java

@@ -60,6 +60,16 @@ public WarcWriter(OutputStream stream) throws IOException {
  this(Channels.newChannel(stream));
  }
 
+ /**
+ * Opens a WARC file for writing. Compression is determined by the file extension.
+ *
+ * @param path the path to the file
+ * @throws IOException if an I/O error occurs
+ */
+ public WarcWriter(Path path) throws IOException {
+ this(FileChannel.open(path, WRITE, CREATE, TRUNCATE_EXISTING), WarcCompression.forPath(path));
+ }
+
  public synchronized void write(WarcRecord record) throws IOException {
  // TODO: buffer headers
  position.addAndGet(channel.write(ByteBuffer.wrap(record.serializeHeader())));

src/org/netpreserve/jwarc/tools/FetchTool.java

@@ -6,13 +6,16 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 
 public class FetchTool {
  public static void main(String[] args) throws IOException, URISyntaxException {
  FetchOptions options = new FetchOptions();
  List<URI> urls = new ArrayList<>();
+ Path outputFile = null;
  for (int i = 0; i < args.length; i++) {
  switch (args[i]) {
  case "-h":
@@ -21,10 +24,12 @@ public static void main(String[] args) throws IOException, URISyntaxException {
  System.out.println("Fetches a URL while writing the request and response as WARC records");
  System.out.println();
  System.out.println("Options:");
- System.out.println("-A, --user-agent STRING Sets the User-Agent header");
- System.out.println(" --read-timeout MILLIS Sets the socket read timeout");
- System.out.println(" --max-length BYTES Truncate response after BYTES received");
- System.out.println(" --max-time MILLIS Truncate response after MILLIS elapsed");
+ System.out.println(" -A, --user-agent STRING Sets the User-Agent header");
+ System.out.println(" --read-timeout MILLIS Sets the socket read timeout");
+ System.out.println(" --max-length BYTES Truncate response after BYTES received");
+ System.out.println(" --max-time MILLIS Truncate response after MILLIS elapsed");
+ System.out.println(" -o, --output-file FILE Write WARC records to FILE instead of stdout");
+ System.out.println();
  System.exit(0);
  break;
  case "-A":
@@ -40,6 +45,10 @@ public static void main(String[] args) throws IOException, URISyntaxException {
  case "--max-time":
  options.maxTime(Integer.parseInt(args[++i]));
  break;
+ case "-o":
+ case "--output-file":
+ outputFile = Paths.get(args[++i]);
+ break;
  default:
  if (args[i].startsWith("-")) {
  System.err.println("Unknown option: " + args[i]);
@@ -52,7 +61,7 @@ public static void main(String[] args) throws IOException, URISyntaxException {
  System.err.println("No URLs specified. Try: jwarc fetch --help");
  System.exit(1);
  }
- try (WarcWriter writer = new WarcWriter(System.out)) {
+ try (WarcWriter writer = outputFile == null ? new WarcWriter(System.out) : new WarcWriter(outputFile)) {
  for (URI url : urls) {
  writer.fetch(url, options);
  }

src/org/netpreserve/jwarc/tools/RecorderTool.java

@@ -74,7 +74,7 @@ public static void main(String[] args) throws Exception {
  }
 
  ServerSocket serverSocket = new ServerSocket(port);
- WarcRecorder warcRecorder = new WarcRecorder(serverSocket, new WarcWriter(outputFile == null ? System.out : Files.newOutputStream(outputFile)));
+ WarcRecorder warcRecorder = new WarcRecorder(serverSocket, outputFile == null ? new WarcWriter(System.out) : new WarcWriter(outputFile));
 
  if (caCertificateSaveFile != null) {
  X509Certificate certificate = warcRecorder.certificateAuthority().certificate();

test/org/netpreserve/jwarc/apitests/WarcWriterTest.java

@@ -65,8 +65,6 @@ public void fetch() throws IOException, NoSuchAlgorithmException, URISyntaxExcep
 
  WarcResponse response = (WarcResponse) warcReader.next()
  .orElseThrow(() -> new RuntimeException("Missing response record"));
- System.out.println(new String(response.serializeHeader()));
- //assertEquals(12, response.http().body().size());
  assertEquals(256, response.http().status());
  assertEquals("present", response.http().headers().first("Test-Header").orElse(null));
  assertTrue(response.blockDigest().isPresent());