Merge pull request #43 from dfir-dd/42-mactime2-writes-invalid-csv-in…

…-some-cases

42 mactime2 writes invalid csv in some cases

src/bin/mactime2/application.rs

@@ -3,6 +3,8 @@ use clap::ValueEnum;
 use clio::Input;
 use strum_macros::Display;
 
+use crate::output::OldCsvOutput;
+
 use super::bodyfile::{BodyfileDecoder, BodyfileReader, BodyfileSorter};
 use super::cli::Cli;
 use super::error::MactimeError;
@@ -21,22 +23,29 @@ enum InputFormat {
 }
 
 #[derive(ValueEnum, Clone, Display)]
-pub (crate) enum OutputFormat {
+pub(crate) enum OutputFormat {
+    /// Comma-Separated Values compliant to RFC 4180
     #[strum(serialize = "csv")]
     Csv,
 
+    /// legacy text format, inherited from the old mactime
     #[strum(serialize = "txt")]
     Txt,
 
+    /// Javascript Object Notation
     #[strum(serialize = "json")]
     Json,
 
+    /// JSON-format to be used by elasticsearch
     #[cfg(feature = "elastic")]
     #[strum(serialize = "elastic")]
     Elastic,
+
+    /// Use the old (non RFC compliant) CSV format that was used by legacy mactime.
+    #[strum(serialize = "old-csv")]
+    OldCsv,
 }
 
-//#[derive(Builder)]
 pub struct Mactime2Application {
     format: OutputFormat,
     bodyfile: Input,
@@ -60,8 +69,12 @@ impl Mactime2Application {
                 BodyfileSorter::default().with_receiver(decoder.get_receiver(), options);
 
             sorter = sorter.with_output(match self.format {
-                OutputFormat::Csv => Box::new(CsvOutput::new(self.dst_zone)),
-                OutputFormat::Txt => Box::new(TxtOutput::new(self.dst_zone)),
+                OutputFormat::OldCsv => {
+                    Box::new(OldCsvOutput::new(std::io::stdout(), self.dst_zone))
+                }
+
+                OutputFormat::Csv => Box::new(CsvOutput::new(std::io::stdout(), self.dst_zone)),
+                OutputFormat::Txt => Box::new(TxtOutput::new(std::io::stdout(), self.dst_zone)),
                 _ => panic!("invalid execution path"),
             });
             Box::new(sorter)

src/bin/mactime2/bodyfile/bodyfile_sorter.rs

@@ -1,7 +1,8 @@
-use dfir_toolkit::common::bodyfile::{Bodyfile3Line, BehavesLikeI64};
+use dfir_toolkit::common::bodyfile::{BehavesLikeI64, Bodyfile3Line};
 use std::borrow::Borrow;
 use std::cmp::Ordering;
 use std::collections::{BTreeMap, HashSet};
+use std::io::{Stdout, Write};
 use std::sync::mpsc::Receiver;
 use std::sync::Arc;
 use std::thread::JoinHandle;
@@ -11,18 +12,21 @@ use crate::filter::{Joinable, RunOptions, Runnable, Sorter};
 
 use super::MACBFlags;
 
-pub trait Mactime2Writer: Send {
-    fn write(&self, timestamp: &i64, entry: &ListEntry) {
-        println!("{}", self.fmt(timestamp, entry));
-    }
-    fn fmt(&self, timestamp: &i64, entry: &ListEntry) -> String;
+pub trait Mactime2Writer<W>: Send
+where
+    W: Write + Send
+{
+    fn write_line(&mut self, timestamp: &i64, entry: &ListEntry) -> std::io::Result<()>;
+
+    #[allow(dead_code)]
+    fn into_writer(self) -> W;
 }
 
 #[derive(Default)]
 pub struct BodyfileSorter {
     worker: Option<JoinHandle<Result<(), MactimeError>>>,
     receiver: Option<Receiver<Bodyfile3Line>>,
-    output: Option<Box<dyn Mactime2Writer>>,
+    output: Option<Box<dyn Mactime2Writer<Stdout>>>,
 }
 
 #[derive(Debug)]
@@ -105,14 +109,14 @@ impl BodyfileSorter {
         self
     }
 
-    pub fn with_output(mut self, output: Box<dyn Mactime2Writer>) -> Self {
+    pub fn with_output(mut self, output: Box<dyn Mactime2Writer<Stdout>>) -> Self {
         self.output = Some(output);
         self
     }
 
     fn worker(
         decoder: Receiver<Bodyfile3Line>,
-        output: Box<dyn Mactime2Writer>,
+        mut output: Box<dyn Mactime2Writer<Stdout>>,
     ) -> Result<(), MactimeError> {
         let mut entries: BTreeMap<i64, Vec<ListEntry>> = BTreeMap::new();
         let mut names: HashSet<(String, String)> = HashSet::new();
@@ -189,7 +193,7 @@ impl BodyfileSorter {
 
         for (ts, entries_at_ts) in entries.iter() {
             for line in entries_at_ts {
-                output.write(ts, line);
+                output.write_line(ts, line)?;
             }
         }
         Ok(())

src/bin/mactime2/bodyfile/macb_flags.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 
 use bitflags::bitflags;
+use serde::Serialize;
 
 bitflags! {
     #[derive(PartialEq, Debug, Clone, Copy)]
@@ -22,3 +23,12 @@ impl fmt::Display for MACBFlags {
         write!(f, "{}{}{}{}", m, a, c, b)
     }
 }
+
+impl Serialize for MACBFlags {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&format!("{self}"))
+    }
+}

src/bin/mactime2/cli.rs

@@ -39,6 +39,7 @@ pub struct Cli {
     /// output as CSV instead of TXT. This is a convenience option, which is identical to `--format=csv`
     /// and will be removed in a future release. If you specified `--format` and `-d`, the latter will be ignored.
     #[clap(short('d'), display_order(610))]
+    #[arg(group="csv")]
     pub(crate) csv_format: bool,
 
     /// output as JSON instead of TXT. This is a convenience option, which is identical to `--format=json`

src/bin/mactime2/error.rs

@@ -2,4 +2,12 @@ use thiserror::Error;
 
 #[derive(Error, Debug)]
 pub enum MactimeError {
+    #[error("An IO Error has occurred: {0}")]
+    IoError(std::io::Error)
+}
+
+impl From<std::io::Error> for MactimeError {
+    fn from(value: std::io::Error) -> Self {
+        Self::IoError(value)
+    }
 }

src/bin/mactime2/output/csv_output.rs

@@ -1,35 +1,81 @@
+use std::io::Write;
+
 use chrono_tz::Tz;
+use csv::WriterBuilder;
 use dfir_toolkit::common::ForensicsTimestamp;
+use serde::Serialize;
 
-use crate::bodyfile::{ListEntry, Mactime2Writer};
+use crate::bodyfile::{ListEntry, MACBFlags, Mactime2Writer};
 
-pub(crate) struct CsvOutput {
+pub(crate) struct CsvOutput<W>
+where
+    W: Write + Send,
+{
     dst_zone: Tz,
+    writer: csv::Writer<W>,
 }
 
-impl CsvOutput {
-    pub fn new(dst_zone: Tz) -> Self {
-        Self { dst_zone }
+pub const CSV_DELIMITER: u8 = b',';
+
+impl<W> CsvOutput<W>
+where
+    W: Write + Send,
+{
+    pub fn new(writer: W, dst_zone: Tz) -> Self {
+        Self {
+            dst_zone,
+            writer: WriterBuilder::new()
+                .delimiter(CSV_DELIMITER)
+                .has_headers(false)
+                .from_writer(writer),
+        }
+    }
+    #[allow(dead_code)]
+    pub fn with_writer(mut self, writer: W) -> Self
+    where
+        W: Write + Send + 'static,
+    {
+        self.writer = WriterBuilder::new().from_writer(writer);
+        self
     }
 }
 
-impl Mactime2Writer for CsvOutput {
-    fn fmt(&self, timestamp: &i64, entry: &ListEntry) -> String {
-        let timestamp = ForensicsTimestamp::from(*timestamp).with_timezone(self.dst_zone);
-        format!(
-            "{},{},{},{},{},{},{},\"{}\"",
-            timestamp,
-            entry.line.get_size(),
-            entry.flags,
-            entry.line.get_mode_as_string(),
-            entry.line.get_uid(),
-            entry.line.get_gid(),
-            entry.line.get_inode(),
-            entry.line.get_name()
-        )
+impl<W> Mactime2Writer<W> for CsvOutput<W>
+where
+    W: Write + Send,
+{
+    fn write_line(&mut self, timestamp: &i64, entry: &ListEntry) -> std::io::Result<()> {
+        let csv_line = CsvLine {
+            timestamp: ForensicsTimestamp::new(*timestamp, self.dst_zone),
+            size: entry.line.get_size(),
+            flags: entry.flags,
+            mode: entry.line.get_mode_as_string(),
+            uid: entry.line.get_uid(),
+            gid: entry.line.get_gid(),
+            inode: entry.line.get_inode(),
+            name: entry.line.get_name(),
+        };
+        self.writer.serialize(csv_line)?;
+        Ok(())
+    }
+
+    fn into_writer(self) -> W {
+        self.writer.into_inner().unwrap()
     }
 }
 
+#[derive(Serialize)]
+struct CsvLine<'e> {
+    timestamp: ForensicsTimestamp,
+    size: &'e u64,
+    flags: MACBFlags,
+    mode: &'e str,
+    uid: &'e u64,
+    gid: &'e u64,
+    inode: &'e str,
+    name: &'e str,
+}
+
 #[cfg(test)]
 mod tests {
     use crate::bodyfile::ListEntry;
@@ -41,6 +87,9 @@ mod tests {
     use chrono_tz::Tz;
     use chrono_tz::TZ_VARIANTS;
     use dfir_toolkit::common::bodyfile::Bodyfile3Line;
+    use std::io::BufRead;
+    use std::io::BufReader;
+    use std::io::Cursor;
     use std::sync::Arc;
 
     fn random_tz() -> Tz {
@@ -51,7 +100,6 @@ mod tests {
     #[allow(non_snake_case)]
     #[test]
     fn test_correct_ts_UTC() {
-        let output = CsvOutput::new(Tz::UTC);
         for _ in 1..10 {
             let unix_ts = rand::random::<u32>() as i64;
             let bf_line = Bodyfile3Line::new().with_crtime(unix_ts.into());
@@ -60,7 +108,11 @@ mod tests {
                 line: Arc::new(bf_line),
             };
 
-            let out_line = output.fmt(&unix_ts, &entry);
+            let mut output = CsvOutput::new(Cursor::new(vec![]), Tz::UTC);
+            output.write_line(&unix_ts, &entry).unwrap();
+            let mut output = BufReader::new(Cursor::new(output.into_writer().into_inner())).lines();
+            let out_line = output.next().unwrap().unwrap();
+
             let out_ts = out_line.split(',').next().unwrap();
             let rfc3339 = DateTime::parse_from_rfc3339(out_ts)
                 .expect(out_ts)
@@ -77,16 +129,20 @@ mod tests {
     fn test_correct_ts_random_tz() -> Result<(), String> {
         for _ in 1..100 {
             let tz = random_tz();
-            let output = CsvOutput::new(tz);
             let unix_ts = rand::random::<u32>() as i64;
             let bf_line = Bodyfile3Line::new().with_crtime(unix_ts.into());
             let entry = ListEntry {
                 flags: MACBFlags::B,
                 line: Arc::new(bf_line),
             };
 
-            let out_line = output.fmt(&unix_ts, &entry);
-            let out_ts = out_line.split(',').next().unwrap();
+            let mut output = CsvOutput::new(Cursor::new(vec![]), tz);
+            let delimiter: char = crate::output::CSV_DELIMITER.into();
+            output.write_line(&unix_ts, &entry).unwrap();
+            let mut output = BufReader::new(Cursor::new(output.into_writer().into_inner())).lines();
+            let out_line = output.next().unwrap().unwrap();
+
+            let out_ts = out_line.split(delimiter).next().unwrap();
             let rfc3339 = match DateTime::parse_from_rfc3339(out_ts) {
                 Ok(ts) => ts,
                 Err(e) => return Err(format!("error while parsing '{}': {}", out_ts, e)),

src/bin/mactime2/output/mod.rs

@@ -1,7 +1,9 @@
 mod csv_output;
+mod old_csv_output;
 mod txt_output;
 mod json_sorter;
 
 pub (crate) use csv_output::*;
+pub (crate) use old_csv_output::*;
 pub (crate) use txt_output::*;
 pub (crate) use json_sorter::*;