From e450230bb1af630375afb156f57890e6d766fd6e Mon Sep 17 00:00:00 2001 From: Jan Starke Date: Wed, 15 May 2024 12:48:38 +0200 Subject: [PATCH 1/6] use generic W: Write+Send to export timeline data --- Cargo.lock | 4 +- src/bin/mactime2/application.rs | 4 +- src/bin/mactime2/bodyfile/bodyfile_sorter.rs | 22 ++-- src/bin/mactime2/bodyfile/macb_flags.rs | 10 ++ src/bin/mactime2/error.rs | 8 ++ src/bin/mactime2/output/csv_output.rs | 105 ++++++++++++++----- src/bin/mactime2/output/txt_output.rs | 65 +++++++++--- src/common/forensics_timestamp.rs | 9 ++ tests/mactime2/is_stable_sorting.rs | 31 ++---- tests/mactime2/mod.rs | 4 +- 10 files changed, 186 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0f5100..541e0d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -702,7 +702,7 @@ dependencies = [ [[package]] name = "dfir-toolkit" -version = "0.10.2" +version = "0.10.3" dependencies = [ "anyhow", "assert-json-diff", @@ -742,7 +742,7 @@ dependencies = [ "more-asserts", "nt_hive2", "num", - "num-derive 0.4.0", + "num-derive 0.3.3", "num-traits", "ouroboros", "phf", diff --git a/src/bin/mactime2/application.rs b/src/bin/mactime2/application.rs index e4fffcd..753ce37 100644 --- a/src/bin/mactime2/application.rs +++ b/src/bin/mactime2/application.rs @@ -62,8 +62,8 @@ impl Mactime2Application { BodyfileSorter::default().with_receiver(decoder.get_receiver(), options); sorter = sorter.with_output(match self.format { - OutputFormat::Csv => Box::new(CsvOutput::new(self.src_zone, self.dst_zone)), - OutputFormat::Txt => Box::new(TxtOutput::new(self.src_zone, self.dst_zone)), + OutputFormat::Csv => Box::new(CsvOutput::new(std::io::stdout(), self.src_zone, self.dst_zone)), + OutputFormat::Txt => Box::new(TxtOutput::new(std::io::stdout(), self.src_zone, self.dst_zone)), _ => panic!("invalid execution path"), }); Box::new(sorter) diff --git a/src/bin/mactime2/bodyfile/bodyfile_sorter.rs b/src/bin/mactime2/bodyfile/bodyfile_sorter.rs index d2f7ea6..aff8040 100644 --- a/src/bin/mactime2/bodyfile/bodyfile_sorter.rs +++ b/src/bin/mactime2/bodyfile/bodyfile_sorter.rs @@ -1,7 +1,8 @@ -use dfir_toolkit::common::bodyfile::{Bodyfile3Line, BehavesLikeI64}; +use dfir_toolkit::common::bodyfile::{BehavesLikeI64, Bodyfile3Line}; use std::borrow::Borrow; use std::cmp::Ordering; use std::collections::{BTreeMap, HashSet}; +use std::io::{Stdout, Write}; use std::sync::mpsc::Receiver; use std::sync::Arc; use std::thread::JoinHandle; @@ -11,18 +12,19 @@ use crate::filter::{Joinable, RunOptions, Runnable, Sorter}; use super::MACBFlags; -pub trait Mactime2Writer: Send { - fn write(&self, timestamp: &i64, entry: &ListEntry) { - println!("{}", self.fmt(timestamp, entry)); - } - fn fmt(&self, timestamp: &i64, entry: &ListEntry) -> String; +pub trait Mactime2Writer: Send +where + W: Write + Send +{ + fn write_line(&mut self, timestamp: &i64, entry: &ListEntry) -> std::io::Result<()>; + fn into_writer(self) -> W; } #[derive(Default)] pub struct BodyfileSorter { worker: Option>>, receiver: Option>, - output: Option>, + output: Option>>, } #[derive(Debug)] @@ -105,14 +107,14 @@ impl BodyfileSorter { self } - pub fn with_output(mut self, output: Box) -> Self { + pub fn with_output(mut self, output: Box>) -> Self { self.output = Some(output); self } fn worker( decoder: Receiver, - output: Box, + mut output: Box>, ) -> Result<(), MactimeError> { let mut entries: BTreeMap> = BTreeMap::new(); let mut names: HashSet<(String, String)> = HashSet::new(); @@ -189,7 +191,7 @@ impl BodyfileSorter { for (ts, entries_at_ts) in entries.iter() { for line in entries_at_ts { - output.write(ts, line); + output.write_line(ts, line)?; } } Ok(()) diff --git a/src/bin/mactime2/bodyfile/macb_flags.rs b/src/bin/mactime2/bodyfile/macb_flags.rs index 95a40b3..bd594d6 100644 --- a/src/bin/mactime2/bodyfile/macb_flags.rs +++ b/src/bin/mactime2/bodyfile/macb_flags.rs @@ -1,6 +1,7 @@ use std::fmt; use bitflags::bitflags; +use serde::Serialize; bitflags! { #[derive(PartialEq, Debug, Clone, Copy)] @@ -22,3 +23,12 @@ impl fmt::Display for MACBFlags { write!(f, "{}{}{}{}", m, a, c, b) } } + +impl Serialize for MACBFlags { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&format!("{self}")) + } +} diff --git a/src/bin/mactime2/error.rs b/src/bin/mactime2/error.rs index 81834c8..a4685bd 100644 --- a/src/bin/mactime2/error.rs +++ b/src/bin/mactime2/error.rs @@ -2,4 +2,12 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum MactimeError { + #[error("An IO Error has occurred: {0}")] + IoError(std::io::Error) +} + +impl From for MactimeError { + fn from(value: std::io::Error) -> Self { + Self::IoError(value) + } } \ No newline at end of file diff --git a/src/bin/mactime2/output/csv_output.rs b/src/bin/mactime2/output/csv_output.rs index 7bf74c7..54b8d92 100644 --- a/src/bin/mactime2/output/csv_output.rs +++ b/src/bin/mactime2/output/csv_output.rs @@ -1,36 +1,83 @@ +use std::io::Write; + use chrono_tz::Tz; +use csv::WriterBuilder; use dfir_toolkit::common::ForensicsTimestamp; +use serde::Serialize; -use crate::bodyfile::{ListEntry, Mactime2Writer}; +use crate::bodyfile::{ListEntry, MACBFlags, Mactime2Writer}; -pub(crate) struct CsvOutput { +pub(crate) struct CsvOutput +where + W: Write + Send, +{ src_zone: Tz, dst_zone: Tz, + writer: csv::Writer, } -impl CsvOutput { - pub fn new(src_zone: Tz, dst_zone: Tz) -> Self { - Self { src_zone, dst_zone } +pub const CSV_DELIMITER: u8 = b','; + +impl CsvOutput +where + W: Write + Send, +{ + pub fn new(writer: W, src_zone: Tz, dst_zone: Tz) -> Self { + Self { + src_zone, + dst_zone, + writer: WriterBuilder::new() + .delimiter(CSV_DELIMITER) + .has_headers(false) + .from_writer(writer), + } + } + #[allow(dead_code)] + pub fn with_writer(mut self, writer: W) -> Self + where + W: Write + Send + 'static, + { + self.writer = WriterBuilder::new().from_writer(writer); + self } } -impl Mactime2Writer for CsvOutput { - fn fmt(&self, timestamp: &i64, entry: &ListEntry) -> String { - let timestamp = ForensicsTimestamp::new(*timestamp, self.src_zone, self.dst_zone); - format!( - "{},{},{},{},{},{},{},\"{}\"", - timestamp, - entry.line.get_size(), - entry.flags, - entry.line.get_mode_as_string(), - entry.line.get_uid(), - entry.line.get_gid(), - entry.line.get_inode(), - entry.line.get_name() - ) +impl Mactime2Writer for CsvOutput +where + W: Write + Send, +{ + fn write_line(&mut self, timestamp: &i64, entry: &ListEntry) -> std::io::Result<()> { + let csv_line = CsvLine { + timestamp: ForensicsTimestamp::new(*timestamp, self.src_zone, self.dst_zone), + size: entry.line.get_size(), + flags: entry.flags, + mode: entry.line.get_mode_as_string(), + uid: entry.line.get_uid(), + gid: entry.line.get_gid(), + inode: entry.line.get_inode(), + name: entry.line.get_name(), + }; + self.writer.serialize(csv_line)?; + Ok(()) + } + + fn into_writer(self) -> W { + self.writer.into_inner().unwrap() } } +#[derive(Serialize)] +struct CsvLine<'e> { + timestamp: ForensicsTimestamp, + size: &'e u64, + flags: MACBFlags, + mode: &'e str, + uid: &'e u64, + gid: &'e u64, + inode: &'e str, + name: &'e str, +} + #[cfg(test)] mod tests { use crate::bodyfile::ListEntry; @@ -42,6 +89,9 @@ mod tests { use chrono_tz::Tz; use chrono_tz::TZ_VARIANTS; use dfir_toolkit::common::bodyfile::Bodyfile3Line; + use std::io::BufRead; + use std::io::BufReader; + use std::io::Cursor; use std::sync::Arc; fn random_tz() -> Tz { @@ -52,7 +102,6 @@ mod tests { #[allow(non_snake_case)] #[test] fn test_correct_ts_UTC() { - let output = CsvOutput::new(Tz::UTC, Tz::UTC); for _ in 1..10 { let unix_ts = rand::random::() as i64; let bf_line = Bodyfile3Line::new().with_crtime(unix_ts.into()); @@ -61,7 +110,11 @@ mod tests { line: Arc::new(bf_line), }; - let out_line = output.fmt(&unix_ts, &entry); + let mut output = CsvOutput::new(Cursor::new(vec![]), Tz::UTC, Tz::UTC); + output.write_line(&unix_ts, &entry).unwrap(); + let mut output = BufReader::new(Cursor::new(output.into_writer().into_inner())).lines(); + let out_line = output.next().unwrap().unwrap(); + let out_ts = out_line.split(',').next().unwrap(); let rfc3339 = DateTime::parse_from_rfc3339(out_ts).expect(out_ts); assert_eq!( @@ -80,7 +133,6 @@ mod tests { fn test_correct_ts_random_tz() -> Result<(), String> { for _ in 1..100 { let tz = random_tz(); - let output = CsvOutput::new(tz, tz); let unix_ts = rand::random::() as i64; let bf_line = Bodyfile3Line::new().with_crtime(unix_ts.into()); let entry = ListEntry { @@ -88,8 +140,13 @@ mod tests { line: Arc::new(bf_line), }; - let out_line = output.fmt(&unix_ts, &entry); - let out_ts = out_line.split(',').next().unwrap(); + let mut output = CsvOutput::new(Cursor::new(vec![]), tz, tz); + let delimiter: char = crate::output::CSV_DELIMITER.into(); + output.write_line(&unix_ts, &entry).unwrap(); + let mut output = BufReader::new(Cursor::new(output.into_writer().into_inner())).lines(); + let out_line = output.next().unwrap().unwrap(); + + let out_ts = out_line.split(delimiter).next().unwrap(); let rfc3339 = match DateTime::parse_from_rfc3339(out_ts) { Ok(ts) => ts, Err(e) => return Err(format!("error while parsing '{}': {}", out_ts, e)), diff --git a/src/bin/mactime2/output/txt_output.rs b/src/bin/mactime2/output/txt_output.rs index 97330eb..3f63cca 100644 --- a/src/bin/mactime2/output/txt_output.rs +++ b/src/bin/mactime2/output/txt_output.rs @@ -1,29 +1,49 @@ use chrono_tz::Tz; use dfir_toolkit::common::ForensicsTimestamp; -use std::cell::RefCell; +use std::{cell::RefCell, io::Write}; use crate::bodyfile::{ListEntry, Mactime2Writer}; -pub struct TxtOutput { +pub struct TxtOutput +where + W: Write + Send, +{ src_zone: Tz, dst_zone: Tz, last_ts: (RefCell, RefCell), empty_ts: RefCell, + writer: W, } -impl TxtOutput { - pub fn new(src_zone: Tz, dst_zone: Tz) -> Self { +impl TxtOutput +where + W: Write + Send, +{ + pub fn new(writer: W, src_zone: Tz, dst_zone: Tz) -> Self { Self { src_zone, dst_zone, last_ts: (RefCell::new(i64::MIN), RefCell::new("".to_owned())), empty_ts: RefCell::new(" ".to_owned()), + writer, } } + + #[allow(dead_code)] + pub fn with_writer(mut self, writer: W) -> Self + where + W: Write + Send + 'static, + { + self.writer = writer; + self + } } -impl Mactime2Writer for TxtOutput { - fn fmt(&self, timestamp: &i64, entry: &ListEntry) -> String { +impl Mactime2Writer for TxtOutput +where + W: Write + Send, +{ + fn write_line(&mut self, timestamp: &i64, entry: &ListEntry) -> std::io::Result<()> { let ts = if *timestamp != *self.last_ts.0.borrow() { *self.last_ts.1.borrow_mut() = ForensicsTimestamp::new(*timestamp, self.src_zone, self.dst_zone).to_string(); @@ -32,7 +52,8 @@ impl Mactime2Writer for TxtOutput { } else { self.empty_ts.borrow() }; - format!( + writeln!( + &mut self.writer, "{} {:>8} {} {:<12} {:<7} {:<7} {} {}", ts, entry.line.get_size(), @@ -44,17 +65,23 @@ impl Mactime2Writer for TxtOutput { entry.line.get_name() ) } + + fn into_writer(self) -> W { + self.writer + } } #[cfg(test)] mod tests { use super::TxtOutput; - use crate::bodyfile::{ListEntry, MACBFlags, Mactime2Writer}; + use crate::bodyfile::Mactime2Writer; + use crate::bodyfile::{ListEntry, MACBFlags}; use chrono::DateTime; use chrono_tz::Tz; use chrono_tz::TZ_VARIANTS; use dfir_toolkit::common::bodyfile::Bodyfile3Line; use dfir_toolkit::common::bodyfile::Created; + use std::io::{BufRead, BufReader, Cursor}; use std::sync::Arc; fn random_tz() -> Tz { @@ -65,7 +92,6 @@ mod tests { #[allow(non_snake_case)] #[test] fn test_correct_ts_UTC() { - let output = TxtOutput::new(Tz::UTC, Tz::UTC); for _ in 1..10 { let unix_ts = rand::random::() as i64; let bf_line = Bodyfile3Line::new().with_crtime(Created::from(unix_ts)); @@ -74,8 +100,13 @@ mod tests { line: Arc::new(bf_line), }; - let out_line = output.fmt(&unix_ts, &entry); - let out_line2 = output.fmt(&unix_ts, &entry); + let mut output = TxtOutput::new(Cursor::new(vec![]), Tz::UTC, Tz::UTC); + output.write_line(&unix_ts, &entry).unwrap(); + output.write_line(&unix_ts, &entry).unwrap(); + let mut output = BufReader::new(Cursor::new(output.into_writer().into_inner())).lines(); + + let out_line = output.next().unwrap().unwrap(); + let out_line2 = output.next().unwrap().unwrap(); assert!(out_line2.starts_with(' ')); let out_ts = out_line.split(' ').next().unwrap(); @@ -83,7 +114,7 @@ mod tests { assert_eq!( unix_ts, rfc3339.timestamp(), - "Timestamp {} converted to '{}' and back to {}", + "Timestamp {} converted to '{}' and back to {}", unix_ts, out_ts, rfc3339.timestamp() @@ -96,7 +127,6 @@ mod tests { fn test_correct_ts_random_tz() -> Result<(), String> { for _ in 1..100 { let tz = random_tz(); - let output = TxtOutput::new(tz, tz); let unix_ts = rand::random::() as i64; let bf_line = Bodyfile3Line::new().with_crtime(Created::from(unix_ts)); let entry = ListEntry { @@ -104,8 +134,13 @@ mod tests { line: Arc::new(bf_line), }; - let out_line = output.fmt(&unix_ts, &entry); - let out_line2 = output.fmt(&unix_ts, &entry); + let mut output = TxtOutput::new(Cursor::new(vec![]), tz, tz); + output.write_line(&unix_ts, &entry).unwrap(); + output.write_line(&unix_ts, &entry).unwrap(); + let mut output = BufReader::new(Cursor::new(output.into_writer().into_inner())).lines(); + + let out_line = output.next().unwrap().unwrap(); + let out_line2 = output.next().unwrap().unwrap(); assert!(out_line2.starts_with(' ')); let out_ts = out_line.split(' ').next().unwrap(); diff --git a/src/common/forensics_timestamp.rs b/src/common/forensics_timestamp.rs index 6027faa..11e7798 100644 --- a/src/common/forensics_timestamp.rs +++ b/src/common/forensics_timestamp.rs @@ -5,6 +5,7 @@ use chrono::offset::TimeZone; use chrono::{DateTime, FixedOffset, LocalResult, NaiveDateTime}; use chrono_tz::Tz; use lazy_static::lazy_static; +use serde::Serialize; lazy_static! { static ref TIMESTAMP_FORMAT: Option = { @@ -84,3 +85,11 @@ impl Display for ForensicsTimestamp { } } } + +impl Serialize for ForensicsTimestamp { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer { + serializer.serialize_str(&format!("{self}")) + } +} diff --git a/tests/mactime2/is_stable_sorting.rs b/tests/mactime2/is_stable_sorting.rs index 740f096..c83b47e 100644 --- a/tests/mactime2/is_stable_sorting.rs +++ b/tests/mactime2/is_stable_sorting.rs @@ -1,6 +1,5 @@ -use std::io::{BufReader, Cursor, BufRead}; -use lazy_regex::regex; use assert_cmd::Command; +use std::io::{BufReader, Cursor}; /// tests if the result of `mactime2` is a stable sort, i.e. if pos(a)<=pos(b) then sorted_pos(a) <= sorted_pos(b) #[test] @@ -20,25 +19,13 @@ fn is_stable_sorted() { .ok(); assert!(result.is_ok()); - let reader = BufReader::new(Cursor::new(result.unwrap().stdout)); - let lines: Vec<_> = reader - .lines() - .map_while(Result::ok) - .map(name_of) + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(BufReader::new(Cursor::new(result.unwrap().stdout))); + let names: Vec<_> = reader + .records() + .filter_map(Result::ok) + .map(|record| record.get(7).unwrap().to_owned()) .collect(); - assert_eq!(lines.len(), 3); - assert_eq!(lines[0].as_ref().unwrap(), "a"); - assert_eq!(lines[1].as_ref().unwrap(), "b"); - assert_eq!(lines[2].as_ref().unwrap(), "c"); + assert_eq!(names, vec!["a", "b", "c"]); } - -fn name_of(line: String) -> Option { - let re = regex!(r#""(?P[^"]*)""#); - let result = re.captures_iter(&line); - for c in result { - if let Some(name) = c.name("name") { - return Some(name.as_str().to_owned()) - } - } - None -} \ No newline at end of file diff --git a/tests/mactime2/mod.rs b/tests/mactime2/mod.rs index 8a5875e..007535d 100644 --- a/tests/mactime2/mod.rs +++ b/tests/mactime2/mod.rs @@ -1,4 +1,6 @@ mod help; mod autocomplete; mod is_sorted; -mod is_stable_sorting; \ No newline at end of file +mod is_stable_sorting; + +mod csv_output; \ No newline at end of file From 8b7d7efc78c9d6a069849d20e1259abb5daa61bb Mon Sep 17 00:00:00 2001 From: Jan Starke Date: Wed, 15 May 2024 12:57:37 +0200 Subject: [PATCH 2/6] use the csv crate to print csv --- Cargo.lock | 2 +- Cargo.toml | 2 +- tests/data/mactime2/csv_test.bodyfile | 1 + tests/mactime2/csv_output.rs | 27 +++++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tests/data/mactime2/csv_test.bodyfile create mode 100644 tests/mactime2/csv_output.rs diff --git a/Cargo.lock b/Cargo.lock index 541e0d6..f9410f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -702,7 +702,7 @@ dependencies = [ [[package]] name = "dfir-toolkit" -version = "0.10.3" +version = "0.10.4" dependencies = [ "anyhow", "assert-json-diff", diff --git a/Cargo.toml b/Cargo.toml index b0540bf..e4be1bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dfir-toolkit" -version = "0.10.3" +version = "0.10.4" edition = "2021" authors = ["Jan Starke ", "Deborah Mahn "] description = "CLI tools for digital forensics and incident response" diff --git a/tests/data/mactime2/csv_test.bodyfile b/tests/data/mactime2/csv_test.bodyfile new file mode 100644 index 0000000..cee425d --- /dev/null +++ b/tests/data/mactime2/csv_test.bodyfile @@ -0,0 +1 @@ +0|{"activity_id":null,"channel_name":"Microsoft-Windows-WER-PayloadHealth/Operational","computer":"WIN-J56D9ENVG6H","custom_data":{"EventData":{"#attributes":{"Name":"WER_PAYLOAD_HEALTH_FAIL"},"BytesUploaded":0,"HttpExchangeResult":2147954402,"PayloadSize":4569,"Protocol":"Watson","RequestStatusCode":0,"ServerName":"umwatson.events.data.microsoft.com","Stage":"s1event","TransportHr":2147954402,"UploadDuration":21094}},"event_id":2,"event_record_id":1,"level":4,"provider_name":"Microsoft-Windows-WER-PayloadHealth","timestamp":"2022-11-16T08:26:43.409044Z"}|0||0|0|0|-1|1668587203|-1|-1 \ No newline at end of file diff --git a/tests/mactime2/csv_output.rs b/tests/mactime2/csv_output.rs new file mode 100644 index 0000000..f370122 --- /dev/null +++ b/tests/mactime2/csv_output.rs @@ -0,0 +1,27 @@ +use std::{ + io::{BufReader, Cursor}, + path::PathBuf, +}; + +use assert_cmd::Command; + +/// tests if the result of `mactime2` is always sorted +#[test] +fn csv_output() { + let mut cmd = Command::cargo_bin("mactime2").unwrap(); + let mut data_path = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); + data_path.push("tests"); + data_path.push("data"); + data_path.push("mactime2"); + data_path.push("csv_test.bodyfile"); + + let result = cmd.arg("-d").arg("-b").arg(data_path).ok(); + assert!(result.is_ok()); + + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(BufReader::new(Cursor::new(result.unwrap().stdout))); + let first_line = reader.records().next().unwrap().unwrap(); + + assert_eq!(first_line.get(7).unwrap(), r##"{"activity_id":null,"channel_name":"Microsoft-Windows-WER-PayloadHealth/Operational","computer":"WIN-J56D9ENVG6H","custom_data":{"EventData":{"#attributes":{"Name":"WER_PAYLOAD_HEALTH_FAIL"},"BytesUploaded":0,"HttpExchangeResult":2147954402,"PayloadSize":4569,"Protocol":"Watson","RequestStatusCode":0,"ServerName":"umwatson.events.data.microsoft.com","Stage":"s1event","TransportHr":2147954402,"UploadDuration":21094}},"event_id":2,"event_record_id":1,"level":4,"provider_name":"Microsoft-Windows-WER-PayloadHealth","timestamp":"2022-11-16T08:26:43.409044Z"}"##); +} From c286fbd0439b9759d969ef861b61595561704ec0 Mon Sep 17 00:00:00 2001 From: Jan Starke Date: Wed, 12 Jun 2024 23:24:45 +0200 Subject: [PATCH 3/6] add old csv format --- src/bin/mactime2/application.rs | 9 +- src/bin/mactime2/cli.rs | 10 ++ src/bin/mactime2/output/mod.rs | 2 + src/bin/mactime2/output/old_csv_output.rs | 129 ++++++++++++++++++++++ 4 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 src/bin/mactime2/output/old_csv_output.rs diff --git a/src/bin/mactime2/application.rs b/src/bin/mactime2/application.rs index e0989e4..2a0f88c 100644 --- a/src/bin/mactime2/application.rs +++ b/src/bin/mactime2/application.rs @@ -3,6 +3,8 @@ use clap::ValueEnum; use clio::Input; use strum_macros::Display; +use crate::output::OldCsvOutput; + use super::bodyfile::{BodyfileDecoder, BodyfileReader, BodyfileSorter}; use super::cli::Cli; use super::error::MactimeError; @@ -38,6 +40,7 @@ pub (crate) enum OutputFormat { pub struct Mactime2Application { format: OutputFormat, + old_csv: bool, bodyfile: Input, dst_zone: Tz, strict_mode: bool, @@ -59,7 +62,10 @@ impl Mactime2Application { BodyfileSorter::default().with_receiver(decoder.get_receiver(), options); sorter = sorter.with_output(match self.format { - OutputFormat::Csv => Box::new(CsvOutput::new(std::io::stdout(), self.dst_zone)), + OutputFormat::Csv => if self.old_csv { + Box::new(OldCsvOutput::new(std::io::stdout(), self.dst_zone)) + } else { + Box::new(CsvOutput::new(std::io::stdout(), self.dst_zone))} OutputFormat::Txt => Box::new(TxtOutput::new(std::io::stdout(), self.dst_zone)), _ => panic!("invalid execution path"), }); @@ -101,6 +107,7 @@ impl From for Mactime2Application { Self { format, + old_csv: cli.old_csv, bodyfile: cli.input_file, dst_zone: cli.dst_zone.into_tz().unwrap(), strict_mode: cli.strict_mode, diff --git a/src/bin/mactime2/cli.rs b/src/bin/mactime2/cli.rs index 9beb263..8c703ff 100644 --- a/src/bin/mactime2/cli.rs +++ b/src/bin/mactime2/cli.rs @@ -39,8 +39,18 @@ pub struct Cli { /// output as CSV instead of TXT. This is a convenience option, which is identical to `--format=csv` /// and will be removed in a future release. If you specified `--format` and `-d`, the latter will be ignored. #[clap(short('d'), display_order(610))] + #[arg(group="csv")] pub(crate) csv_format: bool, + /// use the old CSV format that was used by legacy mactime. + /// + /// Keep in mind that in this format, fields which contain commas will + /// not be wrapped by quotes, as RFC4180 requires it. So, this format + /// is not RFC4180-compliant, which means that you might not be able + /// to use the output together with csv processing tools. + #[clap(long("old-csv"), display_order(615))] + pub(crate) old_csv: bool, + /// output as JSON instead of TXT. This is a convenience option, which is identical to `--format=json` /// and will be removed in a future release. If you specified `--format` and `-j`, the latter will be ignored. #[clap(short('j'), display_order(620))] diff --git a/src/bin/mactime2/output/mod.rs b/src/bin/mactime2/output/mod.rs index 1296d9b..c6b8115 100644 --- a/src/bin/mactime2/output/mod.rs +++ b/src/bin/mactime2/output/mod.rs @@ -1,7 +1,9 @@ mod csv_output; +mod old_csv_output; mod txt_output; mod json_sorter; pub (crate) use csv_output::*; +pub (crate) use old_csv_output::*; pub (crate) use txt_output::*; pub (crate) use json_sorter::*; \ No newline at end of file diff --git a/src/bin/mactime2/output/old_csv_output.rs b/src/bin/mactime2/output/old_csv_output.rs new file mode 100644 index 0000000..ae8d1ec --- /dev/null +++ b/src/bin/mactime2/output/old_csv_output.rs @@ -0,0 +1,129 @@ +use std::io::Write; + +use chrono_tz::Tz; +use dfir_toolkit::common::ForensicsTimestamp; + +use crate::bodyfile::{ListEntry, Mactime2Writer}; + +pub(crate) struct OldCsvOutput +where + W: Write + Send, +{ + dst_zone: Tz, + writer: W, +} + +impl OldCsvOutput +where + W: Write + Send, +{ + pub fn new(writer: W, dst_zone: Tz) -> Self { + Self { dst_zone, writer } + } +} + +impl Mactime2Writer for OldCsvOutput +where + W: Write + Send, +{ + fn write_line(&mut self, timestamp: &i64, entry: &ListEntry) -> std::io::Result<()> { + let timestamp = ForensicsTimestamp::from(*timestamp).with_timezone(self.dst_zone); + write!( + self.writer, + "{},{},{},{},{},{},{},\"{}\"", + timestamp, + entry.line.get_size(), + entry.flags, + entry.line.get_mode_as_string(), + entry.line.get_uid(), + entry.line.get_gid(), + entry.line.get_inode(), + entry.line.get_name() + ) + } + + fn into_writer(self) -> W { + self.writer + } +} + +#[cfg(test)] +mod tests { + use crate::bodyfile::ListEntry; + use crate::bodyfile::MACBFlags; + use crate::bodyfile::Mactime2Writer; + + use super::OldCsvOutput; + use chrono::DateTime; + use chrono_tz::Tz; + use chrono_tz::TZ_VARIANTS; + use dfir_toolkit::common::bodyfile::Bodyfile3Line; + use std::io::Cursor; + use std::io::{BufRead, BufReader}; + use std::sync::Arc; + + fn random_tz() -> Tz { + let index = rand::random::() % TZ_VARIANTS.len(); + TZ_VARIANTS[index] + } + + #[allow(non_snake_case)] + #[test] + fn test_correct_ts_UTC() { + for _ in 1..10 { + let unix_ts = rand::random::() as i64; + let bf_line = Bodyfile3Line::new().with_crtime(unix_ts.into()); + let entry = ListEntry { + flags: MACBFlags::B, + line: Arc::new(bf_line), + }; + + let mut output = OldCsvOutput::new(Cursor::new(vec![]), Tz::UTC); + + output.write_line(&unix_ts, &entry).unwrap(); + let mut output = BufReader::new(Cursor::new(output.into_writer().into_inner())).lines(); + let out_line = output.next().unwrap().unwrap(); + + let out_ts = out_line.split(',').next().unwrap(); + let rfc3339 = DateTime::parse_from_rfc3339(out_ts) + .expect(out_ts) + .timestamp(); + assert_eq!( + unix_ts, rfc3339, + "Timestamp {unix_ts} converted to '{out_ts}' and back to {rfc3339}", + ); + } + } + + #[allow(non_snake_case)] + #[test] + fn test_correct_ts_random_tz() -> Result<(), String> { + for _ in 1..100 { + let tz = random_tz(); + let mut output = OldCsvOutput::new(Cursor::new(vec![]), tz); + + let unix_ts = rand::random::() as i64; + let bf_line = Bodyfile3Line::new().with_crtime(unix_ts.into()); + let entry = ListEntry { + flags: MACBFlags::B, + line: Arc::new(bf_line), + }; + + output.write_line(&unix_ts, &entry).unwrap(); + let mut output = BufReader::new(Cursor::new(output.into_writer().into_inner())).lines(); + let out_line = output.next().unwrap().unwrap(); + + let out_ts = out_line.split(',').next().unwrap(); + let rfc3339 = match DateTime::parse_from_rfc3339(out_ts) { + Ok(ts) => ts, + Err(e) => return Err(format!("error while parsing '{}': {}", out_ts, e)), + }; + let calculated_ts = rfc3339.timestamp(); + assert_eq!( + unix_ts, calculated_ts, + "Timestamp {unix_ts} converted to '{out_ts}' and back to {calculated_ts}", + ); + } + Ok(()) + } +} From 6736b0fcc4eb42adf0b98400c6850dd1f5f206b0 Mon Sep 17 00:00:00 2001 From: Jan Starke Date: Wed, 12 Jun 2024 23:28:14 +0200 Subject: [PATCH 4/6] use correct newlines with old csv format --- src/bin/mactime2/output/old_csv_output.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/mactime2/output/old_csv_output.rs b/src/bin/mactime2/output/old_csv_output.rs index ae8d1ec..a2be1aa 100644 --- a/src/bin/mactime2/output/old_csv_output.rs +++ b/src/bin/mactime2/output/old_csv_output.rs @@ -28,7 +28,7 @@ where { fn write_line(&mut self, timestamp: &i64, entry: &ListEntry) -> std::io::Result<()> { let timestamp = ForensicsTimestamp::from(*timestamp).with_timezone(self.dst_zone); - write!( + writeln!( self.writer, "{},{},{},{},{},{},{},\"{}\"", timestamp, From 9e706562db1c6df1cb49f0cce314c0f8a5b12c7d Mon Sep 17 00:00:00 2001 From: Jan Starke Date: Thu, 13 Jun 2024 11:16:29 +0200 Subject: [PATCH 5/6] use format for old-csv --- src/bin/mactime2/application.rs | 23 +++++++++++++++++------ src/bin/mactime2/cli.rs | 9 --------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/bin/mactime2/application.rs b/src/bin/mactime2/application.rs index 2a0f88c..e2ffb8d 100644 --- a/src/bin/mactime2/application.rs +++ b/src/bin/mactime2/application.rs @@ -23,24 +23,35 @@ enum InputFormat { } #[derive(ValueEnum, Clone, Display)] -pub (crate) enum OutputFormat { +pub(crate) enum OutputFormat { + /// Comma-Separated Values compliant to RFC 4180 #[strum(serialize = "csv")] Csv, + /// legacy text format, inherited from the old mactime #[strum(serialize = "txt")] Txt, + /// Javascript Object Notation #[strum(serialize = "json")] Json, + /// JSON-format to be used by elasticsearch #[cfg(feature = "elastic")] #[strum(serialize = "elastic")] Elastic, + + /// Use the old CSV format that was used by legacy mactime. + /// Keep in mind that in this format, fields which contain commas will + /// not be wrapped by quotes, as RFC4180 requires it. So, this format + /// is not RFC4180-compliant, which means that you might not be able + /// to use the output together with csv processing tools. + #[strum(serialize = "old-csv")] + OldCsv, } pub struct Mactime2Application { format: OutputFormat, - old_csv: bool, bodyfile: Input, dst_zone: Tz, strict_mode: bool, @@ -62,10 +73,11 @@ impl Mactime2Application { BodyfileSorter::default().with_receiver(decoder.get_receiver(), options); sorter = sorter.with_output(match self.format { - OutputFormat::Csv => if self.old_csv { + OutputFormat::OldCsv => { Box::new(OldCsvOutput::new(std::io::stdout(), self.dst_zone)) - } else { - Box::new(CsvOutput::new(std::io::stdout(), self.dst_zone))} + } + + OutputFormat::Csv => Box::new(CsvOutput::new(std::io::stdout(), self.dst_zone)), OutputFormat::Txt => Box::new(TxtOutput::new(std::io::stdout(), self.dst_zone)), _ => panic!("invalid execution path"), }); @@ -107,7 +119,6 @@ impl From for Mactime2Application { Self { format, - old_csv: cli.old_csv, bodyfile: cli.input_file, dst_zone: cli.dst_zone.into_tz().unwrap(), strict_mode: cli.strict_mode, diff --git a/src/bin/mactime2/cli.rs b/src/bin/mactime2/cli.rs index 8c703ff..faab3df 100644 --- a/src/bin/mactime2/cli.rs +++ b/src/bin/mactime2/cli.rs @@ -42,15 +42,6 @@ pub struct Cli { #[arg(group="csv")] pub(crate) csv_format: bool, - /// use the old CSV format that was used by legacy mactime. - /// - /// Keep in mind that in this format, fields which contain commas will - /// not be wrapped by quotes, as RFC4180 requires it. So, this format - /// is not RFC4180-compliant, which means that you might not be able - /// to use the output together with csv processing tools. - #[clap(long("old-csv"), display_order(615))] - pub(crate) old_csv: bool, - /// output as JSON instead of TXT. This is a convenience option, which is identical to `--format=json` /// and will be removed in a future release. If you specified `--format` and `-j`, the latter will be ignored. #[clap(short('j'), display_order(620))] From 7829d978e6ea94817bdf72c56aa72425a37deb68 Mon Sep 17 00:00:00 2001 From: Jan Starke Date: Thu, 13 Jun 2024 11:25:02 +0200 Subject: [PATCH 6/6] use shorter comment for old-csv --- src/bin/mactime2/application.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/bin/mactime2/application.rs b/src/bin/mactime2/application.rs index e2ffb8d..8fb6cb2 100644 --- a/src/bin/mactime2/application.rs +++ b/src/bin/mactime2/application.rs @@ -41,11 +41,7 @@ pub(crate) enum OutputFormat { #[strum(serialize = "elastic")] Elastic, - /// Use the old CSV format that was used by legacy mactime. - /// Keep in mind that in this format, fields which contain commas will - /// not be wrapped by quotes, as RFC4180 requires it. So, this format - /// is not RFC4180-compliant, which means that you might not be able - /// to use the output together with csv processing tools. + /// Use the old (non RFC compliant) CSV format that was used by legacy mactime. #[strum(serialize = "old-csv")] OldCsv, }