Wav compressor optimizer integration

brro-compressor/Cargo.toml

@@ -13,4 +13,8 @@ log = "0.4.0"
 clap = {version = "4.3.14", features = ["derive"] }
 bincode = "2.0.0-rc.3"
 rustfft = "6.1.0"
-tempfile = "3.2"
+tempfile = "3.2"
+
+regex = "1.9.1"
+hound = "3.5"
+median = "0.3.2"

brro-compressor/src/lib.rs

@@ -9,3 +9,6 @@ pub mod preprocessor;
 pub mod utils;
 mod header;
 mod data;
+
+pub mod optimizer;
+pub mod types;

brro-compressor/src/main.rs

@@ -1,26 +1,69 @@
-use log::{info, debug};
+use std::{path::Path};
 use clap::{Parser, command, arg};
+use log::{debug, error, info};
+use brro_compressor::compressor;
+
+use brro_compressor::optimizer::optimizer;
+use brro_compressor::utils::reader;
+use brro_compressor::utils::writer;
+
+fn process_args(input_path: &str, arguments: &Args) {
+    let path = Path::new(input_path);
+
+    let new_name = format!("{}-compressed", path.file_name().unwrap().to_string_lossy());
+    let base_dir = path.with_file_name(new_name);
+
+    writer::initialize_directory(&base_dir).expect("Failed to initialize directory");
+
+    if arguments.directory {
+        let (file_contents, file_names) = reader::stream_reader(path).expect("TODO: panic message");
+        for (index, data) in file_contents.iter().enumerate() {
+            let (vec_data, tag) = data;
+            let optimizer_results = optimizer::process_data(vec_data, tag);
+
+            let optimizer_results_f: Vec<f64> = optimizer_results.iter().map(|&x| x as f64).collect();
+
+            let mut compressed: Vec<u8> = Vec::new();
+            if arguments.noop {
+                compressed = compressor::noop::noop(&optimizer_results_f);
+            } else if arguments.constant {
+                compressed = compressor::constant::constant(&optimizer_results_f);
+            }
+
+            let file_name =  writer::replace_extension(&file_names[index], "txt)");
+            let new_path = base_dir.join(&file_name);
+            let mut file = writer::create_streaming_writer(&new_path).expect("TODO: panic message");
+            writer::write_data_to_stream(&mut file, &compressed).expect("Failed to write compressed data");
+        }
+    } else {
+        // process_file(input_path.into());
+    }
+}
 
 #[derive(Parser, Default, Debug)]
 #[command(author, version, about, long_about = None)]
 struct Args {
     /// input file
     input: String,
 
-    /// Set this file to process a whole directory
     #[arg(short, action)]
     directory: bool,
 
+    /// Write optimized samples to a file, named as optimized.out
+    #[arg(long, action)]
+    noop: bool,
+
+    /// Write optimized samples to a file, named as optimized.out
+    #[arg(long, action)]
+    constant: bool,
+
 }
 
 fn main() {
+    // How to break the float part??? --> THERE ARE NO FLOATS!
+    // https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/s2-proc-stat
     env_logger::init();
     let arguments = Args::parse();
     debug!("{:?}", arguments);
-    info!("Start!");
-    // The baseline work should be: streaming read -> compress -> streaming write
-    // 1st Stage: Create a streaming reader
-    // 2nd Stage: Create a streaming writer
-    // The system should be able to read a stream and write it within the defined architecture
-
+    process_args(&arguments.input, &arguments);
 }

brro-compressor/src/optimizer/mod.rs

@@ -0,0 +1 @@
+pub mod optimizer;

brro-compressor/src/optimizer/optimizer.rs

@@ -0,0 +1,68 @@
+// Lucas - Once the project is far enough along I strongly reccomend reenabling dead code checks
+#![allow(dead_code)]
+
+use median::Filter;
+use log::{debug, error, info};
+use types::metric_tag::MetricTag;
+use crate::types;
+
+impl MetricTag {
+    #[allow(clippy::wrong_self_convention)]
+    fn from_float(&self, x: f64) -> i64 {
+        match self {
+            MetricTag::Other => {
+                0
+            }
+            MetricTag::NotFloat | MetricTag::QuasiRandom => {
+                x as i64
+            }
+            MetricTag::Percent(y) => {
+                to_multiply_and_truncate(x, *y)
+            }
+            MetricTag::Duration(y) => {
+                to_multiply_and_truncate(x, *y)
+            }
+            MetricTag::Bytes(y) => {
+                (x as i64) / (*y as i64)
+            }
+        }
+    }
+}
+
+/// Converts a float via multiplication and truncation
+fn to_multiply_and_truncate(number: f64, mul: i32) -> i64 {
+    (number * mul as f64) as i64
+}
+
+fn to_median_filter(data: &Vec<f64>) -> Vec<i64> {
+    let mut filtered = Vec::with_capacity(data.len());
+    // 10minutes of data
+    let mut filter = Filter::new(50);
+    for point in data {
+        let point_int = MetricTag::QuasiRandom.from_float(*point);
+        let median = filter.consume(point_int);
+        filtered.push(median)
+    }
+    filtered
+}
+
+
+pub fn process_data(wav_data: &Vec<f64>, tag: &MetricTag) -> Vec<i64> {
+    let mut _bitdepth = 64;
+    let mut _dc_component: i64 = 0;
+    let mut _fractional = true;
+
+    debug!("Tag: {:?}", tag);
+    let data = match tag {
+        MetricTag::Other => Vec::new(),
+        MetricTag::QuasiRandom => to_median_filter(&wav_data),
+        _ => {
+            wav_data
+                .iter()
+                .map(|x| tag.from_float(*x))
+                .collect()
+        }
+    };
+    _fractional = false;
+    return data;
+}

brro-compressor/src/types/metric_tag.rs

@@ -0,0 +1,14 @@
+#[derive(Debug)]
+pub enum MetricTag {
+    Percent(i32),
+    // If it is a percent reduce significant digits to 2
+    Duration(i32),
+    // if it is a duration reduce precision to 1 microsecond
+    NotFloat,
+    // A metric that has a float representation but shouldn't (Eg. Precision is not needed)
+    QuasiRandom,
+    // A metric that exhibits a quasi random sample behavior. (E.g. Network deltas, heap memory)
+    Bytes(i32),
+    // Data that is in bytes... Make it MB, or KB
+    Other, // Everything else
+}

brro-compressor/src/types/mod.rs

@@ -0,0 +1 @@
+pub mod metric_tag;

brro-compressor/src/utils/reader.rs

@@ -1,7 +1,11 @@
 // Implement a streaming reader here
 use std::fs;
 use std::io::{self, Read};
-use std::path::{Path};
+use std::path::{Path, PathBuf};
+use log::debug;
+use regex::Regex;
+use types::metric_tag::MetricTag;
+use crate::types;
 
 // Function to check if a file is a WAV file
 fn is_wav_file(file_path: &Path) -> io::Result<bool> {
@@ -15,10 +19,13 @@ fn is_wav_file(file_path: &Path) -> io::Result<bool> {
 }
 
 // Function to process a WAV file
-fn process_wav_file(file_path: &Path) -> io::Result<()> {
-    // Handle WAV file processing here (for example, analyzing or converting)
-    println!("Processing WAV file: {:?}", file_path);
-    Ok(())
+fn process_wav_file(file_path: &Path) -> io::Result<(Vec<f64>, MetricTag)> {
+    let full_path_str = file_path.to_str().unwrap_or("");
+    debug!("File: {} ,", full_path_str);
+    let wav_data = read_metrics_from_wav(full_path_str);
+    // Depending on Metric Tag, apply a transformation
+    let tag = tag_metric(full_path_str);
+    return Ok((wav_data, tag));
 }
 
 // Function to process a RAW file
@@ -29,7 +36,12 @@ fn process_raw_file(file_path: &Path) -> io::Result<()> {
 }
 
 // Function to read and process files in a directory
-fn stream_reader(directory_path: &Path) -> io::Result<()> {
+pub fn stream_reader(directory_path: &Path) -> io::Result<(Vec<(Vec<f64>, MetricTag)>, Vec<String>)> {
+    let mut results: Vec<(Vec<f64>, MetricTag)> = Vec::new();
+
+    let mut filenames: Vec<String> = Vec::new();
+
+
     // Iterate through entries (files and subdirectories) in the given directory
     for entry in fs::read_dir(directory_path)? {
         // Unwrap the entry to get the actual entry information
@@ -38,16 +50,91 @@ fn stream_reader(directory_path: &Path) -> io::Result<()> {
         // Get the path of the entry (file or directory)
         let file_path = entry.path();
 
+        // Add the filename to the list
+        if let Some(filename) = file_path.file_name() {
+            if let Some(filename_str) = filename.to_str() {
+                filenames.push(filename_str.to_string());
+            }
+        }
+
         // Check if the file is a WAV file
         if is_wav_file(&file_path)? {
             // If it's a WAV file, process it using the process_wav_file function
-            process_wav_file(&file_path)?;
+            let wav_result = process_wav_file(&file_path)?;
+            results.push(wav_result);
         } else {
             // If it's not a WAV file, process it as a RAW file using the process_raw_file function
             process_raw_file(&file_path)?;
         }
     }
-    Ok(())
+    Ok((results, filenames))
+}
+
+/*
+Reads a WAV file, checks the channels and the information contained there. From that
+information takes a decision on the best channel, block size and bitrate for the BRRO
+encoders.
+*/
+fn tag_metric(filename: &str) -> MetricTag {
+    // Should sort this by the probability of each tag, so the ones that are more common are dealt first
+    // If it says percent_ or _utilization
+    let mut regex = Regex::new(r"(?m)percent_|_utilization").unwrap();
+    if regex.captures(filename).is_some() {
+        // 2 significant digits resolution (Linux resolution)
+        return MetricTag::Percent(100);
+    }
+    // if it says _client_request
+    regex = Regex::new(r"(?m)_client_request").unwrap();
+    if regex.captures(filename).is_some() {
+        // Fractional requests are nothing but an averaging artifact
+        return MetricTag::NotFloat;
+    }
+    // if it says _seconds
+    regex = Regex::new(r"(?m)_seconds").unwrap();
+    if regex.captures(filename).is_some() {
+        // 1 micro second resolution
+        return MetricTag::Duration(1_000_000);
+    }
+    // if it says _seconds
+    regex = Regex::new(r"(?m)_networkindelta|_networkoutdelta|_heapmemoryused_").unwrap();
+    if regex.captures(filename).is_some() {
+        return MetricTag::QuasiRandom;
+    }
+    MetricTag::Other
+}
+
+fn read_metrics_from_wav(filename: &str) -> Vec<f64> {
+    let r_reader = hound::WavReader::open(filename);
+    let mut reader = match r_reader {
+        Ok(reader) => reader,
+        Err(_err) => { return Vec::new(); }
+    };
+    let num_channels = reader.spec().channels as usize;
+
+    let mut raw_data: Vec<f64> = Vec::new();
+    let mut u64_holder: [u16; 4] = [0, 0, 0, 0];
+
+    // Iterate over the samples and channels and push each sample to the vector
+    let mut current_channel: usize = 0;
+    for sample in reader.samples::<i16>() {
+        u64_holder[current_channel] = sample.unwrap() as u16;
+        current_channel += 1;
+        if current_channel == num_channels {
+            raw_data.push(join_u16_into_f64(u64_holder));
+            current_channel = 0;
+        }
+    }
+    raw_data
+}
+
+fn join_u16_into_f64(bits: [u16; 4]) -> f64 {
+    let u64_bits = (bits[0] as u64) |
+        ((bits[1] as u64) << 16) |
+        ((bits[2] as u64) << 32) |
+        ((bits[3] as u64) << 48);
+
+
+    f64::from_bits(u64_bits)
 }
 
 // Import the testing module

brro-compressor/src/utils/writer.rs

@@ -3,17 +3,35 @@ use std::io::{self, Write};
 use std::path::Path;
 
 // Function to create a streaming writer for a file
-fn create_streaming_writer(file_path: &Path) -> io::Result<File> {
+pub fn create_streaming_writer(file_path: &Path) -> io::Result<File> {
     // Open the file for writing, creating it if it doesn't exist
     File::create(file_path)
 }
 
 // Function to write data to a streaming writer
-fn write_data_to_stream(writer: &mut File, data: &[u8]) -> io::Result<()> {
-    writer.write_all(data)?;
+pub fn write_data_to_stream(writer: &mut File, data: &[u8]) -> io::Result<()> {
+    // Convert each byte to a string representation and collect them into a Vec<String>
+    let strings: Vec<String> = data.iter().map(|&byte| byte.to_string()).collect();
+
+    // Join the Vec<String> into a single String, separated by spaces (or any other delimiter you prefer)
+    let output = strings.join(" ");
+
+    writer.write_all(output.as_bytes())?;
     Ok(())
 }
-
+pub fn initialize_directory(base_dir: &Path) -> io::Result<()> {
+    if !base_dir.exists() {
+        fs::create_dir_all(base_dir)?;
+    }
+    Ok(())
+}
+pub fn replace_extension(filename: &String, new_extension: &str) -> String {
+    let path = Path::new(&filename);
+    let without_extension = path.file_stem().unwrap_or_default(); // gets the filename without extension
+    let mut new_path = PathBuf::from(without_extension);
+    new_path.set_extension(new_extension);
+    new_path.to_string_lossy().into_owned()
+}
 #[cfg(test)]
 mod tests {
     use super::*;