feat: the convert command converts trace files to sqlite database

Cargo.toml

@@ -53,6 +53,7 @@ tui-textarea = { version = "0.4", optional = true }
 ellipse = "0.2.0"
 ark-ff = "0.4.2"
 memmap2 = "0.9.0"
+rusqlite = "0.30.0"
 
 [target.'cfg(all(target_arch = "x86_64", target_feature = "avx"))'.dependencies]
 simd-json = "0.13"

shell.nix

@@ -3,7 +3,7 @@
 pkgs.mkShell {
   buildInputs = [
     pkgs.cargo pkgs.rust-analyzer pkgs.rustc pkgs.rustfmt pkgs.clippy
-    pkgs.libiconv
+    pkgs.libiconv pkgs.sqlite pkgs.sqlite-interactive
     pkgs.git-cliff
   ];
 

src/exporters/convert.rs

@@ -0,0 +1,68 @@
+use crate::{
+    compiler::ConstraintSet,
+    pretty::{Base, Pretty},
+    utils::purify,
+};
+use anyhow::*;
+use itertools::Itertools;
+use rusqlite::Connection;
+
+pub(crate) fn to_sqlite(cs: &ConstraintSet, exclude: &[String], filename: &str) -> Result<()> {
+    let db = Connection::open(filename)?;
+
+    for module in cs.columns.modules() {
+        if exclude.contains(&module) {
+            continue;
+        }
+        println!("Exporting {}", &module);
+        let column_names = cs
+            .columns
+            .iter_module(&module)
+            .map(|c| cs.handle(&c.0))
+            .collect::<Vec<_>>();
+        let sql_column_headers = column_names
+            .iter()
+            .map(|h| format!("{}_ TEXT NOT NULL", h.name))
+            .join(", ");
+        let sql_column_names = column_names
+            .iter()
+            .map(|h| format!("{}_ ", h.name))
+            .join(", ");
+        db.execute(&format!("DROP TABLE IF EXISTS {}_", purify(&module)), ())?;
+        db.execute(
+            &format!("CREATE TABLE {}_ ({})", purify(&module), sql_column_headers),
+            (),
+        )?;
+        let max_i = cs.iter_len(&module);
+        if max_i == 0 {
+            continue;
+        }
+
+        for i in 0..max_i {
+            let vals = cs
+                .columns
+                .iter_module(&module)
+                .map(|col| {
+                    format!(
+                        "\"{}\"",
+                        cs.columns
+                            .get(&col.0, i.try_into().unwrap(), false)
+                            .unwrap_or_default()
+                            .pretty_with_base(Base::Hex)
+                    )
+                })
+                .join(", ");
+            db.execute(
+                &format!(
+                    "INSERT INTO {}_ ({}) VALUES ({})",
+                    purify(&module),
+                    &sql_column_names,
+                    &vals
+                ),
+                (),
+            )?;
+        }
+    }
+
+    Ok(())
+}

src/exporters/mod.rs

@@ -4,6 +4,7 @@ use log::*;
 pub mod besu;
 #[cfg(feature = "conflater")]
 pub mod conflater;
+pub mod convert;
 pub(crate) mod debugger;
 #[cfg(feature = "exporters")]
 pub mod latex;

src/main.rs

@@ -130,6 +130,29 @@ enum Commands {
         constraints_filename: Option<String>,
     },
     /// Given a set of constraints and a trace file, fill the computed columns
+    Convert {
+        #[arg(
+            short = 'T',
+            long = "trace",
+            required = true,
+            help = "the trace to convert"
+        )]
+        tracefile: String,
+
+        #[arg(
+            long = "exclude",
+            help = "do not export these modules",
+            value_delimiter = ','
+        )]
+        exclude: Option<Vec<String>>,
+
+        #[arg(short = 'o', long = "out", help = "where to write the computed trace")]
+        outfile: Option<String>,
+
+        #[arg(short='f', long="format", value_parser=["sorts", "nhood"], value_delimiter=',', global=true)]
+        auto_constraints: Vec<String>,
+    },
+    /// Given a set of constraints and a trace file, fill the computed columns
     Compute {
         #[arg(
             short = 'T',
@@ -662,6 +685,28 @@ fn main() -> Result<()> {
                 constraints_filename,
             )?;
         }
+        Commands::Convert {
+            tracefile,
+            outfile,
+            auto_constraints,
+            exclude,
+        } => {
+            // if auto_constraints {
+            //     builder.auto_constraints(AutoConstraint::all());
+            // }
+
+            let mut cs = builder.into_constraint_set()?;
+            compute::compute_trace(&tracefile, &mut cs, false)
+                .with_context(|| format!("while computing from `{}`", tracefile))?;
+            exporters::convert::to_sqlite(
+                &cs,
+                &exclude.unwrap_or_default(),
+                outfile
+                    .as_ref()
+                    .map(String::as_str)
+                    .unwrap_or("trace.sqlite"),
+            )?;
+        }
         Commands::Compute {
             tracefile,
             outfile,