From 21fc1f45e33ee0ae86cd1c06d6e0654897a5561b Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 17:41:04 -0500
Subject: [PATCH 01/24] updated logging

---
 bin/utils/seed_mpc_db.rs | 69 +++++++++++++++++++++-------------------
 1 file changed, 36 insertions(+), 33 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 3243089..da510a6 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,4 +1,7 @@
 use clap::Args;
+use eyre::Error;
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
 use indicatif::ProgressBar;
 use mpc::config::DbConfig;
 use mpc::db::Db;
@@ -28,19 +31,17 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
     let mut templates: Vec<Template> = Vec::with_capacity(args.num_templates);
 
     tracing::info!("Generating templates");
-    let pb = ProgressBar::new(args.num_templates as u64)
-        .with_message("Generating templates");
-
     let mut rng = thread_rng();
 
     for _ in 0..args.num_templates {
+        tracing::info!(
+            "Generating template {}/{}",
+            templates.len(),
+            args.num_templates
+        );
         templates.push(rng.gen());
-
-        pb.inc(1);
     }
 
-    pb.finish_with_message("done");
-
     let coordinator_db = Db::new(&DbConfig {
         url: args.coordinator_db_url.clone(),
         migrate: true,
@@ -61,25 +62,17 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         );
     }
 
-    tracing::info!("Seeding databases");
-    let pb =
-        ProgressBar::new(args.num_templates as u64).with_message("Seeding DBs");
+    let num_chunks = (templates.len() / args.batch_size) + 1;
 
-    for (idx, chunk) in templates.chunks(args.batch_size).enumerate() {
-        tracing::info!(
-            "Seeding chunk {}/{}",
-            idx + 1,
-            (templates.len() / args.batch_size) + 1
-        );
+    //TODO: update logging to num shares rather than chunks
 
+    for (idx, chunk) in templates.chunks(args.batch_size).enumerate() {
         let mut chunk_masks = Vec::with_capacity(chunk.len());
         let mut chunk_shares: Vec<_> = (0..participant_dbs.len())
             .map(|_| Vec::with_capacity(chunk.len()))
             .collect();
 
-        tracing::info!("Encoding shares");
-        let pb = ProgressBar::new(chunk.len() as u64)
-            .with_message("Encoding shares");
+        tracing::info!("Encoding shares {}/{}", idx + 1, num_chunks);
         for (offset, template) in chunk.iter().enumerate() {
             let shares =
                 mpc::distance::encode(template).share(participant_dbs.len());
@@ -90,31 +83,41 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
             for (idx, share) in shares.iter().enumerate() {
                 chunk_shares[idx].push((id as u64, *share));
             }
+        }
+
+        let mut tasks = FuturesUnordered::new();
 
-            pb.inc(1);
+        for (i, db) in participant_dbs.iter().enumerate() {
+            tracing::info!(
+                "Inserting shares chunk {}/{num_chunks} into participant DB {i}",
+                idx + 1
+            );
+
+            tasks.push(Box::pin(db.insert_shares(&chunk_shares[idx])));
         }
 
-        let mut tasks = vec![];
+        tracing::info!(
+            "Inserting masks chunk {}/{num_chunks} into coordinator DB",
+            idx + 1
+        );
+
+        let coordinator_task: Result<(), eyre::Report> =
+            coordinator_db.insert_masks(&chunk_masks).await;
 
-        for (idx, db) in participant_dbs.iter().enumerate() {
-            tracing::info!("Inserting shares into participant DB {idx}");
+        tasks.push(coordinator_task);
 
-            tasks.push(db.insert_shares(&chunk_shares[idx]));
+        while let Some(result) = tasks.next().await {
+            println!("Task completed with result: {:?}", result);
         }
 
-        tracing::info!("Inserting masks into coordinator DB");
         let (coordinator, participants) = tokio::join!(
-            coordinator_db.insert_masks(&chunk_masks),
-            futures::future::join_all(tasks),
+            ,
+            tasks.await.collect::<Result<_, _>>(),
         );
 
-        coordinator?;
-        participants.into_iter().collect::<Result<_, _>>()?;
-
-        pb.inc(args.batch_size as u64);
+        // coordinator?;
+        // participants.into_iter().collect::<Result<_, _>>()?;
     }
 
-    pb.finish_with_message("done");
-
     Ok(())
 }

From 32f403102eb31bf40d2d3d7a1450ffe4a45c4489 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 20:45:50 -0500
Subject: [PATCH 02/24] insert all shares/masks concurrently, update logs

---
 bin/utils/seed_mpc_db.rs | 129 ++++++++++++++++++++++++++++-----------
 bin/utils/utils.rs       |   2 -
 2 files changed, 92 insertions(+), 39 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index da510a6..5776e5c 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,9 +1,13 @@
+use std::mem;
+use std::sync::Arc;
+
 use clap::Args;
 use eyre::Error;
 use futures::stream::FuturesUnordered;
-use futures::StreamExt;
+use futures::{FutureExt, StreamExt};
 use indicatif::ProgressBar;
 use mpc::config::DbConfig;
+use mpc::coordinator;
 use mpc::db::Db;
 use mpc::template::Template;
 use rand::{thread_rng, Rng};
@@ -19,7 +23,7 @@ pub struct SeedMPCDb {
     #[clap(short, long, default_value = "3000000")]
     pub num_templates: usize,
 
-    #[clap(short, long, default_value = "10000")]
+    #[clap(short, long, default_value = "3000")]
     pub batch_size: usize,
 }
 
@@ -30,11 +34,11 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
 
     let mut templates: Vec<Template> = Vec::with_capacity(args.num_templates);
 
-    tracing::info!("Generating templates");
+    println!("Generating templates");
     let mut rng = thread_rng();
 
     for _ in 0..args.num_templates {
-        tracing::info!(
+        println!(
             "Generating template {}/{}",
             templates.len(),
             args.num_templates
@@ -42,29 +46,31 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         templates.push(rng.gen());
     }
 
-    let coordinator_db = Db::new(&DbConfig {
-        url: args.coordinator_db_url.clone(),
-        migrate: true,
-        create: true,
-    })
-    .await?;
+    let coordinator_db = Arc::new(
+        Db::new(&DbConfig {
+            url: args.coordinator_db_url.clone(),
+            migrate: true,
+            create: true,
+        })
+        .await?,
+    );
 
     let mut participant_dbs = vec![];
 
     for db_config in args.participant_db_url.iter() {
-        participant_dbs.push(
+        participant_dbs.push(Arc::new(
             Db::new(&DbConfig {
                 url: db_config.clone(),
                 migrate: true,
                 create: true,
             })
             .await?,
-        );
+        ));
     }
 
-    let num_chunks = (templates.len() / args.batch_size) + 1;
-
-    //TODO: update logging to num shares rather than chunks
+    //TODO: would be nice to make this parallel
+    let mut batched_shares = vec![];
+    let mut batched_masks = vec![];
 
     for (idx, chunk) in templates.chunks(args.batch_size).enumerate() {
         let mut chunk_masks = Vec::with_capacity(chunk.len());
@@ -72,8 +78,12 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
             .map(|_| Vec::with_capacity(chunk.len()))
             .collect();
 
-        tracing::info!("Encoding shares {}/{}", idx + 1, num_chunks);
         for (offset, template) in chunk.iter().enumerate() {
+            println!(
+                "Encoding shares {}/{}",
+                idx * args.batch_size + offset,
+                templates.len()
+            );
             let shares =
                 mpc::distance::encode(template).share(participant_dbs.len());
 
@@ -85,39 +95,84 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
             }
         }
 
-        let mut tasks = FuturesUnordered::new();
+        batched_shares.push(chunk_shares);
+        batched_masks.push(chunk_masks);
+    }
 
-        for (i, db) in participant_dbs.iter().enumerate() {
-            tracing::info!(
-                "Inserting shares chunk {}/{num_chunks} into participant DB {i}",
-                idx + 1
+    let mut tasks = FuturesUnordered::new();
+
+    let mut i = 0;
+
+    //Insert into dbs
+    for (masks, mut shares) in
+        batched_masks.into_iter().zip(batched_shares.into_iter())
+    {
+        for (idx, db) in participant_dbs.iter().enumerate() {
+            let participant_shares = mem::take(&mut shares[idx]);
+            let db = db.clone();
+
+            println!(
+                "Inserting shares {} to {} into participant {} db",
+                i,
+                i + args.batch_size,
+                idx,
             );
 
-            tasks.push(Box::pin(db.insert_shares(&chunk_shares[idx])));
+            tasks.push(tokio::spawn(async move {
+                db.insert_shares(&participant_shares).await?;
+                Ok::<_, Error>(Insertion::Shares(idx))
+            }));
         }
 
-        tracing::info!(
-            "Inserting masks chunk {}/{num_chunks} into coordinator DB",
-            idx + 1
+        println!(
+            "Inserting shares {} to {} into coordinator db",
+            i,
+            i + args.batch_size,
         );
 
-        let coordinator_task: Result<(), eyre::Report> =
-            coordinator_db.insert_masks(&chunk_masks).await;
+        let coordinator_db = coordinator_db.clone();
+        tasks.push(tokio::spawn(async move {
+            coordinator_db.insert_masks(&masks).await?;
+            Ok::<_, Error>(Insertion::Masks)
+        }));
 
-        tasks.push(coordinator_task);
+        i += args.batch_size;
+    }
 
-        while let Some(result) = tasks.next().await {
-            println!("Task completed with result: {:?}", result);
-        }
+    //wait for tasks
 
-        let (coordinator, participants) = tokio::join!(
-            ,
-            tasks.await.collect::<Result<_, _>>(),
-        );
+    let mut participant_counters = vec![0_usize; participant_dbs.len()];
+    let mut coordinator_counter = 0;
 
-        // coordinator?;
-        // participants.into_iter().collect::<Result<_, _>>()?;
+    while let Some(result) = tasks.next().await {
+        match result?? {
+            Insertion::Shares(idx) => {
+                let counter = participant_counters[idx] + args.batch_size;
+                participant_counters[idx] = counter;
+
+                println!(
+                    "Successfully inserted shares {}/{} into participant {} db",
+                    counter,
+                    templates.len(),
+                    idx,
+                );
+            }
+            Insertion::Masks => {
+                coordinator_counter += args.batch_size;
+                println!(
+                    "Successfully inserted masks {}/{} into coordinator db",
+                    coordinator_counter,
+                    templates.len()
+                );
+            }
+        }
     }
 
     Ok(())
 }
+
+//Insertion type to keep track of progress for each db
+pub enum Insertion {
+    Shares(usize),
+    Masks,
+}
diff --git a/bin/utils/utils.rs b/bin/utils/utils.rs
index 9ee7423..3e9169e 100644
--- a/bin/utils/utils.rs
+++ b/bin/utils/utils.rs
@@ -27,8 +27,6 @@ enum Opt {
 async fn main() -> eyre::Result<()> {
     dotenv::dotenv().ok();
 
-    let _shutdown_tracing_provider = StdoutBattery::init();
-
     let args = Opt::parse();
 
     match args {

From deffeb13815a512a0902d128c43745c44625c3d5 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 22:27:02 -0500
Subject: [PATCH 03/24] encode shares in parallel

---
 bin/utils/seed_mpc_db.rs | 101 ++++++++++++++++++++-------------------
 1 file changed, 52 insertions(+), 49 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 5776e5c..6b4b1bf 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,4 +1,5 @@
 use std::mem;
+use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
 use clap::Args;
@@ -6,11 +7,17 @@ use eyre::Error;
 use futures::stream::FuturesUnordered;
 use futures::{FutureExt, StreamExt};
 use indicatif::ProgressBar;
+use metrics::atomics::AtomicU64;
 use mpc::config::DbConfig;
 use mpc::coordinator;
 use mpc::db::Db;
+use mpc::distance::EncodedBits;
 use mpc::template::Template;
 use rand::{thread_rng, Rng};
+use rayon::iter::{
+    IndexedParallelIterator, IntoParallelIterator, ParallelIterator,
+};
+use sqlx::Encode;
 
 #[derive(Debug, Clone, Args)]
 pub struct SeedMPCDb {
@@ -78,18 +85,34 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
             .map(|_| Vec::with_capacity(chunk.len()))
             .collect();
 
-        for (offset, template) in chunk.iter().enumerate() {
-            println!(
-                "Encoding shares {}/{}",
-                idx * args.batch_size + offset,
-                templates.len()
-            );
-            let shares =
-                mpc::distance::encode(template).share(participant_dbs.len());
+        let encoded_shares_counter =
+            AtomicU64::new((idx * args.batch_size) as u64);
 
+        let shares_chunk = chunk
+            .into_par_iter()
+            .enumerate()
+            .map(|(i, template)| {
+                println!(
+                    "Encoding template {}/{}",
+                    encoded_shares_counter.load(Ordering::Relaxed),
+                    args.num_templates
+                );
+                let shares = mpc::distance::encode(template)
+                    .share(participant_dbs.len());
+
+                encoded_shares_counter.fetch_add(1, Ordering::Relaxed);
+
+                shares
+            })
+            .collect::<Vec<Box<[EncodedBits]>>>();
+
+        for (offset, (shares, template)) in
+            shares_chunk.iter().zip(chunk).enumerate()
+        {
             let id = offset + (idx * args.batch_size);
 
             chunk_masks.push((id as u64, template.mask));
+
             for (idx, share) in shares.iter().enumerate() {
                 chunk_shares[idx].push((id as u64, *share));
             }
@@ -99,37 +122,26 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         batched_masks.push(chunk_masks);
     }
 
-    let mut tasks = FuturesUnordered::new();
-
     let mut i = 0;
 
     //Insert into dbs
+
+    //TODO: insert in chunks and wait in chunks
     for (masks, mut shares) in
         batched_masks.into_iter().zip(batched_shares.into_iter())
     {
+        let mut tasks = FuturesUnordered::new();
+
         for (idx, db) in participant_dbs.iter().enumerate() {
             let participant_shares = mem::take(&mut shares[idx]);
             let db = db.clone();
 
-            println!(
-                "Inserting shares {} to {} into participant {} db",
-                i,
-                i + args.batch_size,
-                idx,
-            );
-
             tasks.push(tokio::spawn(async move {
                 db.insert_shares(&participant_shares).await?;
                 Ok::<_, Error>(Insertion::Shares(idx))
             }));
         }
 
-        println!(
-            "Inserting shares {} to {} into coordinator db",
-            i,
-            i + args.batch_size,
-        );
-
         let coordinator_db = coordinator_db.clone();
         tasks.push(tokio::spawn(async move {
             coordinator_db.insert_masks(&masks).await?;
@@ -137,33 +149,24 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         }));
 
         i += args.batch_size;
-    }
-
-    //wait for tasks
-
-    let mut participant_counters = vec![0_usize; participant_dbs.len()];
-    let mut coordinator_counter = 0;
-
-    while let Some(result) = tasks.next().await {
-        match result?? {
-            Insertion::Shares(idx) => {
-                let counter = participant_counters[idx] + args.batch_size;
-                participant_counters[idx] = counter;
 
-                println!(
-                    "Successfully inserted shares {}/{} into participant {} db",
-                    counter,
-                    templates.len(),
-                    idx,
-                );
-            }
-            Insertion::Masks => {
-                coordinator_counter += args.batch_size;
-                println!(
-                    "Successfully inserted masks {}/{} into coordinator db",
-                    coordinator_counter,
-                    templates.len()
-                );
+        while let Some(result) = tasks.next().await {
+            match result?? {
+                Insertion::Shares(idx) => {
+                    println!(
+                        "Inserted shares {}/{} into participant {} db",
+                        i,
+                        templates.len(),
+                        idx,
+                    );
+                }
+                Insertion::Masks => {
+                    println!(
+                        "Inserted masks {}/{} into coordinator db",
+                        i,
+                        templates.len()
+                    );
+                }
             }
         }
     }

From ebc265d68230ac007c6481afaea0d7bf095303f7 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 22:32:45 -0500
Subject: [PATCH 04/24] template creation parallel

---
 bin/utils/seed_mpc_db.rs | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 6b4b1bf..607eee5 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -39,19 +39,22 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         return Err(eyre::eyre!("No participant DBs provided"));
     }
 
-    let mut templates: Vec<Template> = Vec::with_capacity(args.num_templates);
-
-    println!("Generating templates");
-    let mut rng = thread_rng();
-
-    for _ in 0..args.num_templates {
-        println!(
-            "Generating template {}/{}",
-            templates.len(),
-            args.num_templates
-        );
-        templates.push(rng.gen());
-    }
+    let template_counter = AtomicU64::new(0);
+    let templates = (0..args.num_templates)
+        .into_par_iter()
+        .map(|_| {
+            let mut rng = thread_rng();
+
+            println!(
+                "Generating template {}/{}",
+                template_counter.load(Ordering::Relaxed),
+                args.num_templates
+            );
+            let template = rng.gen();
+            template_counter.fetch_add(1, Ordering::Relaxed);
+            template
+        })
+        .collect::<Vec<Template>>();
 
     let coordinator_db = Arc::new(
         Db::new(&DbConfig {
@@ -75,7 +78,6 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         ));
     }
 
-    //TODO: would be nice to make this parallel
     let mut batched_shares = vec![];
     let mut batched_masks = vec![];
 
@@ -124,9 +126,6 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
 
     let mut i = 0;
 
-    //Insert into dbs
-
-    //TODO: insert in chunks and wait in chunks
     for (masks, mut shares) in
         batched_masks.into_iter().zip(batched_shares.into_iter())
     {

From 2e8a6b6427e7a1148faab5466e1461ddf221a14e Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 22:54:56 -0500
Subject: [PATCH 05/24] refactored logic

---
 bin/utils/seed_mpc_db.rs | 116 ++++++++++++++++++++++++++++-----------
 1 file changed, 83 insertions(+), 33 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 607eee5..be0acec 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,3 +1,4 @@
+use core::num;
 use std::mem;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
@@ -8,11 +9,12 @@ use futures::stream::FuturesUnordered;
 use futures::{FutureExt, StreamExt};
 use indicatif::ProgressBar;
 use metrics::atomics::AtomicU64;
+use mpc::bits::Bits;
 use mpc::config::DbConfig;
-use mpc::coordinator;
 use mpc::db::Db;
 use mpc::distance::EncodedBits;
 use mpc::template::Template;
+use mpc::{coordinator, template};
 use rand::{thread_rng, Rng};
 use rayon::iter::{
     IndexedParallelIterator, IntoParallelIterator, ParallelIterator,
@@ -39,23 +41,35 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         return Err(eyre::eyre!("No participant DBs provided"));
     }
 
-    let template_counter = AtomicU64::new(0);
-    let templates = (0..args.num_templates)
-        .into_par_iter()
-        .map(|_| {
-            let mut rng = thread_rng();
+    let (coordinator_db, participant_dbs) = initialize_dbs(args).await?;
 
-            println!(
-                "Generating template {}/{}",
-                template_counter.load(Ordering::Relaxed),
-                args.num_templates
-            );
-            let template = rng.gen();
-            template_counter.fetch_add(1, Ordering::Relaxed);
-            template
-        })
-        .collect::<Vec<Template>>();
+    let templates = generate_templates(args);
 
+    let (batched_masks, batched_shares) =
+        generate_shares_and_masks(args, templates);
+
+    insert_masks_and_shares(
+        batched_masks,
+        batched_shares,
+        coordinator_db,
+        participant_dbs,
+        args.batch_size,
+        args.num_templates,
+    )
+    .await?;
+
+    Ok(())
+}
+
+//Insertion type to keep track of progress for each db
+pub enum Insertion {
+    Shares(usize),
+    Masks,
+}
+
+async fn initialize_dbs(
+    args: &SeedMPCDb,
+) -> eyre::Result<(Arc<Db>, Vec<Arc<Db>>)> {
     let coordinator_db = Arc::new(
         Db::new(&DbConfig {
             url: args.coordinator_db_url.clone(),
@@ -78,12 +92,47 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         ));
     }
 
+    Ok((coordinator_db, participant_dbs))
+}
+
+fn generate_templates(args: &SeedMPCDb) -> Vec<Template> {
+    // Generate templates
+    let template_counter = AtomicU64::new(0);
+    let templates = (0..args.num_templates)
+        .into_par_iter()
+        .map(|_| {
+            let mut rng = thread_rng();
+
+            println!(
+                "Generating template {}/{}",
+                template_counter.load(Ordering::Relaxed),
+                args.num_templates
+            );
+            let template = rng.gen();
+            template_counter.fetch_add(1, Ordering::Relaxed);
+            template
+        })
+        .collect::<Vec<Template>>();
+
+    templates
+}
+
+pub type BatchedShares = Vec<Vec<Vec<(u64, EncodedBits)>>>;
+pub type BatchedMasks = Vec<Vec<(u64, Bits)>>;
+
+fn generate_shares_and_masks(
+    args: &SeedMPCDb,
+    templates: Vec<Template>,
+) -> (BatchedMasks, BatchedShares) {
+    // Generate shares and masks
     let mut batched_shares = vec![];
     let mut batched_masks = vec![];
 
+    let num_participants = args.participant_db_url.len();
+
     for (idx, chunk) in templates.chunks(args.batch_size).enumerate() {
         let mut chunk_masks = Vec::with_capacity(chunk.len());
-        let mut chunk_shares: Vec<_> = (0..participant_dbs.len())
+        let mut chunk_shares: Vec<_> = (0..num_participants)
             .map(|_| Vec::with_capacity(chunk.len()))
             .collect();
 
@@ -99,8 +148,8 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
                     encoded_shares_counter.load(Ordering::Relaxed),
                     args.num_templates
                 );
-                let shares = mpc::distance::encode(template)
-                    .share(participant_dbs.len());
+                let shares =
+                    mpc::distance::encode(template).share(num_participants);
 
                 encoded_shares_counter.fetch_add(1, Ordering::Relaxed);
 
@@ -124,8 +173,19 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         batched_masks.push(chunk_masks);
     }
 
-    let mut i = 0;
+    (batched_masks, batched_shares)
+}
 
+async fn insert_masks_and_shares(
+    batched_masks: BatchedMasks,
+    batched_shares: BatchedShares,
+    coordinator_db: Arc<Db>,
+    participant_dbs: Vec<Arc<Db>>,
+    batch_size: usize,
+    num_templates: usize,
+) -> eyre::Result<()> {
+    // Commit shares and masks to db
+    let mut i = 0;
     for (masks, mut shares) in
         batched_masks.into_iter().zip(batched_shares.into_iter())
     {
@@ -147,34 +207,24 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
             Ok::<_, Error>(Insertion::Masks)
         }));
 
-        i += args.batch_size;
+        i += batch_size;
 
         while let Some(result) = tasks.next().await {
             match result?? {
                 Insertion::Shares(idx) => {
                     println!(
                         "Inserted shares {}/{} into participant {} db",
-                        i,
-                        templates.len(),
-                        idx,
+                        i, num_templates, idx,
                     );
                 }
                 Insertion::Masks => {
                     println!(
                         "Inserted masks {}/{} into coordinator db",
-                        i,
-                        templates.len()
+                        i, num_templates
                     );
                 }
             }
         }
     }
-
     Ok(())
 }
-
-//Insertion type to keep track of progress for each db
-pub enum Insertion {
-    Shares(usize),
-    Masks,
-}

From 75554242d980624395460f25e5debe6dcedb2533 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 23:06:56 -0500
Subject: [PATCH 06/24] separated coordinator and participant insertion

---
 bin/utils/seed_mpc_db.rs | 92 +++++++++++++++++++++++++---------------
 1 file changed, 58 insertions(+), 34 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index be0acec..0a44f88 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -61,12 +61,6 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
     Ok(())
 }
 
-//Insertion type to keep track of progress for each db
-pub enum Insertion {
-    Shares(usize),
-    Masks,
-}
-
 async fn initialize_dbs(
     args: &SeedMPCDb,
 ) -> eyre::Result<(Arc<Db>, Vec<Arc<Db>>)> {
@@ -185,45 +179,75 @@ async fn insert_masks_and_shares(
     num_templates: usize,
 ) -> eyre::Result<()> {
     // Commit shares and masks to db
-    let mut i = 0;
-    for (masks, mut shares) in
-        batched_masks.into_iter().zip(batched_shares.into_iter())
-    {
-        let mut tasks = FuturesUnordered::new();
 
+    let mut tasks = FuturesUnordered::new();
+
+    tasks.push(tokio::spawn(insert_masks(
+        batched_masks,
+        coordinator_db,
+        batch_size,
+        num_templates,
+    )));
+
+    tasks.push(tokio::spawn(insert_shares(
+        batched_shares,
+        participant_dbs,
+        batch_size,
+        num_templates,
+    )));
+
+    while let Some(result) = tasks.next().await {
+        result??;
+    }
+
+    Ok(())
+}
+
+async fn insert_masks(
+    batched_masks: BatchedMasks,
+    coordinator_db: Arc<Db>,
+    batch_size: usize,
+    num_templates: usize,
+) -> eyre::Result<()> {
+    for (i, masks) in batched_masks.iter().enumerate() {
+        coordinator_db.insert_masks(&masks).await?;
+
+        println!(
+            "Inserted masks {}/{} into coordinator db",
+            (i + 1) * batch_size,
+            num_templates
+        );
+    }
+    Ok(())
+}
+
+async fn insert_shares(
+    batched_shares: BatchedShares,
+    participant_dbs: Vec<Arc<Db>>,
+    batch_size: usize,
+    num_templates: usize,
+) -> eyre::Result<()> {
+    let mut tasks = FuturesUnordered::new();
+
+    for (i, mut shares) in batched_shares.into_iter().enumerate() {
         for (idx, db) in participant_dbs.iter().enumerate() {
             let participant_shares = mem::take(&mut shares[idx]);
             let db = db.clone();
 
             tasks.push(tokio::spawn(async move {
                 db.insert_shares(&participant_shares).await?;
-                Ok::<_, Error>(Insertion::Shares(idx))
+                Ok::<_, Error>(idx)
             }));
         }
 
-        let coordinator_db = coordinator_db.clone();
-        tasks.push(tokio::spawn(async move {
-            coordinator_db.insert_masks(&masks).await?;
-            Ok::<_, Error>(Insertion::Masks)
-        }));
-
-        i += batch_size;
-
         while let Some(result) = tasks.next().await {
-            match result?? {
-                Insertion::Shares(idx) => {
-                    println!(
-                        "Inserted shares {}/{} into participant {} db",
-                        i, num_templates, idx,
-                    );
-                }
-                Insertion::Masks => {
-                    println!(
-                        "Inserted masks {}/{} into coordinator db",
-                        i, num_templates
-                    );
-                }
-            }
+            let idx = result??;
+            println!(
+                "Inserted shares {}/{} into participant {} db",
+                (i + 1) * batch_size,
+                num_templates,
+                idx,
+            );
         }
     }
     Ok(())

From 9ead0dc401dd4d2151ee20b1b19f7703910b2127 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 23:18:13 -0500
Subject: [PATCH 07/24] added time elapsed

---
 bin/utils/seed_mpc_db.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 5776e5c..3c17442 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -28,6 +28,8 @@ pub struct SeedMPCDb {
 }
 
 pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
+    let now = std::time::Instant::now();
+
     if args.participant_db_url.is_empty() {
         return Err(eyre::eyre!("No participant DBs provided"));
     }
@@ -168,6 +170,8 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         }
     }
 
+    println!("Time elapsed {:?}", now.elapsed());
+
     Ok(())
 }
 

From 8cb87ffb01fdc6496c9e86b70a496fbc3f6445d8 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 23:19:34 -0500
Subject: [PATCH 08/24] added time elapsed

---
 bin/utils/seed_mpc_db.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 0a44f88..55c9896 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -37,6 +37,8 @@ pub struct SeedMPCDb {
 }
 
 pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
+    let now = std::time::Instant::now();
+
     if args.participant_db_url.is_empty() {
         return Err(eyre::eyre!("No participant DBs provided"));
     }
@@ -58,6 +60,8 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
     )
     .await?;
 
+    println!("Time elapsed: {:?}", now.elapsed());
+
     Ok(())
 }
 

From 34f42c2c30dbaaefc3ed91a6280b5e739554c624 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 23:25:30 -0500
Subject: [PATCH 09/24] updated insertion logic

---
 bin/utils/seed_mpc_db.rs | 43 +++++++++++++++++++++++++---------------
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 55c9896..a6e6226 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -6,20 +6,17 @@ use std::sync::Arc;
 use clap::Args;
 use eyre::Error;
 use futures::stream::FuturesUnordered;
-use futures::{FutureExt, StreamExt};
-use indicatif::ProgressBar;
+use futures::StreamExt;
 use metrics::atomics::AtomicU64;
 use mpc::bits::Bits;
 use mpc::config::DbConfig;
 use mpc::db::Db;
 use mpc::distance::EncodedBits;
 use mpc::template::Template;
-use mpc::{coordinator, template};
 use rand::{thread_rng, Rng};
 use rayon::iter::{
     IndexedParallelIterator, IntoParallelIterator, ParallelIterator,
 };
-use sqlx::Encode;
 
 #[derive(Debug, Clone, Args)]
 pub struct SeedMPCDb {
@@ -50,6 +47,7 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
     let (batched_masks, batched_shares) =
         generate_shares_and_masks(args, templates);
 
+    println!("Inserting masks and shares into db");
     insert_masks_and_shares(
         batched_masks,
         batched_shares,
@@ -213,15 +211,25 @@ async fn insert_masks(
     batch_size: usize,
     num_templates: usize,
 ) -> eyre::Result<()> {
-    for (i, masks) in batched_masks.iter().enumerate() {
-        coordinator_db.insert_masks(&masks).await?;
+    let mut tasks = FuturesUnordered::new();
+    let mut i = 0;
+
+    for masks in batched_masks.iter() {
+        tasks.push(coordinator_db.insert_masks(&masks));
+    }
+
+    while let Some(result) = tasks.next().await {
+        result?;
 
         println!(
             "Inserted masks {}/{} into coordinator db",
             (i + 1) * batch_size,
             num_templates
         );
+
+        i += 1;
     }
+
     Ok(())
 }
 
@@ -232,8 +240,9 @@ async fn insert_shares(
     num_templates: usize,
 ) -> eyre::Result<()> {
     let mut tasks = FuturesUnordered::new();
+    let mut i = 0;
 
-    for (i, mut shares) in batched_shares.into_iter().enumerate() {
+    for mut shares in batched_shares.into_iter() {
         for (idx, db) in participant_dbs.iter().enumerate() {
             let participant_shares = mem::take(&mut shares[idx]);
             let db = db.clone();
@@ -243,16 +252,18 @@ async fn insert_shares(
                 Ok::<_, Error>(idx)
             }));
         }
+    }
 
-        while let Some(result) = tasks.next().await {
-            let idx = result??;
-            println!(
-                "Inserted shares {}/{} into participant {} db",
-                (i + 1) * batch_size,
-                num_templates,
-                idx,
-            );
-        }
+    while let Some(result) = tasks.next().await {
+        let idx = result??;
+        println!(
+            "Inserted shares {}/{} into participant {} db",
+            (i + 1) * batch_size,
+            num_templates,
+            idx,
+        );
+
+        i += 1;
     }
     Ok(())
 }

From ea42212ba03ece9bb5a05e3ad4c97dc08ee6427b Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 23:41:06 -0500
Subject: [PATCH 10/24] added print log while waiting for sync

---
 bin/utils/seed_mpc_db.rs | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index a6e6226..46e9017 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,4 +1,5 @@
 use core::num;
+use std::collections::HashMap;
 use std::mem;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
@@ -48,6 +49,15 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         generate_shares_and_masks(args, templates);
 
     println!("Inserting masks and shares into db");
+
+    // Print a message while waiting for insertions to complete
+    let print_handle = tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(tokio::time::Duration::from_secs(4)).await;
+            println!("Waiting for db seed to complete...")
+        }
+    });
+
     insert_masks_and_shares(
         batched_masks,
         batched_shares,
@@ -58,6 +68,8 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
     )
     .await?;
 
+    print_handle.abort();
+
     println!("Time elapsed: {:?}", now.elapsed());
 
     Ok(())
@@ -137,8 +149,7 @@ fn generate_shares_and_masks(
 
         let shares_chunk = chunk
             .into_par_iter()
-            .enumerate()
-            .map(|(i, template)| {
+            .map(|template| {
                 println!(
                     "Encoding template {}/{}",
                     encoded_shares_counter.load(Ordering::Relaxed),
@@ -240,7 +251,11 @@ async fn insert_shares(
     num_templates: usize,
 ) -> eyre::Result<()> {
     let mut tasks = FuturesUnordered::new();
-    let mut i = 0;
+    let mut counters = HashMap::new();
+
+    for (i, _) in participant_dbs.iter().enumerate() {
+        counters.insert(i, 0_usize);
+    }
 
     for mut shares in batched_shares.into_iter() {
         for (idx, db) in participant_dbs.iter().enumerate() {
@@ -256,14 +271,16 @@ async fn insert_shares(
 
     while let Some(result) = tasks.next().await {
         let idx = result??;
+
+        let counter = counters.get_mut(&idx).expect("Could not get counter");
         println!(
             "Inserted shares {}/{} into participant {} db",
-            (i + 1) * batch_size,
+            (*counter + 1) * batch_size,
             num_templates,
             idx,
         );
 
-        i += 1;
+        *counter += 1;
     }
     Ok(())
 }

From 477fd4403836a2b9f634f23111883a460e30c838 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Wed, 28 Feb 2024 23:47:44 -0500
Subject: [PATCH 11/24] adjusted logs

---
 bin/utils/seed_mpc_db.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 46e9017..ce53318 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -232,10 +232,11 @@ async fn insert_masks(
     while let Some(result) = tasks.next().await {
         result?;
 
+        let num_inserted = std::cmp::min((i + 1) * batch_size, num_templates);
+
         println!(
             "Inserted masks {}/{} into coordinator db",
-            (i + 1) * batch_size,
-            num_templates
+            num_inserted, num_templates
         );
 
         i += 1;
@@ -273,13 +274,14 @@ async fn insert_shares(
         let idx = result??;
 
         let counter = counters.get_mut(&idx).expect("Could not get counter");
+
+        let num_inserted =
+            std::cmp::min((*counter + 1) * batch_size, num_templates);
+
         println!(
             "Inserted shares {}/{} into participant {} db",
-            (*counter + 1) * batch_size,
-            num_templates,
-            idx,
+            num_inserted, num_templates, idx,
         );
-
         *counter += 1;
     }
     Ok(())

From f14ddb6e8f31264b042562ef9a63e7b6f1d0f2d1 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 01:03:30 -0500
Subject: [PATCH 12/24] updated progress bars

---
 bin/utils/seed_mpc_db.rs | 135 ++++++++++++++++++++-------------------
 1 file changed, 68 insertions(+), 67 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index ce53318..a566528 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,13 +1,15 @@
 use core::num;
 use std::collections::HashMap;
 use std::mem;
-use std::sync::atomic::Ordering;
 use std::sync::Arc;
+use std::thread::available_parallelism;
+use std::time::Duration;
 
 use clap::Args;
 use eyre::Error;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
+use indicatif::{ProgressBar, ProgressStyle};
 use metrics::atomics::AtomicU64;
 use mpc::bits::Bits;
 use mpc::config::DbConfig;
@@ -18,6 +20,7 @@ use rand::{thread_rng, Rng};
 use rayon::iter::{
     IndexedParallelIterator, IntoParallelIterator, ParallelIterator,
 };
+use rayon::{current_num_threads, ThreadPoolBuilder};
 
 #[derive(Debug, Clone, Args)]
 pub struct SeedMPCDb {
@@ -27,36 +30,39 @@ pub struct SeedMPCDb {
     #[clap(short, long)]
     pub participant_db_url: Vec<String>,
 
-    #[clap(short, long, default_value = "3000000")]
+    #[clap(short, long, default_value = "1000000")]
     pub num_templates: usize,
 
-    #[clap(short, long, default_value = "3000")]
+    #[clap(short, long, default_value = "5000")]
     pub batch_size: usize,
 }
 
 pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
-    let now = std::time::Instant::now();
-
     if args.participant_db_url.is_empty() {
         return Err(eyre::eyre!("No participant DBs provided"));
     }
 
     let (coordinator_db, participant_dbs) = initialize_dbs(args).await?;
 
+    // Configure rayon global thread pool
+    let mut pool_builder = ThreadPoolBuilder::new();
+    pool_builder = pool_builder.num_threads(available_parallelism()?.into());
+    pool_builder.build_global()?;
+
+    eprintln!(
+        "Using {} compute threads on {} cores.",
+        current_num_threads(),
+        available_parallelism()?
+    );
+
+    let now = std::time::Instant::now();
+
     let templates = generate_templates(args);
+    println!("Templates generated in {:?}", now.elapsed());
 
     let (batched_masks, batched_shares) =
         generate_shares_and_masks(args, templates);
-
-    println!("Inserting masks and shares into db");
-
-    // Print a message while waiting for insertions to complete
-    let print_handle = tokio::spawn(async move {
-        loop {
-            tokio::time::sleep(tokio::time::Duration::from_secs(4)).await;
-            println!("Waiting for db seed to complete...")
-        }
-    });
+    println!("Shares and masks generated in {:?}", now.elapsed());
 
     insert_masks_and_shares(
         batched_masks,
@@ -64,12 +70,9 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         coordinator_db,
         participant_dbs,
         args.batch_size,
-        args.num_templates,
     )
     .await?;
 
-    print_handle.abort();
-
     println!("Time elapsed: {:?}", now.elapsed());
 
     Ok(())
@@ -104,24 +107,28 @@ async fn initialize_dbs(
 }
 
 fn generate_templates(args: &SeedMPCDb) -> Vec<Template> {
+    let pb = ProgressBar::new(args.num_templates as u64)
+        .with_message("Generating templates...");
+
+    pb.set_style(ProgressStyle::default_bar()
+        .template("{spinner:.green} {msg} [{elapsed_precise}] [{wide_bar:.green}] {pos:>7}/{len:7} ({eta})")
+        .expect("Could not create progress bar"));
+
     // Generate templates
-    let template_counter = AtomicU64::new(0);
     let templates = (0..args.num_templates)
         .into_par_iter()
         .map(|_| {
             let mut rng = thread_rng();
 
-            println!(
-                "Generating template {}/{}",
-                template_counter.load(Ordering::Relaxed),
-                args.num_templates
-            );
             let template = rng.gen();
-            template_counter.fetch_add(1, Ordering::Relaxed);
+
+            pb.inc(1);
             template
         })
         .collect::<Vec<Template>>();
 
+    pb.finish_with_message("Created templates");
+
     templates
 }
 
@@ -138,28 +145,26 @@ fn generate_shares_and_masks(
 
     let num_participants = args.participant_db_url.len();
 
+    let pb = ProgressBar::new(args.num_templates as u64)
+        .with_message("Generating shares and masks...");
+
+    pb.set_style(ProgressStyle::default_bar()
+        .template("{spinner:.green} {msg} [{elapsed_precise}] [{wide_bar:.green}] {pos:>7}/{len:7} ({eta})")
+        .expect("Could not create progress bar"));
+
     for (idx, chunk) in templates.chunks(args.batch_size).enumerate() {
         let mut chunk_masks = Vec::with_capacity(chunk.len());
         let mut chunk_shares: Vec<_> = (0..num_participants)
             .map(|_| Vec::with_capacity(chunk.len()))
             .collect();
 
-        let encoded_shares_counter =
-            AtomicU64::new((idx * args.batch_size) as u64);
-
         let shares_chunk = chunk
             .into_par_iter()
             .map(|template| {
-                println!(
-                    "Encoding template {}/{}",
-                    encoded_shares_counter.load(Ordering::Relaxed),
-                    args.num_templates
-                );
                 let shares =
                     mpc::distance::encode(template).share(num_participants);
 
-                encoded_shares_counter.fetch_add(1, Ordering::Relaxed);
-
+                pb.inc(1);
                 shares
             })
             .collect::<Vec<Box<[EncodedBits]>>>();
@@ -180,6 +185,8 @@ fn generate_shares_and_masks(
         batched_masks.push(chunk_masks);
     }
 
+    pb.finish_with_message("Created shares and masks");
+
     (batched_masks, batched_shares)
 }
 
@@ -189,24 +196,22 @@ async fn insert_masks_and_shares(
     coordinator_db: Arc<Db>,
     participant_dbs: Vec<Arc<Db>>,
     batch_size: usize,
-    num_templates: usize,
 ) -> eyre::Result<()> {
-    // Commit shares and masks to db
+    println!("Inserting masks and shares into db...");
 
+    // Commit shares and masks to db
     let mut tasks = FuturesUnordered::new();
 
     tasks.push(tokio::spawn(insert_masks(
         batched_masks,
         coordinator_db,
         batch_size,
-        num_templates,
     )));
 
     tasks.push(tokio::spawn(insert_shares(
         batched_shares,
         participant_dbs,
         batch_size,
-        num_templates,
     )));
 
     while let Some(result) = tasks.next().await {
@@ -219,44 +224,47 @@ async fn insert_masks_and_shares(
 async fn insert_masks(
     batched_masks: BatchedMasks,
     coordinator_db: Arc<Db>,
-    batch_size: usize,
     num_templates: usize,
 ) -> eyre::Result<()> {
+    let pb = ProgressBar::new(num_templates as u64)
+        .with_message("Inserting masks...");
+
+    pb.set_style(ProgressStyle::default_bar()
+    .template("{spinner:.green} {msg} [{elapsed_precise}] [{wide_bar:.green}] {pos:>7}/{len:7} ({eta})")
+    .expect("Could not create progress bar"));
+
+    pb.inc(0);
+
     let mut tasks = FuturesUnordered::new();
-    let mut i = 0;
 
     for masks in batched_masks.iter() {
         tasks.push(coordinator_db.insert_masks(&masks));
+        tokio::time::sleep(Duration::from_millis(10)).await;
     }
 
     while let Some(result) = tasks.next().await {
         result?;
-
-        let num_inserted = std::cmp::min((i + 1) * batch_size, num_templates);
-
-        println!(
-            "Inserted masks {}/{} into coordinator db",
-            num_inserted, num_templates
-        );
-
-        i += 1;
     }
 
+    pb.finish_with_message("Inserted masks");
+
     Ok(())
 }
 
 async fn insert_shares(
     batched_shares: BatchedShares,
     participant_dbs: Vec<Arc<Db>>,
-    batch_size: usize,
     num_templates: usize,
 ) -> eyre::Result<()> {
-    let mut tasks = FuturesUnordered::new();
-    let mut counters = HashMap::new();
+    let pb = ProgressBar::new(num_templates as u64)
+        .with_message("Inserting shares...");
 
-    for (i, _) in participant_dbs.iter().enumerate() {
-        counters.insert(i, 0_usize);
-    }
+    pb.set_style(ProgressStyle::default_bar()
+        .template("{spinner:.green} {msg} [{elapsed_precise}] [{wide_bar:.green}] {pos:>7}/{len:7} ({eta})")
+        .expect("Could not create progress bar"));
+    pb.inc(0);
+
+    let mut tasks = FuturesUnordered::new();
 
     for mut shares in batched_shares.into_iter() {
         for (idx, db) in participant_dbs.iter().enumerate() {
@@ -265,24 +273,17 @@ async fn insert_shares(
 
             tasks.push(tokio::spawn(async move {
                 db.insert_shares(&participant_shares).await?;
+                tokio::time::sleep(Duration::from_millis(10)).await;
                 Ok::<_, Error>(idx)
             }));
         }
     }
 
     while let Some(result) = tasks.next().await {
-        let idx = result??;
-
-        let counter = counters.get_mut(&idx).expect("Could not get counter");
+        result??;
+    }
 
-        let num_inserted =
-            std::cmp::min((*counter + 1) * batch_size, num_templates);
+    pb.finish_with_message("Inserted shares");
 
-        println!(
-            "Inserted shares {}/{} into participant {} db",
-            num_inserted, num_templates, idx,
-        );
-        *counter += 1;
-    }
     Ok(())
 }

From 0b6f3ea9924669359c48600fdd3f2db7ca24b8b2 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 01:04:27 -0500
Subject: [PATCH 13/24] clippy

---
 bin/utils/seed_mpc_db.rs | 20 ++------------------
 bin/utils/utils.rs       |  2 +-
 2 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index a566528..0e5b033 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,5 +1,3 @@
-use core::num;
-use std::collections::HashMap;
 use std::mem;
 use std::sync::Arc;
 use std::thread::available_parallelism;
@@ -10,16 +8,13 @@ use eyre::Error;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
 use indicatif::{ProgressBar, ProgressStyle};
-use metrics::atomics::AtomicU64;
 use mpc::bits::Bits;
 use mpc::config::DbConfig;
 use mpc::db::Db;
 use mpc::distance::EncodedBits;
 use mpc::template::Template;
 use rand::{thread_rng, Rng};
-use rayon::iter::{
-    IndexedParallelIterator, IntoParallelIterator, ParallelIterator,
-};
+use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use rayon::{current_num_threads, ThreadPoolBuilder};
 
 #[derive(Debug, Clone, Args)]
@@ -44,17 +39,6 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
 
     let (coordinator_db, participant_dbs) = initialize_dbs(args).await?;
 
-    // Configure rayon global thread pool
-    let mut pool_builder = ThreadPoolBuilder::new();
-    pool_builder = pool_builder.num_threads(available_parallelism()?.into());
-    pool_builder.build_global()?;
-
-    eprintln!(
-        "Using {} compute threads on {} cores.",
-        current_num_threads(),
-        available_parallelism()?
-    );
-
     let now = std::time::Instant::now();
 
     let templates = generate_templates(args);
@@ -238,7 +222,7 @@ async fn insert_masks(
     let mut tasks = FuturesUnordered::new();
 
     for masks in batched_masks.iter() {
-        tasks.push(coordinator_db.insert_masks(&masks));
+        tasks.push(coordinator_db.insert_masks(masks));
         tokio::time::sleep(Duration::from_millis(10)).await;
     }
 
diff --git a/bin/utils/utils.rs b/bin/utils/utils.rs
index 3e9169e..1b7febc 100644
--- a/bin/utils/utils.rs
+++ b/bin/utils/utils.rs
@@ -6,7 +6,7 @@ use seed_iris_db::{seed_iris_db, SeedIrisDb};
 use seed_mpc_db::{seed_mpc_db, SeedMPCDb};
 use sqs_query::{sqs_query, SQSQuery};
 use sqs_receive::{sqs_receive, SQSReceive};
-use telemetry_batteries::tracing::stdout::StdoutBattery;
+
 
 mod generate_mock_templates;
 mod seed_iris_db;

From daf921287261beba90d16267f0fd0ee6d6ca8f73 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 01:04:59 -0500
Subject: [PATCH 14/24] removed unused imports

---
 bin/utils/seed_mpc_db.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 0e5b033..4353fbc 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,6 +1,5 @@
 use std::mem;
 use std::sync::Arc;
-use std::thread::available_parallelism;
 use std::time::Duration;
 
 use clap::Args;
@@ -15,7 +14,6 @@ use mpc::distance::EncodedBits;
 use mpc::template::Template;
 use rand::{thread_rng, Rng};
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
-use rayon::{current_num_threads, ThreadPoolBuilder};
 
 #[derive(Debug, Clone, Args)]
 pub struct SeedMPCDb {

From d5137e1b6702ca85d5a25e04458501872d2f4400 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 01:14:00 -0500
Subject: [PATCH 15/24] updated progress bars

---
 bin/utils/seed_mpc_db.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index eb24ba5..dd7a1e9 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -53,6 +53,7 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
         batched_shares,
         coordinator_db,
         participant_dbs,
+        args.num_templates,
         args.batch_size,
     )
     .await?;
@@ -179,6 +180,7 @@ async fn insert_masks_and_shares(
     batched_shares: BatchedShares,
     coordinator_db: Arc<Db>,
     participant_dbs: Vec<Arc<Db>>,
+    num_templates: usize,
     batch_size: usize,
 ) -> eyre::Result<()> {
     println!("Inserting masks and shares into db...");
@@ -189,12 +191,14 @@ async fn insert_masks_and_shares(
     tasks.push(tokio::spawn(insert_masks(
         batched_masks,
         coordinator_db,
+        num_templates,
         batch_size,
     )));
 
     tasks.push(tokio::spawn(insert_shares(
         batched_shares,
         participant_dbs,
+        num_templates,
         batch_size,
     )));
 
@@ -202,8 +206,6 @@ async fn insert_masks_and_shares(
         result??;
     }
 
-    println!("Time elapsed {:?}", now.elapsed());
-
     Ok(())
 }
 
@@ -211,6 +213,7 @@ async fn insert_masks(
     batched_masks: BatchedMasks,
     coordinator_db: Arc<Db>,
     num_templates: usize,
+    batch_size: usize,
 ) -> eyre::Result<()> {
     let pb = ProgressBar::new(num_templates as u64)
         .with_message("Inserting masks...");
@@ -230,6 +233,7 @@ async fn insert_masks(
 
     while let Some(result) = tasks.next().await {
         result?;
+        pb.inc(batch_size as u64);
     }
 
     pb.finish_with_message("Inserted masks");
@@ -241,6 +245,7 @@ async fn insert_shares(
     batched_shares: BatchedShares,
     participant_dbs: Vec<Arc<Db>>,
     num_templates: usize,
+    batch_size: usize,
 ) -> eyre::Result<()> {
     let pb = ProgressBar::new(num_templates as u64)
         .with_message("Inserting shares...");
@@ -267,6 +272,7 @@ async fn insert_shares(
 
     while let Some(result) = tasks.next().await {
         result??;
+        pb.inc(batch_size as u64);
     }
 
     pb.finish_with_message("Inserted shares");

From e1bb31b2048fdae6d6edd485d233c4bf5ad9509e Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 01:14:33 -0500
Subject: [PATCH 16/24] removed unused var

---
 bin/utils/seed_mpc_db.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index dd7a1e9..0eec130 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -31,8 +31,6 @@ pub struct SeedMPCDb {
 }
 
 pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
-    let now = std::time::Instant::now();
-
     if args.participant_db_url.is_empty() {
         return Err(eyre::eyre!("No participant DBs provided"));
     }

From 6c014d735e9a77e14eed7e8e33732f564873a700 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 01:21:44 -0500
Subject: [PATCH 17/24] fmt

---
 bin/utils/utils.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/utils/utils.rs b/bin/utils/utils.rs
index 1b7febc..feb74ae 100644
--- a/bin/utils/utils.rs
+++ b/bin/utils/utils.rs
@@ -7,7 +7,6 @@ use seed_mpc_db::{seed_mpc_db, SeedMPCDb};
 use sqs_query::{sqs_query, SQSQuery};
 use sqs_receive::{sqs_receive, SQSReceive};
 
-
 mod generate_mock_templates;
 mod seed_iris_db;
 mod seed_mpc_db;

From 23a1ffb6e01c1182474db8be3a884ccbe406e2ba Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 10:03:59 -0500
Subject: [PATCH 18/24] adjust for 1 based indexing

---
 bin/utils/seed_mpc_db.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 0eec130..a5d9713 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -155,7 +155,7 @@ fn generate_shares_and_masks(
         for (offset, (shares, template)) in
             shares_chunk.iter().zip(chunk).enumerate()
         {
-            let id = offset + (idx * args.batch_size);
+            let id = offset + (idx * args.batch_size) + 1;
 
             chunk_masks.push((id as u64, template.mask));
 

From b43980969629be71099cc912f18e1ec1bff55f99 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 18:17:18 -0500
Subject: [PATCH 19/24] sequentially insert masks and shares, update progress
 bar

---
 bin/utils/seed_mpc_db.rs | 119 +++++++++++++++++++++------------------
 1 file changed, 65 insertions(+), 54 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index a5d9713..b07e183 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,14 +1,13 @@
 use std::mem;
 use std::sync::Arc;
-use std::time::Duration;
 
 use clap::Args;
-use eyre::Error;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
-use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use mpc::bits::Bits;
 use mpc::config::DbConfig;
+use mpc::coordinator;
 use mpc::db::Db;
 use mpc::distance::EncodedBits;
 use mpc::template::Template;
@@ -136,8 +135,8 @@ fn generate_shares_and_masks(
         .expect("Could not create progress bar"));
 
     for (idx, chunk) in templates.chunks(args.batch_size).enumerate() {
-        let mut chunk_masks = Vec::with_capacity(chunk.len());
-        let mut chunk_shares: Vec<_> = (0..num_participants)
+        let mut batch_masks = Vec::with_capacity(chunk.len());
+        let mut batch_shares: Vec<_> = (0..num_participants)
             .map(|_| Vec::with_capacity(chunk.len()))
             .collect();
 
@@ -157,15 +156,18 @@ fn generate_shares_and_masks(
         {
             let id = offset + (idx * args.batch_size) + 1;
 
-            chunk_masks.push((id as u64, template.mask));
+            batch_masks.push((id as u64, template.mask));
 
+            // < <share 0, share 1>, <share 0, share 1>  >
             for (idx, share) in shares.iter().enumerate() {
-                chunk_shares[idx].push((id as u64, *share));
+                batch_shares[idx].push((id as u64, *share));
             }
         }
 
-        batched_shares.push(chunk_shares);
-        batched_masks.push(chunk_masks);
+        // vec of batches
+        // < Batch< <share 0, share 1>, <share 0, share 1>  >, Batch< <share 0, share 1>, <share 0, share 1>  > >
+        batched_shares.push(batch_shares);
+        batched_masks.push(batch_masks);
     }
 
     pb.finish_with_message("Created shares and masks");
@@ -184,24 +186,57 @@ async fn insert_masks_and_shares(
     println!("Inserting masks and shares into db...");
 
     // Commit shares and masks to db
-    let mut tasks = FuturesUnordered::new();
+    let mpb = MultiProgress::new();
+    let style = ProgressStyle::default_bar().template("{spinner:.green} {msg} [{elapsed_precise}] [{wide_bar:.green}] {pos:>7}/{len:7} ({eta})")?;
+
+    let participant_progress_bars = participant_dbs
+        .iter()
+        .enumerate()
+        .map(|(i, _)| {
+            mpb.add(ProgressBar::new(num_templates as u64).with_message(
+                format!("Inserting shares for participant {}", i),
+            ))
+        })
+        .collect::<Vec<_>>();
 
-    tasks.push(tokio::spawn(insert_masks(
-        batched_masks,
-        coordinator_db,
-        num_templates,
-        batch_size,
-    )));
+    for pb in participant_progress_bars.iter() {
+        pb.set_style(style.clone());
+    }
 
-    tasks.push(tokio::spawn(insert_shares(
-        batched_shares,
-        participant_dbs,
-        num_templates,
-        batch_size,
-    )));
+    let coordinator_progress_bar = mpb.add(
+        ProgressBar::new(num_templates as u64)
+            .with_message("Inserting masks for coordinator"),
+    );
+    coordinator_progress_bar.set_style(style.clone());
+
+    for (shares, masks) in
+        batched_shares.into_iter().zip(batched_masks.into_iter())
+    {
+        let mut tasks = FuturesUnordered::new();
+
+        for (idx, db) in participant_dbs.iter().enumerate() {
+            let shares = shares[idx].clone();
+            let db = db.clone();
+            let pb = participant_progress_bars[idx].clone();
+
+            tasks.push(tokio::spawn(async move {
+                db.insert_shares(&shares).await?;
+                pb.inc(batch_size as u64);
+                eyre::Result::<()>::Ok(())
+            }));
+        }
+
+        let coordinator_db = coordinator_db.clone();
+        let coordinator_progress_bar = coordinator_progress_bar.clone();
+        tasks.push(tokio::spawn(async move {
+            coordinator_db.insert_masks(&masks).await?;
+            coordinator_progress_bar.inc(batch_size as u64);
+            eyre::Result::<()>::Ok(())
+        }));
 
-    while let Some(result) = tasks.next().await {
-        result??;
+        while let Some(result) = tasks.next().await {
+            result??;
+        }
     }
 
     Ok(())
@@ -210,31 +245,17 @@ async fn insert_masks_and_shares(
 async fn insert_masks(
     batched_masks: BatchedMasks,
     coordinator_db: Arc<Db>,
-    num_templates: usize,
     batch_size: usize,
+    progress_bar: ProgressBar,
 ) -> eyre::Result<()> {
-    let pb = ProgressBar::new(num_templates as u64)
-        .with_message("Inserting masks...");
-
-    pb.set_style(ProgressStyle::default_bar()
-    .template("{spinner:.green} {msg} [{elapsed_precise}] [{wide_bar:.green}] {pos:>7}/{len:7} ({eta})")
-    .expect("Could not create progress bar"));
-
-    pb.inc(0);
-
-    let mut tasks = FuturesUnordered::new();
+    progress_bar.inc(0);
 
     for masks in batched_masks.iter() {
-        tasks.push(coordinator_db.insert_masks(masks));
-        tokio::time::sleep(Duration::from_millis(10)).await;
+        coordinator_db.insert_masks(masks).await?;
+        progress_bar.inc(batch_size as u64);
     }
 
-    while let Some(result) = tasks.next().await {
-        result?;
-        pb.inc(batch_size as u64);
-    }
-
-    pb.finish_with_message("Inserted masks");
+    progress_bar.finish_with_message("Inserted masks");
 
     Ok(())
 }
@@ -253,23 +274,13 @@ async fn insert_shares(
         .expect("Could not create progress bar"));
     pb.inc(0);
 
-    let mut tasks = FuturesUnordered::new();
-
     for mut shares in batched_shares.into_iter() {
         for (idx, db) in participant_dbs.iter().enumerate() {
             let participant_shares = mem::take(&mut shares[idx]);
             let db = db.clone();
 
-            tasks.push(tokio::spawn(async move {
-                db.insert_shares(&participant_shares).await?;
-                tokio::time::sleep(Duration::from_millis(10)).await;
-                Ok::<_, Error>(idx)
-            }));
+            db.insert_shares(&participant_shares).await?;
         }
-    }
-
-    while let Some(result) = tasks.next().await {
-        result??;
         pb.inc(batch_size as u64);
     }
 

From f62a8db14f963e95947d6e34559bab6e79e4e1d3 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 19:15:49 -0500
Subject: [PATCH 20/24] updated insert starting from latest serial id

---
 bin/utils/seed_mpc_db.rs | 64 +++++++++++++---------------------------
 src/db.rs                | 40 +++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 44 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index b07e183..24a8bad 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,4 +1,3 @@
-use std::mem;
 use std::sync::Arc;
 
 use clap::Args;
@@ -7,7 +6,6 @@ use futures::StreamExt;
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use mpc::bits::Bits;
 use mpc::config::DbConfig;
-use mpc::coordinator;
 use mpc::db::Db;
 use mpc::distance::EncodedBits;
 use mpc::template::Template;
@@ -38,11 +36,18 @@ pub async fn seed_mpc_db(args: &SeedMPCDb) -> eyre::Result<()> {
 
     let now = std::time::Instant::now();
 
+    let latest_serial_id =
+        get_latest_serial_id(coordinator_db.clone(), participant_dbs.clone())
+            .await?;
+
     let templates = generate_templates(args);
     println!("Templates generated in {:?}", now.elapsed());
 
-    let (batched_masks, batched_shares) =
-        generate_shares_and_masks(args, templates);
+    let (batched_masks, batched_shares) = generate_shares_and_masks(
+        args,
+        templates,
+        (latest_serial_id + 1) as usize,
+    );
     println!("Shares and masks generated in {:?}", now.elapsed());
 
     insert_masks_and_shares(
@@ -120,6 +125,7 @@ pub type BatchedMasks = Vec<Vec<(u64, Bits)>>;
 fn generate_shares_and_masks(
     args: &SeedMPCDb,
     templates: Vec<Template>,
+    next_serial_id: usize,
 ) -> (BatchedMasks, BatchedShares) {
     // Generate shares and masks
     let mut batched_shares = vec![];
@@ -154,7 +160,7 @@ fn generate_shares_and_masks(
         for (offset, (shares, template)) in
             shares_chunk.iter().zip(chunk).enumerate()
         {
-            let id = offset + (idx * args.batch_size) + 1;
+            let id = offset + (idx * args.batch_size) + next_serial_id;
 
             batch_masks.push((id as u64, template.mask));
 
@@ -242,49 +248,19 @@ async fn insert_masks_and_shares(
     Ok(())
 }
 
-async fn insert_masks(
-    batched_masks: BatchedMasks,
+async fn get_latest_serial_id(
     coordinator_db: Arc<Db>,
-    batch_size: usize,
-    progress_bar: ProgressBar,
-) -> eyre::Result<()> {
-    progress_bar.inc(0);
-
-    for masks in batched_masks.iter() {
-        coordinator_db.insert_masks(masks).await?;
-        progress_bar.inc(batch_size as u64);
-    }
-
-    progress_bar.finish_with_message("Inserted masks");
-
-    Ok(())
-}
-
-async fn insert_shares(
-    batched_shares: BatchedShares,
     participant_dbs: Vec<Arc<Db>>,
-    num_templates: usize,
-    batch_size: usize,
-) -> eyre::Result<()> {
-    let pb = ProgressBar::new(num_templates as u64)
-        .with_message("Inserting shares...");
-
-    pb.set_style(ProgressStyle::default_bar()
-        .template("{spinner:.green} {msg} [{elapsed_precise}] [{wide_bar:.green}] {pos:>7}/{len:7} ({eta})")
-        .expect("Could not create progress bar"));
-    pb.inc(0);
+) -> eyre::Result<u64> {
+    let coordinator_serial_id = coordinator_db.fetch_latest_mask_id().await?;
+    println!("Coordinator serial id: {}", coordinator_serial_id);
 
-    for mut shares in batched_shares.into_iter() {
-        for (idx, db) in participant_dbs.iter().enumerate() {
-            let participant_shares = mem::take(&mut shares[idx]);
-            let db = db.clone();
+    for (i, db) in participant_dbs.iter().enumerate() {
+        let participant_id = db.fetch_latest_share_id().await?;
 
-            db.insert_shares(&participant_shares).await?;
-        }
-        pb.inc(batch_size as u64);
+        println!("Participant {} ids: {}", i, participant_id);
+        assert_eq!(coordinator_serial_id, participant_id);
     }
 
-    pb.finish_with_message("Inserted shares");
-
-    Ok(())
+    Ok(coordinator_serial_id)
 }
diff --git a/src/db.rs b/src/db.rs
index 5c262c9..3417605 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -53,6 +53,46 @@ impl Db {
         Ok(filter_sequential_items(masks, 1 + id as i64))
     }
 
+    #[tracing::instrument(skip(self))]
+    pub async fn fetch_latest_mask_id(&self) -> eyre::Result<u64> {
+        let mask_id = sqlx::query_as::<_, (i64,)>(
+            r#"
+            SELECT id
+            FROM masks
+            ORDER BY id DESC
+            LIMIT 1
+        "#,
+        )
+        .fetch_one(&self.pool)
+        .await;
+
+        match mask_id {
+            Ok(mask_id) => Ok(mask_id.0 as u64),
+            Err(sqlx::Error::RowNotFound) => Ok(0),
+            Err(err) => Err(err.into()),
+        }
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub async fn fetch_latest_share_id(&self) -> eyre::Result<u64> {
+        let share_id = sqlx::query_as::<_, (i64,)>(
+            r#"
+            SELECT id
+            FROM shares
+            ORDER BY id DESC
+            LIMIT 1
+        "#,
+        )
+        .fetch_one(&self.pool)
+        .await;
+
+        match share_id {
+            Ok(share_id) => Ok(share_id.0 as u64),
+            Err(sqlx::Error::RowNotFound) => Ok(0),
+            Err(err) => Err(err.into()),
+        }
+    }
+
     #[tracing::instrument(skip(self))]
     pub async fn insert_masks(
         &self,

From f790819f0337ac08317a833a4be39057f80042a9 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Thu, 29 Feb 2024 21:29:03 -0500
Subject: [PATCH 21/24] updated default vals

---
 bin/utils/seed_mpc_db.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index 24a8bad..6562159 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -20,10 +20,10 @@ pub struct SeedMPCDb {
     #[clap(short, long)]
     pub participant_db_url: Vec<String>,
 
-    #[clap(short, long, default_value = "1000000")]
+    #[clap(short, long, default_value = "100000")]
     pub num_templates: usize,
 
-    #[clap(short, long, default_value = "5000")]
+    #[clap(short, long, default_value = "100")]
     pub batch_size: usize,
 }
 
@@ -164,14 +164,11 @@ fn generate_shares_and_masks(
 
             batch_masks.push((id as u64, template.mask));
 
-            // < <share 0, share 1>, <share 0, share 1>  >
             for (idx, share) in shares.iter().enumerate() {
                 batch_shares[idx].push((id as u64, *share));
             }
         }
 
-        // vec of batches
-        // < Batch< <share 0, share 1>, <share 0, share 1>  >, Batch< <share 0, share 1>, <share 0, share 1>  > >
         batched_shares.push(batch_shares);
         batched_masks.push(batch_masks);
     }
@@ -259,7 +256,10 @@ async fn get_latest_serial_id(
         let participant_id = db.fetch_latest_share_id().await?;
 
         println!("Participant {} ids: {}", i, participant_id);
-        assert_eq!(coordinator_serial_id, participant_id);
+        assert_eq!(
+            coordinator_serial_id, participant_id,
+            "Databases are not in sync"
+        );
     }
 
     Ok(coordinator_serial_id)

From 308b98178485df6ad925df4d31772d12614f1c27 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Fri, 1 Mar 2024 11:02:36 -0500
Subject: [PATCH 22/24] clippy

---
 bin/utils/seed_mpc_db.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index e49538b..fa82a89 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,4 +1,4 @@
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc};
 
 use clap::Args;
 use futures::stream::FuturesUnordered;

From 7ac48871e9ea288d76d60aca1c293f0c5e171dbc Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Fri, 1 Mar 2024 12:41:30 -0500
Subject: [PATCH 23/24] cargo fmt

---
 bin/utils/seed_mpc_db.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/utils/seed_mpc_db.rs b/bin/utils/seed_mpc_db.rs
index fa82a89..1083837 100644
--- a/bin/utils/seed_mpc_db.rs
+++ b/bin/utils/seed_mpc_db.rs
@@ -1,4 +1,4 @@
-use std::sync::{Arc};
+use std::sync::Arc;
 
 use clap::Args;
 use futures::stream::FuturesUnordered;

From 8c8739c638c0dfdc35221b15cba4513f16503977 Mon Sep 17 00:00:00 2001
From: 0xKitsune <0xKitsune@protonmail.com>
Date: Fri, 1 Mar 2024 12:50:01 -0500
Subject: [PATCH 24/24] uncomment e2e

---
 compose.yml | 62 ++++++++++++++++++++++++++---------------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/compose.yml b/compose.yml
index 6a12346..6971c7c 100644
--- a/compose.yml
+++ b/compose.yml
@@ -107,37 +107,37 @@ services:
       - 8434:5432
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
-  # e2e_test:
-  #   depends_on:
-  #     - localstack
-  #     - coordinator
-  #     - participant_0
-  #     - participant_1
-  #     - coordinator_db
-  #     - participant_0_db
-  #     - participant_1_db
-  #   build:
-  #     context: .
-  #     dockerfile: Dockerfile
-  #     args:
-  #       - BIN=e2e
-  #   command: [ "--signup-sequence", "/signup_sequence.json" ]
-  #   volumes:
-  #     - ./bin/e2e/signup_sequence.json:/signup_sequence.json
-  #   environment:
-  #     - 'RUST_LOG=info'
-  #     - 'E2E__AWS__ENDPOINT=http://localstack:4566'
-  #     - 'E2E__AWS__REGION=us-east-1'
-  #     - 'E2E__DB_SYNC__COORDINATOR_DB_URL=postgres://postgres:postgres@coordinator_db:5432/db'
-  #     - 'E2E__DB_SYNC__COORDINATOR_DB_SYNC_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/coordinator-db-sync-queue'
-  #     - 'E2E__DB_SYNC__PARTICIPANT_0_DB_SYNC_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/participant-0-db-sync-queue'
-  #     - 'E2E__DB_SYNC__PARTICIPANT_1_DB_SYNC_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/participant-1-db-sync-queue'
-  #     - 'E2E__COORDINATOR_QUEUE__QUERY_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/coordinator-uniqueness-check.fifo'
-  #     - 'E2E__COORDINATOR_QUEUE__RESULTS_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/coordinator-results-queue.fifo'
-  #     # AWS env vars - they don't matter but are required
-  #     - 'AWS_ACCESS_KEY_ID=test'
-  #     - 'AWS_SECRET_ACCESS_KEY=test'
-  #     - 'AWS_DEFAULT_REGION=us-east-1'
+  e2e_test:
+    depends_on:
+      - localstack
+      - coordinator
+      - participant_0
+      - participant_1
+      - coordinator_db
+      - participant_0_db
+      - participant_1_db
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - BIN=e2e
+    command: [ "--signup-sequence", "/signup_sequence.json" ]
+    volumes:
+      - ./bin/e2e/signup_sequence.json:/signup_sequence.json
+    environment:
+      - 'RUST_LOG=info'
+      - 'E2E__AWS__ENDPOINT=http://localstack:4566'
+      - 'E2E__AWS__REGION=us-east-1'
+      - 'E2E__DB_SYNC__COORDINATOR_DB_URL=postgres://postgres:postgres@coordinator_db:5432/db'
+      - 'E2E__DB_SYNC__COORDINATOR_DB_SYNC_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/coordinator-db-sync-queue'
+      - 'E2E__DB_SYNC__PARTICIPANT_0_DB_SYNC_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/participant-0-db-sync-queue'
+      - 'E2E__DB_SYNC__PARTICIPANT_1_DB_SYNC_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/participant-1-db-sync-queue'
+      - 'E2E__COORDINATOR_QUEUE__QUERY_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/coordinator-uniqueness-check.fifo'
+      - 'E2E__COORDINATOR_QUEUE__RESULTS_QUEUE=http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/coordinator-results-queue.fifo'
+      # AWS env vars - they don't matter but are required
+      - 'AWS_ACCESS_KEY_ID=test'
+      - 'AWS_SECRET_ACCESS_KEY=test'
+      - 'AWS_DEFAULT_REGION=us-east-1'
 
 networks:
   default: