From 7608df017ba75fe98526bc970b274de7ef21d52a Mon Sep 17 00:00:00 2001 From: Aditya Kulkarni Date: Mon, 1 Aug 2022 12:11:06 -0500 Subject: [PATCH] v1.8.0-beta2 checkpoint & track orchard notes update balance command allow mixed pool transactions --- Cargo.lock | 29 +- Cargo.toml | 21 +- cli/Cargo.toml | 2 +- cli/src/version.rs | 2 +- lib/Cargo.toml | 2 + lib/src/blaze/block_witness_data.rs | 152 ++++++++- lib/src/blaze/fetch_full_tx.rs | 4 +- lib/src/blaze/syncdata.rs | 9 +- lib/src/blaze/test_utils.rs | 7 + lib/src/blaze/trial_decryptions.rs | 35 +- lib/src/blaze/update_notes.rs | 15 +- lib/src/compact_formats.rs | 2 +- lib/src/grpc_connector.rs | 9 +- lib/src/lightclient.rs | 96 +++++- lib/src/lightclient/lightclient_config.rs | 6 +- lib/src/lightclient/test_server.rs | 30 +- lib/src/lightclient/tests.rs | 2 +- lib/src/lightwallet.rs | 380 +++++++++++++++++++--- lib/src/lightwallet/data.rs | 74 +++-- lib/src/lightwallet/keys.rs | 17 + lib/src/lightwallet/wallet_txns.rs | 127 +++++++- 21 files changed, 836 insertions(+), 185 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94c9bf17..403c9fc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,7 +782,7 @@ checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" [[package]] name = "equihash" version = "0.2.0" -source = "git+https://github.com/zcash/librustzcash?rev=09567fc280463f5797a77f1d764205be8ad8e64a#09567fc280463f5797a77f1d764205be8ad8e64a" +source = "git+https://github.com/adityapk00/librustzcash?rev=adf49f8b848a5ac85e1476354614eeae9880206a#adf49f8b848a5ac85e1476354614eeae9880206a" dependencies = [ "blake2b_simd", "byteorder", @@ -791,7 +791,7 @@ dependencies = [ [[package]] name = "f4jumble" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash?rev=09567fc280463f5797a77f1d764205be8ad8e64a#09567fc280463f5797a77f1d764205be8ad8e64a" +source = "git+https://github.com/adityapk00/librustzcash?rev=adf49f8b848a5ac85e1476354614eeae9880206a#adf49f8b848a5ac85e1476354614eeae9880206a" dependencies = [ "blake2b_simd", ] @@ -1662,8 +1662,7 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "orchard" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7619db7f917afd9b1139044c595fab1b6166de2db62317794b5f5e34a2104ae1" +source = "git+https://github.com/adityapk00/orchard?rev=0a960a380f4e9c3472c9260f3df61cd5e50d51b0#0a960a380f4e9c3472c9260f3df61cd5e50d51b0" dependencies = [ "aes", "bitvec", @@ -3622,7 +3621,7 @@ dependencies = [ [[package]] name = "zcash_address" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash?rev=09567fc280463f5797a77f1d764205be8ad8e64a#09567fc280463f5797a77f1d764205be8ad8e64a" +source = "git+https://github.com/adityapk00/librustzcash?rev=adf49f8b848a5ac85e1476354614eeae9880206a#adf49f8b848a5ac85e1476354614eeae9880206a" dependencies = [ "bech32", "bs58", @@ -3633,7 +3632,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.5.0" -source = "git+https://github.com/zcash/librustzcash?rev=09567fc280463f5797a77f1d764205be8ad8e64a#09567fc280463f5797a77f1d764205be8ad8e64a" +source = "git+https://github.com/adityapk00/librustzcash?rev=adf49f8b848a5ac85e1476354614eeae9880206a#adf49f8b848a5ac85e1476354614eeae9880206a" dependencies = [ "base64", "bech32", @@ -3660,7 +3659,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash?rev=09567fc280463f5797a77f1d764205be8ad8e64a#09567fc280463f5797a77f1d764205be8ad8e64a" +source = "git+https://github.com/adityapk00/librustzcash?rev=adf49f8b848a5ac85e1476354614eeae9880206a#adf49f8b848a5ac85e1476354614eeae9880206a" dependencies = [ "byteorder", "nonempty", @@ -3669,7 +3668,7 @@ dependencies = [ [[package]] name = "zcash_note_encryption" version = "0.1.0" -source = "git+https://github.com/zcash/librustzcash?rev=09567fc280463f5797a77f1d764205be8ad8e64a#09567fc280463f5797a77f1d764205be8ad8e64a" +source = "git+https://github.com/adityapk00/librustzcash?rev=adf49f8b848a5ac85e1476354614eeae9880206a#adf49f8b848a5ac85e1476354614eeae9880206a" dependencies = [ "chacha20", "chacha20poly1305", @@ -3680,7 +3679,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.7.0" -source = "git+https://github.com/zcash/librustzcash?rev=09567fc280463f5797a77f1d764205be8ad8e64a#09567fc280463f5797a77f1d764205be8ad8e64a" +source = "git+https://github.com/adityapk00/librustzcash?rev=adf49f8b848a5ac85e1476354614eeae9880206a#adf49f8b848a5ac85e1476354614eeae9880206a" dependencies = [ "aes", "bip0039", @@ -3709,6 +3708,7 @@ dependencies = [ "secp256k1", "sha2 0.9.9", "subtle 2.4.1", + "zcash_address", "zcash_encoding", "zcash_note_encryption", ] @@ -3716,8 +3716,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98bf5f6af051dd929263f279b21b9c04c1f30116c70f3c190de2566677f245ef" +source = "git+https://github.com/adityapk00/librustzcash?rev=adf49f8b848a5ac85e1476354614eeae9880206a#adf49f8b848a5ac85e1476354614eeae9880206a" dependencies = [ "bellman", "blake2b_simd", @@ -3736,7 +3735,7 @@ dependencies = [ [[package]] name = "zecwallet-cli" -version = "1.8.0-beta1" +version = "1.8.0-beta2" dependencies = [ "byteorder", "clap", @@ -3766,6 +3765,7 @@ dependencies = [ "group", "hex 0.3.2", "http", + "incrementalmerkletree", "json", "jubjub", "lazy_static", @@ -3818,8 +3818,3 @@ dependencies = [ "syn", "synstructure", ] - -[[patch.unused]] -name = "zcash_proofs" -version = "0.7.0" -source = "git+https://github.com/zcash/librustzcash?rev=09567fc280463f5797a77f1d764205be8ad8e64a#09567fc280463f5797a77f1d764205be8ad8e64a" diff --git a/Cargo.toml b/Cargo.toml index c99d9643..de613dc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,9 +8,18 @@ members = [ debug = false [patch.crates-io] -zcash_address = { git = "https://github.com/zcash/librustzcash", rev = "09567fc280463f5797a77f1d764205be8ad8e64a"} -zcash_primitives = { git = "https://github.com/zcash/librustzcash", rev = "09567fc280463f5797a77f1d764205be8ad8e64a"} -zcash_client_backend = { git = "https://github.com/zcash/librustzcash", rev = "09567fc280463f5797a77f1d764205be8ad8e64a"} -zcash_note_encryption = { git = "https://github.com/zcash/librustzcash", rev = "09567fc280463f5797a77f1d764205be8ad8e64a"} -zcash_encoding = { git = "https://github.com/zcash/librustzcash", rev = "09567fc280463f5797a77f1d764205be8ad8e64a"} -zcash_proofs = { git = "https://github.com/zcash/librustzcash", rev = "09567fc280463f5797a77f1d764205be8ad8e64a"} \ No newline at end of file +zcash_address = { git = "https://github.com/adityapk00/librustzcash", rev = "adf49f8b848a5ac85e1476354614eeae9880206a"} +zcash_primitives = { git = "https://github.com/adityapk00/librustzcash", rev = "adf49f8b848a5ac85e1476354614eeae9880206a"} +zcash_client_backend = { git = "https://github.com/adityapk00/librustzcash", rev = "adf49f8b848a5ac85e1476354614eeae9880206a"} +zcash_note_encryption = { git = "https://github.com/adityapk00/librustzcash", rev = "adf49f8b848a5ac85e1476354614eeae9880206a"} +zcash_encoding = { git = "https://github.com/adityapk00/librustzcash", rev = "adf49f8b848a5ac85e1476354614eeae9880206a"} +zcash_proofs = { git = "https://github.com/adityapk00/librustzcash", rev = "adf49f8b848a5ac85e1476354614eeae9880206a"} +orchard = { git = "https://github.com/adityapk00/orchard", rev = "0a960a380f4e9c3472c9260f3df61cd5e50d51b0" } + +#zcash_address = { path = "../librustzcash/components/zcash_address/" } +#zcash_primitives = { path = "../librustzcash/zcash_primitives/" } +#zcash_client_backend = { path = "../librustzcash/zcash_client_backend" } +#zcash_note_encryption = { path = "../librustzcash/components/zcash_note_encryption" } +#zcash_encoding = { path = "../librustzcash/components/zcash_encoding/" } +#zcash_proofs = { path = "../librustzcash/zcash_proofs/" } +#orchard = { path = "../orchard/" } \ No newline at end of file diff --git a/cli/Cargo.toml b/cli/Cargo.toml index f2ce5994..59e84aca 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zecwallet-cli" -version = "1.8.0-beta1" +version = "1.8.0-beta2" edition = "2018" [dependencies] diff --git a/cli/src/version.rs b/cli/src/version.rs index 3d53afc2..7f4024bb 100644 --- a/cli/src/version.rs +++ b/cli/src/version.rs @@ -1 +1 @@ -pub const VERSION: &str = "1.8.0-beta1"; +pub const VERSION: &str = "1.8.0-beta2"; diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 4aab567b..de99e586 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -49,6 +49,8 @@ group = "0.12" rust-embed = { version = "6.3.0", features = ["debug-embed"] } orchard = "0.2.0" +incrementalmerkletree = "0.3" + zcash_address = "0.1.0" zcash_primitives = { version = "0.7.0", features = ["transparent-inputs"] } zcash_client_backend = "0.5.0" diff --git a/lib/src/blaze/block_witness_data.rs b/lib/src/blaze/block_witness_data.rs index 39f65a63..1c9ab9a2 100644 --- a/lib/src/blaze/block_witness_data.rs +++ b/lib/src/blaze/block_witness_data.rs @@ -1,3 +1,5 @@ +use crate::compact_formats::vec_to_array; +use crate::lightwallet::MAX_CHECKPOINTS; use crate::{ compact_formats::{CompactBlock, CompactTx, TreeState}, grpc_connector::GrpcConnector, @@ -8,11 +10,15 @@ use crate::{ lightwallet::{ data::{BlockData, WalletTx, WitnessCache}, wallet_txns::WalletTxns, + MERKLE_DEPTH, }, }; - use futures::{stream::FuturesOrdered, StreamExt}; use http::Uri; +use incrementalmerkletree::{bridgetree::BridgeTree, Tree}; +use log::info; +use orchard::{note::ExtractedNoteCommitment, tree::MerkleHashOrchard}; +use std::collections::HashMap; use std::{sync::Arc, time::Duration}; use tokio::{ sync::{ @@ -32,7 +38,7 @@ use zcash_primitives::{ use super::{fixed_size_buffer::FixedSizeBuffer, sync_status::SyncStatus}; pub struct BlockAndWitnessData { - // List of all blocks and their hashes/commitment trees. Stored from smallest block height to tallest block height + // List of all blocks and their hashes/commitment trees. blocks[0] is the tallest block height in this batch blocks: Arc>>, // List of existing blocks in the wallet. Used for reorgs @@ -48,6 +54,12 @@ pub struct BlockAndWitnessData { // Heighest verified tree verified_tree: Option, + // Orchard notes to track. block height -> tx_num -> output_num + orchard_note_positions: Arc>>>>, + + // Orchard witnesses + orchard_witnesses: Arc>>>, + // Link to the syncstatus where we can update progress sync_status: Arc>, @@ -62,6 +74,8 @@ impl BlockAndWitnessData { verification_list: Arc::new(RwLock::new(vec![])), batch_size: 1_000, verified_tree: None, + orchard_note_positions: Arc::new(RwLock::new(HashMap::new())), + orchard_witnesses: Arc::new(RwLock::new(None)), sync_status, sapling_activation_height: config.sapling_activation_height, } @@ -75,7 +89,12 @@ impl BlockAndWitnessData { s } - pub async fn setup_sync(&mut self, existing_blocks: Vec, verified_tree: Option) { + pub async fn setup_sync( + &mut self, + existing_blocks: Vec, + verified_tree: Option, + orchard_witnesses: Arc>>>, + ) { if !existing_blocks.is_empty() { if existing_blocks.first().unwrap().height < existing_blocks.last().unwrap().height { panic!("Blocks are in wrong order"); @@ -86,6 +105,8 @@ impl BlockAndWitnessData { self.blocks.write().await.clear(); + self.orchard_witnesses = orchard_witnesses; + self.existing_blocks.write().await.clear(); self.existing_blocks.write().await.extend(existing_blocks); } @@ -369,6 +390,110 @@ impl BlockAndWitnessData { return (h, tx); } + pub async fn track_orchard_note(&self, height: u64, tx_num: usize, output_num: u32) { + // Remember this note position + let mut block_map = self.orchard_note_positions.write().await; + + let txid_map = block_map.entry(height).or_default(); + let output_nums = txid_map.entry(tx_num).or_default(); + output_nums.push(output_num); + } + + pub async fn update_orchard_spends_and_witnesses(&self, wallet_txns: Arc>) { + // Go over all the blocks + if let Some(orchard_witnesses) = self.orchard_witnesses.write().await.as_mut() { + // Read Lock + let blocks = self.blocks.read().await; + if blocks.is_empty() { + return; + } + + let mut orchard_note_positions = self.orchard_note_positions.write().await; + + // The last 100 blocks need to be checkpointed + let mut last_hundred = (blocks[0].height as i64) - (MAX_CHECKPOINTS as i64); + if last_hundred < 0 { + last_hundred = 0; + } + + // List of all the wallet's nullifiers + let o_nullifiers = wallet_txns.read().await.get_unspent_o_nullifiers(); + + for i in (0..blocks.len()).rev() { + let cb = &blocks.get(i as usize).unwrap().cb(); + + for (tx_num, ctx) in cb.vtx.iter().enumerate() { + for (output_num, action) in ctx.actions.iter().enumerate() { + let output_num = output_num as u32; + orchard_witnesses.append(&MerkleHashOrchard::from_cmx( + &ExtractedNoteCommitment::from_bytes(vec_to_array(&action.cmx)).unwrap(), + )); + + // Check if this orchard note needs to be tracked + if let Some(block_map) = orchard_note_positions.get(&cb.height) { + if let Some(output_nums) = block_map.get(&tx_num) { + if output_nums.contains(&output_num) { + let pos = orchard_witnesses.witness(); + + // Update the wallet_tx with the note + wallet_txns + .write() + .await + .set_o_note_witness((cb.height, tx_num, output_num), pos); + info!("Witnessing note at position {:?}", pos.unwrap()); + } + } + } + + // Check if the nullifier in this action belongs to us, which means it has been spent + for (wallet_nullifier, value, source_txid) in o_nullifiers.iter() { + if orchard::note::Nullifier::from_bytes(vec_to_array(&action.nullifier)).unwrap() + == *wallet_nullifier + { + // This was our spend. + let txid = WalletTx::new_txid(&ctx.hash); + + info!("An orchard note from {} was spent in {}", source_txid, txid); + + // 1. Mark the note as spent + wallet_txns.write().await.mark_txid_o_nf_spent( + source_txid, + &wallet_nullifier, + &txid, + cb.height(), + ); + + // 2. Update the spent notes in the wallet + let maybe_position = wallet_txns.write().await.add_new_o_spent( + txid, + cb.height(), + false, + cb.time, + *wallet_nullifier, + *value, + *source_txid, + ); + + // 3. Remove the note from the incremental witness tree tracking. + if let Some(position) = maybe_position { + orchard_witnesses.remove_witness(position); + } + } + } + } + } + + // See if we need to checkpoint + if cb.height >= last_hundred as u64 { + orchard_witnesses.checkpoint(); + } + } + + orchard_witnesses.garbage_collect(); + orchard_note_positions.clear(); + } + } + async fn wait_for_first_block(&self) -> u64 { while self.blocks.read().await.is_empty() { yield_now().await; @@ -445,7 +570,7 @@ impl BlockAndWitnessData { let tree = if prev_height < self.sapling_activation_height { CommitmentTree::empty() } else { - let tree_state = GrpcConnector::get_sapling_tree(uri, prev_height).await?; + let tree_state = GrpcConnector::get_merkle_tree(uri, prev_height).await?; let sapling_tree = hex::decode(&tree_state.tree).unwrap(); // self.verification_list.write().await.push(tree_state); CommitmentTree::read(&sapling_tree[..]).map_err(|e| format!("{}", e))? @@ -619,7 +744,8 @@ mod test { let cb = FakeCompactBlock::new(1, BlockHash([0u8; 32])).into_cb(); let blks = vec![BlockData::new(cb)]; - nw.setup_sync(blks.clone(), None).await; + let orchard_witnesses = Arc::new(RwLock::new(None)); + nw.setup_sync(blks.clone(), None, orchard_witnesses).await; let finished_blks = nw.finish_get_blocks(1).await; assert_eq!(blks[0].hash(), finished_blks[0].hash()); @@ -634,7 +760,9 @@ mod test { ); let existing_blocks = FakeCompactBlockList::new(200).into_blockdatas(); - nw.setup_sync(existing_blocks.clone(), None).await; + + let orchard_witnesses = Arc::new(RwLock::new(None)); + nw.setup_sync(existing_blocks.clone(), None, orchard_witnesses).await; let finished_blks = nw.finish_get_blocks(100).await; assert_eq!(finished_blks.len(), 100); @@ -660,7 +788,9 @@ mod test { let sync_status = Arc::new(RwLock::new(SyncStatus::default())); let mut nw = BlockAndWitnessData::new(&config, sync_status); - nw.setup_sync(vec![], None).await; + + let orchard_witnesses = Arc::new(RwLock::new(None)); + nw.setup_sync(vec![], None, orchard_witnesses).await; let (reorg_tx, mut reorg_rx) = unbounded_channel(); @@ -708,7 +838,9 @@ mod test { let end_block = blocks.last().unwrap().height; let mut nw = BlockAndWitnessData::new_with_batchsize(&config, 25); - nw.setup_sync(existing_blocks, None).await; + + let orchard_witnesses = Arc::new(RwLock::new(None)); + nw.setup_sync(existing_blocks, None, orchard_witnesses).await; let (reorg_tx, mut reorg_rx) = unbounded_channel(); @@ -801,7 +933,9 @@ mod test { let sync_status = Arc::new(RwLock::new(SyncStatus::default())); let mut nw = BlockAndWitnessData::new(&config, sync_status); - nw.setup_sync(existing_blocks, None).await; + + let orchard_witnesses = Arc::new(RwLock::new(None)); + nw.setup_sync(existing_blocks, None, orchard_witnesses).await; let (reorg_tx, mut reorg_rx) = unbounded_channel(); diff --git a/lib/src/blaze/fetch_full_tx.rs b/lib/src/blaze/fetch_full_tx.rs index fbd12cdb..6874df6c 100644 --- a/lib/src/blaze/fetch_full_tx.rs +++ b/lib/src/blaze/fetch_full_tx.rs @@ -264,11 +264,11 @@ impl FetchFullTxns

{ // Step 3: Check if any of the nullifiers spent in this Tx are ours. We only need this for unconfirmed txns, // because for txns in the block, we will check the nullifiers from the blockdata if unconfirmed { - let unspent_nullifiers = wallet_txns.read().await.get_unspent_nullifiers(); + let unspent_nullifiers = wallet_txns.read().await.get_unspent_s_nullifiers(); if let Some(s_bundle) = tx.sapling_bundle() { for s in s_bundle.shielded_spends.iter() { if let Some((nf, value, txid)) = unspent_nullifiers.iter().find(|(nf, _, _)| *nf == s.nullifier) { - wallet_txns.write().await.add_new_spent( + wallet_txns.write().await.add_new_s_spent( tx.txid(), height, unconfirmed, diff --git a/lib/src/blaze/syncdata.rs b/lib/src/blaze/syncdata.rs index 1280d931..31453aee 100644 --- a/lib/src/blaze/syncdata.rs +++ b/lib/src/blaze/syncdata.rs @@ -1,12 +1,14 @@ use std::sync::Arc; use http::Uri; +use incrementalmerkletree::bridgetree::BridgeTree; +use orchard::tree::MerkleHashOrchard; use tokio::sync::RwLock; use zcash_primitives::consensus; use super::{block_witness_data::BlockAndWitnessData, sync_status::SyncStatus}; use crate::compact_formats::TreeState; -use crate::lightwallet::WalletOptions; +use crate::lightwallet::{WalletOptions, MERKLE_DEPTH}; use crate::{lightclient::lightclient_config::LightClientConfig, lightwallet::data::BlockData}; pub struct BlazeSyncData { @@ -39,6 +41,7 @@ impl BlazeSyncData { batch_num: usize, existing_blocks: Vec, verified_tree: Option, + orchard_witnesses: Arc>>>, wallet_options: WalletOptions, ) { if start_block < end_block { @@ -53,7 +56,9 @@ impl BlazeSyncData { self.wallet_options = wallet_options; - self.block_data.setup_sync(existing_blocks, verified_tree).await; + self.block_data + .setup_sync(existing_blocks, verified_tree, orchard_witnesses) + .await; } // Finish up the sync diff --git a/lib/src/blaze/test_utils.rs b/lib/src/blaze/test_utils.rs index 665c9474..31c76718 100644 --- a/lib/src/blaze/test_utils.rs +++ b/lib/src/blaze/test_utils.rs @@ -10,6 +10,7 @@ use crate::{ }; use ff::{Field, PrimeField}; use group::GroupEncoding; +use orchard::tree::MerkleHashOrchard; use prost::Message; use rand::{rngs::OsRng, RngCore}; use secp256k1::PublicKey; @@ -53,6 +54,12 @@ pub fn tree_to_string(tree: &CommitmentTree) -> String { hex::encode(b1) } +pub fn orchardtree_to_string(tree: &CommitmentTree) -> String { + let mut b1 = vec![]; + tree.write(&mut b1).unwrap(); + hex::encode(b1) +} + pub fn incw_to_string(inc_witness: &IncrementalWitness) -> String { let mut b1 = vec![]; inc_witness.write(&mut b1).unwrap(); diff --git a/lib/src/blaze/trial_decryptions.rs b/lib/src/blaze/trial_decryptions.rs index 3e913da4..7eeb7b54 100644 --- a/lib/src/blaze/trial_decryptions.rs +++ b/lib/src/blaze/trial_decryptions.rs @@ -5,7 +5,6 @@ use crate::{ use futures::{stream::FuturesUnordered, StreamExt}; use log::info; use orchard::{keys::IncomingViewingKey, note_encryption::OrchardDomain}; -use prost::Message; use std::convert::TryFrom; use zcash_note_encryption::batch::try_compact_note_decryption; @@ -146,8 +145,6 @@ impl TrialDecryptions

{ { // Orchard - let actions_total = ctx.actions.len(); - let orchard_actions = ctx .actions .into_iter() @@ -174,8 +171,8 @@ impl TrialDecryptions

{ // } let decrypts = try_compact_note_decryption(o_ivks.as_ref(), orchard_actions.as_ref()); - for (dec_num, maybe_decrypted) in decrypts.into_iter().enumerate() { - if let Some((note, to)) = maybe_decrypted { + for (output_num, maybe_decrypted) in decrypts.into_iter().enumerate() { + if let Some(((note, _to), ivk_num)) = maybe_decrypted { // println!( // "An orchard note was decrypted! {}:{:?}", // note.value().inner(), @@ -183,17 +180,7 @@ impl TrialDecryptions

{ // ); wallet_tx = true; - let mut action_bytes = vec![]; - orchard_actions - .get(dec_num) - .unwrap() - .1 - .encode(&mut action_bytes) - .unwrap(); - let ctx_hash = ctx_hash.clone(); - let output_num = dec_num % actions_total; - let ivk_num = dec_num / actions_total; let keys = keys.read().await; let detected_txid_sender = detected_txid_sender.clone(); @@ -201,14 +188,22 @@ impl TrialDecryptions

{ let fvk = keys.okeys[ivk_num].fvk(); let have_spending_key = keys.have_orchard_spending_key(fvk); + // Tell the orchard witness tree to track this note. + bsync_data + .read() + .await + .block_data + .track_orchard_note(cb.height, tx_num, output_num as u32) + .await; + let txid = WalletTx::new_txid(&ctx_hash); wallet_txns.write().await.add_new_orchard_note( txid, height, false, timestamp, - action_bytes, note, + (height.into(), tx_num, output_num as u32), fvk, have_spending_key, ); @@ -224,7 +219,7 @@ impl TrialDecryptions

{ { // Sapling let outputs_total = ctx.outputs.len(); - + // if outputs_total < 100 { let outputs = ctx .outputs .into_iter() @@ -235,12 +230,11 @@ impl TrialDecryptions

{ let decrypts = try_compact_note_decryption(s_ivks.as_ref(), outputs.as_ref()); for (dec_num, maybe_decrypted) in decrypts.into_iter().enumerate() { - if let Some((note, to)) = maybe_decrypted { + if let Some(((note, to), ivk_num)) = maybe_decrypted { wallet_tx = true; let ctx_hash = ctx_hash.clone(); let output_num = dec_num % outputs_total; - let ivk_num = dec_num / outputs_total; let keys = keys.clone(); let bsync_data = bsync_data.clone(); @@ -263,7 +257,7 @@ impl TrialDecryptions

{ .await?; let txid = WalletTx::new_txid(&ctx_hash); - let nullifier = note.nf(&extfvk.fvk.vk, witness.position() as u64); + let nullifier = note.nf(&extfvk.fvk.vk.nk, witness.position() as u64); wallet_txns.write().await.add_new_sapling_note( txid.clone(), @@ -288,6 +282,7 @@ impl TrialDecryptions

{ })); } } + // } } // Check option to see if we are fetching all txns. diff --git a/lib/src/blaze/update_notes.rs b/lib/src/blaze/update_notes.rs index f4d0cae0..d2a0b4ba 100644 --- a/lib/src/blaze/update_notes.rs +++ b/lib/src/blaze/update_notes.rs @@ -74,7 +74,7 @@ impl UpdateNotes { wallet_txns .write() .await - .set_note_witnesses(&txid, &nullifier, witnesses); + .set_s_note_witnesses(&txid, &nullifier, witnesses); } } @@ -151,14 +151,15 @@ impl UpdateNotes { let spent_at_height = BlockHeight::from_u32(spent_height as u32); // Mark this note as being spent - let value = - wallet_txns - .write() - .await - .mark_txid_nf_spent(txid, &nf, &spent_txid, spent_at_height); + let value = wallet_txns.write().await.mark_txid_s_nf_spent( + &txid, + &nf, + &spent_txid, + spent_at_height, + ); // Record the future tx, the one that has spent the nullifiers recieved in this Tx in the wallet - wallet_txns.write().await.add_new_spent( + wallet_txns.write().await.add_new_s_spent( spent_txid, spent_at_height, false, diff --git a/lib/src/compact_formats.rs b/lib/src/compact_formats.rs index 8c781968..57210973 100644 --- a/lib/src/compact_formats.rs +++ b/lib/src/compact_formats.rs @@ -115,7 +115,7 @@ impl ShieldedOutput, 52_usize> for CompactSaplin } } -fn vec_to_array<'a, T, const N: usize>(vec: &'a Vec) -> &'a [T; N] { +pub fn vec_to_array<'a, T, const N: usize>(vec: &'a Vec) -> &'a [T; N] { <&[T; N]>::try_from(&vec[..]).unwrap() } diff --git a/lib/src/grpc_connector.rs b/lib/src/grpc_connector.rs index 958cda21..f4af73db 100644 --- a/lib/src/grpc_connector.rs +++ b/lib/src/grpc_connector.rs @@ -1,12 +1,12 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::ServerCert; use crate::compact_formats::compact_tx_streamer_client::CompactTxStreamerClient; use crate::compact_formats::{ BlockId, BlockRange, ChainSpec, CompactBlock, Empty, LightdInfo, PriceRequest, PriceResponse, RawTransaction, TransparentAddressBlockFilter, TreeState, TxFilter, }; +use crate::ServerCert; use futures::stream::FuturesUnordered; use futures::StreamExt; use log::warn; @@ -15,7 +15,7 @@ use tokio::sync::mpsc::{Sender, UnboundedReceiver}; use tokio::sync::oneshot; use tokio::task::JoinHandle; -use tonic::transport::{ClientTlsConfig, Certificate}; +use tonic::transport::{Certificate, ClientTlsConfig}; use tonic::{ transport::{Channel, Error}, Request, @@ -23,7 +23,6 @@ use tonic::{ use zcash_primitives::consensus::{self, BlockHeight, BranchId}; use zcash_primitives::transaction::{Transaction, TxId}; - #[derive(Clone)] pub struct GrpcConnector { uri: http::Uri, @@ -72,7 +71,7 @@ impl GrpcConnector { let uri = uri.clone(); while let Some((height, result_tx)) = rx.recv().await { result_tx - .send(Self::get_sapling_tree(uri.clone(), height).await) + .send(Self::get_merkle_tree(uri.clone(), height).await) .unwrap() } }); @@ -342,7 +341,7 @@ impl GrpcConnector { Ok(()) } - pub async fn get_sapling_tree(uri: http::Uri, height: u64) -> Result { + pub async fn get_merkle_tree(uri: http::Uri, height: u64) -> Result { let client = Arc::new(GrpcConnector::new(uri)); let mut client = client .get_client() diff --git a/lib/src/lightclient.rs b/lib/src/lightclient.rs index 57b205b1..612a6cb4 100644 --- a/lib/src/lightclient.rs +++ b/lib/src/lightclient.rs @@ -8,11 +8,13 @@ use crate::{ compact_formats::RawTransaction, grpc_connector::GrpcConnector, lightclient::lightclient_config::MAX_REORG, - lightwallet::{self, data::WalletTx, message::Message, now, LightWallet}, + lightwallet::{self, data::WalletTx, message::Message, now, LightWallet, MAX_CHECKPOINTS, MERKLE_DEPTH}, }; use futures::{stream::FuturesUnordered, StreamExt}; +use incrementalmerkletree::bridgetree::BridgeTree; use json::{array, object, JsonValue}; use log::{error, info, warn}; +use orchard::tree::MerkleHashOrchard; use std::{ cmp, collections::HashSet, @@ -34,6 +36,7 @@ use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, BranchId}, memo::{Memo, MemoBytes}, + merkle_tree::CommitmentTree, transaction::{components::amount::DEFAULT_FEE, Transaction, TxId}, }; use zcash_proofs::prover::LocalTxProver; @@ -448,6 +451,15 @@ impl LightClient

{ } pub async fn do_balance(&self) -> JsonValue { + // Collect UA addresses + let mut ua_addresses = vec![]; + for uaddress in self.wallet.keys().read().await.get_all_uaddresses() { + ua_addresses.push(object! { + "address" => uaddress.clone(), + "balance" => self.wallet.uabalance(Some(uaddress.clone())).await, + }); + } + // Collect z addresses let mut z_addresses = vec![]; for zaddress in self.wallet.keys().read().await.get_all_zaddresses() { @@ -473,11 +485,13 @@ impl LightClient

{ } object! { + "uabalance" => self.wallet.uabalance(None).await, "zbalance" => self.wallet.zbalance(None).await, "verified_zbalance" => self.wallet.verified_zbalance(None).await, "spendable_zbalance" => self.wallet.spendable_zbalance(None).await, "unverified_zbalance" => self.wallet.unverified_zbalance(None).await, "tbalance" => self.wallet.tbalance(None).await, + "ua_addresses" => ua_addresses, "z_addresses" => z_addresses, "t_addresses" => t_addresses, } @@ -647,25 +661,25 @@ impl LightClient

{ let anchor_height = BlockHeight::from_u32(self.wallet.get_anchor_height().await); { - // First, collect all extfvk's that are spendable (i.e., we have the private key) - let spendable_address: HashSet = self + // Collect orchard notes + // First, collect all UA's that are spendable (i.e., we have the private key) + let spendable_oaddress: HashSet = self .wallet .keys() .read() .await - .get_all_spendable_zaddresses() + .get_all_spendable_oaddresses() .into_iter() .collect(); - // Collect orchard notes self.wallet.txns.read().await.current.iter() .flat_map( |(txid, wtx)| { - let spendable_address = spendable_address.clone(); + let spendable_address = spendable_oaddress.clone(); wtx.o_notes.iter().filter_map(move |nd| if !all_notes && nd.spent.is_some() { None } else { - let address = LightWallet::

::orchard_ua_address (&self.config, &nd.note_address); + let address = LightWallet::

::orchard_ua_address(&self.config, &nd.note.recipient()); let spendable = spendable_address.contains(&address) && wtx.block <= anchor_height && nd.spent.is_none() && nd.unconfirmed_spent.is_none(); @@ -674,7 +688,7 @@ impl LightClient

{ "created_in_block" => created_block, "datetime" => wtx.datetime, "created_in_txid" => format!("{}", txid), - "value" => nd.note_value, + "value" => nd.note.value().inner(), "unconfirmed" => wtx.unconfirmed, "is_change" => nd.is_change, "address" => address, @@ -697,9 +711,19 @@ impl LightClient

{ }); // Collect Sapling notes + // First, collect all extfvk's that are spendable (i.e., we have the private key) + let spendable_zaddress: HashSet = self + .wallet + .keys() + .read() + .await + .get_all_spendable_zaddresses() + .into_iter() + .collect(); + self.wallet.txns.read().await.current.iter() .flat_map( |(txid, wtx)| { - let spendable_address = spendable_address.clone(); + let spendable_address = spendable_zaddress.clone(); wtx.s_notes.iter().filter_map(move |nd| if !all_notes && nd.spent.is_some() { None @@ -933,9 +957,9 @@ impl LightClient

{ "datetime" => v.datetime, "position" => i, "txid" => format!("{}", v.txid), - "amount" => nd.note_value, + "amount" => nd.note.value().inner(), "zec_price" => v.zec_price.map(|p| (p * 100.0).round() / 100.0), - "address" => LightWallet::

::orchard_ua_address(&self.config, &nd.note_address), + "address" => LightWallet::

::orchard_ua_address(&self.config, &nd.note.recipient()), "memo" => LightWallet::

::memo_str(nd.memo.clone()) }; @@ -1429,6 +1453,43 @@ impl LightClient

{ let start_block = latest_block; let end_block = last_scanned_height + 1; + // Make sure that the wallet has an orchard tree first + { + let mut orchard_witnesses = self.wallet.orchard_witnesses.write().await; + + if orchard_witnesses.is_none() { + let tree_state = GrpcConnector::get_merkle_tree(uri.clone(), last_scanned_height).await?; + println!( + "Attempting to get orchard tree from block {} with str `{}`", + last_scanned_height, tree_state.orchard_tree + ); + + // Populate the orchard witnesses from the previous block's frontier + let orchard_tree = hex::decode(tree_state.orchard_tree).unwrap(); + // let o: io::Result> = + // LightWallet::

::read_nonempty_frontier_v1(&orchard_tree[..]); + // if let Ok(frontier) = o { + // let witnesses = BridgeTree::<_, MERKLE_DEPTH>::from_frontier(100, frontier); + // *orchard_witnesses = Some(witnesses); + // } + let witnesses = if orchard_tree.len() > 0 { + let tree = + CommitmentTree::::read(&orchard_tree[..]).map_err(|e| format!("{}", e))?; + if let Some(frontier) = tree.to_frontier::().value() { + println!("Creating orchard tree from frontier"); + BridgeTree::<_, MERKLE_DEPTH>::from_frontier(MAX_CHECKPOINTS, frontier.clone()) + } else { + println!("Creating empty tree"); + BridgeTree::<_, MERKLE_DEPTH>::new(MAX_CHECKPOINTS) + } + } else { + println!("LightwalletD returned empty orchard tree, creating empty tree"); + BridgeTree::<_, MERKLE_DEPTH>::new(MAX_CHECKPOINTS) + }; + *orchard_witnesses = Some(witnesses); + } + } + // Before we start, we need to do a few things // 1. Pre-populate the last 100 blocks, in case of reorgs bsync_data @@ -1440,6 +1501,7 @@ impl LightClient

{ batch_num, self.wallet.get_blocks().await, self.wallet.verified_tree.read().await.clone(), + self.wallet.orchard_witnesses.clone(), *self.wallet.wallet_options.read().await, ) .await; @@ -1550,11 +1612,19 @@ impl LightClient

{ info!("Sync finished, doing post-processing"); // Post sync, we have to do a bunch of stuff - // 1. Get the last 100 blocks and store it into the wallet, needed for future re-orgs + // 1. Update the Orchard witnesses. This calculates the positions of all the notes found in this batch. + bsync_data + .read() + .await + .block_data + .update_orchard_spends_and_witnesses(self.wallet.txns.clone()) + .await; + + // 2. Get the last 100 blocks and store it into the wallet, needed for future re-orgs let blocks = bsync_data.read().await.block_data.finish_get_blocks(MAX_REORG).await; self.wallet.set_blocks(blocks).await; - // 2. If sync was successfull, also try to get historical prices + // 3. If sync was successfull, also try to get historical prices self.update_historical_prices().await; // 4. Remove the witnesses for spent notes more than 100 blocks old, since now there diff --git a/lib/src/lightclient/lightclient_config.rs b/lib/src/lightclient/lightclient_config.rs index c4bdfae1..39dd1627 100644 --- a/lib/src/lightclient/lightclient_config.rs +++ b/lib/src/lightclient/lightclient_config.rs @@ -75,6 +75,10 @@ impl Parameters for UnitTestNetwork { fn b58_script_address_prefix(&self) -> [u8; 2] { constants::mainnet::B58_SCRIPT_ADDRESS_PREFIX } + + fn address_network(&self) -> Option { + Some(zcash_address::Network::Main) + } } pub const UNITTEST_NETWORK: UnitTestNetwork = UnitTestNetwork; @@ -298,7 +302,7 @@ impl LightClientConfig

{ } info!("Getting sapling tree from LightwalletD at height {}", height); - match GrpcConnector::get_sapling_tree(self.server.clone(), height).await { + match GrpcConnector::get_merkle_tree(self.server.clone(), height).await { Ok(tree_state) => { let hash = tree_state.hash.clone(); let tree = tree_state.tree.clone(); diff --git a/lib/src/lightclient/test_server.rs b/lib/src/lightclient/test_server.rs index bd130720..d6c8cbbb 100644 --- a/lib/src/lightclient/test_server.rs +++ b/lib/src/lightclient/test_server.rs @@ -1,4 +1,4 @@ -use crate::blaze::test_utils::{tree_to_string, FakeCompactBlockList}; +use crate::blaze::test_utils::{orchardtree_to_string, tree_to_string, FakeCompactBlockList}; use crate::compact_formats::compact_tx_streamer_server::CompactTxStreamer; use crate::compact_formats::compact_tx_streamer_server::CompactTxStreamerServer; use crate::compact_formats::{ @@ -9,6 +9,7 @@ use crate::compact_formats::{ use crate::lightwallet::data::WalletTx; use crate::lightwallet::now; use futures::{FutureExt, Stream}; +use orchard::tree::MerkleHashOrchard; use rand::rngs::OsRng; use rand::Rng; use std::cmp; @@ -438,20 +439,23 @@ impl CompactTxStreamer for Tes }); let mut ts = TreeState::default(); - ts.hash = BlockHash::from_slice( - &self - .data - .read() - .await - .blocks - .iter() - .find(|cb| cb.height == block.height) - .expect(format!("Couldn't find block {}", block.height).as_str()) - .hash[..], - ) - .to_string(); + let hash = if let Some(b) = self + .data + .read() + .await + .blocks + .iter() + .find(|cb| cb.height == block.height) + { + b.hash.clone() + } else { + [0u8; 32].to_vec() + }; + + ts.hash = BlockHash::from_slice(&hash[..]).to_string(); ts.height = block.height; ts.tree = tree_to_string(&tree); + ts.orchard_tree = orchardtree_to_string(&CommitmentTree::::empty()); Ok(Response::new(ts)) } diff --git a/lib/src/lightclient/tests.rs b/lib/src/lightclient/tests.rs index 9294f28f..04577683 100644 --- a/lib/src/lightclient/tests.rs +++ b/lib/src/lightclient/tests.rs @@ -576,7 +576,7 @@ async fn z_to_z_scan_together() { tree }); let witness = IncrementalWitness::from_tree(&tree); - let nf = note.nf(&extfvk1.fvk.vk, witness.position() as u64); + let nf = note.nf(&extfvk1.fvk.vk.nk, witness.position() as u64); let pa = if let Some(RecipientAddress::Shielded(pa)) = RecipientAddress::decode(&config.get_params(), EXT_ZADDR) { pa diff --git a/lib/src/lightwallet.rs b/lib/src/lightwallet.rs index bd6912d3..ef9911b6 100644 --- a/lib/src/lightwallet.rs +++ b/lib/src/lightwallet.rs @@ -5,19 +5,39 @@ use crate::{ blaze::fetch_full_tx::FetchFullTxns, lightclient::lightclient_config::LightClientConfig, lightwallet::{ - data::SpendableNote, + data::SpendableSaplingNote, walletzkey::{WalletZKey, WalletZKeyType}, }, }; +use incrementalmerkletree::{bridgetree::BridgeTree, Position, Tree}; +use std::collections::BTreeMap; +use std::convert::TryInto; +use std::io; + +use zcash_encoding::{Optional, Vector}; +use zcash_primitives::{ + consensus::BlockHeight, + merkle_tree::incremental::{read_position, write_position}, + transaction::components::Amount, +}; + +use orchard::{ + tree::{MerkleHashOrchard, MerklePath}, + Address, +}; + use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use futures::Future; +use incrementalmerkletree::bridgetree::Checkpoint; +use incrementalmerkletree::Hashable; use log::{error, info, warn}; -use orchard::Address; + +use orchard::Anchor; use std::sync::mpsc; use std::{ cmp, collections::HashMap, - io::{self, Error, ErrorKind, Read, Write}, + io::{Error, ErrorKind, Read, Write}, sync::{atomic::AtomicU64, Arc}, time::SystemTime, }; @@ -28,21 +48,23 @@ use zcash_client_backend::{ address, encoding::{decode_extended_full_viewing_key, decode_extended_spending_key, encode_payment_address}, }; -use zcash_encoding::{Optional, Vector}; + +use zcash_primitives::consensus; use zcash_primitives::memo::MemoBytes; +use zcash_primitives::merkle_tree::incremental::{read_bridge, read_leu64_usize, write_bridge, write_usize_leu64}; +use zcash_primitives::merkle_tree::HashSer; use zcash_primitives::sapling::prover::TxProver; -use zcash_primitives::{consensus, transaction}; use zcash_primitives::{ - consensus::BlockHeight, legacy::Script, memo::Memo, transaction::{ builder::Builder, - components::{amount::DEFAULT_FEE, Amount, OutPoint, TxOut}, + components::{amount::DEFAULT_FEE, OutPoint, TxOut}, }, zip32::ExtendedFullViewingKey, }; +use self::data::SpendableOrchardNote; use self::{ data::{BlockData, SaplingNoteData, Utxo, WalletZecPriceInfo}, keys::Keys, @@ -60,6 +82,9 @@ mod walletokey; pub(crate) mod wallettkey; mod walletzkey; +pub const MERKLE_DEPTH: u8 = 32; +pub const MAX_CHECKPOINTS: usize = 100; + pub fn now() -> u64 { SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) @@ -170,6 +195,9 @@ pub struct LightWallet

{ // Heighest verified block pub(crate) verified_tree: Arc>>, + // The Orchard incremental tree + pub(crate) orchard_witnesses: Arc>>>, + // Progress of an outgoing tx send_progress: Arc>, @@ -179,7 +207,7 @@ pub struct LightWallet

{ impl LightWallet

{ pub fn serialized_version() -> u64 { - return 24; + return 25; } pub fn new( @@ -198,6 +226,7 @@ impl LightWallet

{ blocks: Arc::new(RwLock::new(vec![])), wallet_options: Arc::new(RwLock::new(WalletOptions::default())), config, + orchard_witnesses: Arc::new(RwLock::new(None)), birthday: AtomicU64::new(height), verified_tree: Arc::new(RwLock::new(None)), send_progress: Arc::new(RwLock::new(SendProgress::new(0))), @@ -205,6 +234,72 @@ impl LightWallet

{ }) } + pub fn read_tree(mut reader: R) -> io::Result> { + let _version = reader.read_u64::()?; + + let prior_bridges = Vector::read(&mut reader, |r| read_bridge(r))?; + let current_bridge = Optional::read(&mut reader, |r| read_bridge(r))?; + let saved: BTreeMap = Vector::read_collected(&mut reader, |mut r| { + Ok((read_position(&mut r)?, read_leu64_usize(&mut r)?)) + })?; + + let checkpoints = Vector::read_collected(&mut reader, |r| Self::read_checkpoint_v2(r))?; + let max_checkpoints = read_leu64_usize(&mut reader)?; + + BridgeTree::from_parts(prior_bridges, current_bridge, saved, checkpoints, max_checkpoints).map_err(|err| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "Consistency violation found when attempting to deserialize Merkle tree: {:?}", + err + ), + ) + }) + } + + fn write_tree( + mut writer: W, + tree: &BridgeTree, + ) -> io::Result<()> { + writer.write_u64::(Self::serialized_version())?; + + Vector::write(&mut writer, tree.prior_bridges(), |mut w, b| write_bridge(&mut w, b))?; + Optional::write(&mut writer, tree.current_bridge().as_ref(), |mut w, b| { + write_bridge(&mut w, b) + })?; + Vector::write_sized(&mut writer, tree.witnessed_indices().iter(), |mut w, (pos, i)| { + write_position(&mut w, *pos)?; + write_usize_leu64(&mut w, *i) + })?; + Vector::write(&mut writer, tree.checkpoints(), |w, c| Self::write_checkpoint_v2(w, c))?; + write_usize_leu64(&mut writer, tree.max_checkpoints())?; + + Ok(()) + } + + pub fn write_checkpoint_v2(mut writer: W, checkpoint: &Checkpoint) -> io::Result<()> { + write_usize_leu64(&mut writer, checkpoint.bridges_len())?; + writer.write_u8(if checkpoint.is_witnessed() { 1 } else { 0 })?; + Vector::write_sized(&mut writer, checkpoint.witnessed().iter(), |w, p| write_position(w, *p))?; + Vector::write_sized(&mut writer, checkpoint.forgotten().iter(), |mut w, (pos, idx)| { + write_position(&mut w, *pos)?; + write_usize_leu64(&mut w, *idx) + })?; + + Ok(()) + } + + pub fn read_checkpoint_v2(mut reader: R) -> io::Result { + Ok(Checkpoint::from_parts( + read_leu64_usize(&mut reader)?, + reader.read_u8()? == 1, + Vector::read_collected(&mut reader, |r| read_position(r))?, + Vector::read_collected(&mut reader, |mut r| { + Ok((read_position(&mut r)?, read_leu64_usize(&mut r)?)) + })?, + )) + } + pub async fn read(mut reader: R, config: &LightClientConfig

) -> io::Result { let version = reader.read_u64::()?; if version > Self::serialized_version() { @@ -290,12 +385,20 @@ impl LightWallet

{ WalletZecPriceInfo::read(&mut reader)? }; + // Reach the orchard tree + let orchard_witnesses = if version <= 24 { + None + } else { + Optional::read(&mut reader, |r| Self::read_tree(r))? + }; + let mut lw = Self { keys: Arc::new(RwLock::new(keys)), txns: Arc::new(RwLock::new(txns)), blocks: Arc::new(RwLock::new(blocks)), config: config.clone(), wallet_options: Arc::new(RwLock::new(wallet_options)), + orchard_witnesses: Arc::new(RwLock::new(orchard_witnesses)), birthday: AtomicU64::new(birthday), verified_tree: Arc::new(RwLock::new(verified_tree)), send_progress: Arc::new(RwLock::new(SendProgress::new(0))), @@ -352,6 +455,11 @@ impl LightWallet

{ // Price info self.price.read().await.write(&mut writer)?; + // Write the Tree + Optional::write(&mut writer, self.orchard_witnesses.read().await.as_ref(), |w, o| { + Self::write_tree(w, o) + })?; + Ok(()) } @@ -683,6 +791,31 @@ impl LightWallet

{ } } + pub async fn uabalance(&self, addr: Option) -> u64 { + self.txns + .read() + .await + .current + .values() + .map(|tx| { + tx.o_notes + .iter() + .filter(|nd| match addr.as_ref() { + Some(a) => *a == LightWallet::

::orchard_ua_address(&self.config, &nd.note.recipient()), + None => true, + }) + .map(|nd| { + if nd.spent.is_none() && nd.unconfirmed_spent.is_none() { + nd.note.value().inner() + } else { + 0 + } + }) + .sum::() + }) + .sum::() + } + pub async fn zbalance(&self, addr: Option) -> u64 { self.txns .read() @@ -974,7 +1107,9 @@ impl LightWallet

{ target_amount: Amount, transparent_only: bool, shield_transparenent: bool, - ) -> (Vec, Vec, Amount) { + allow_sapling: bool, + allow_orchard: bool, + ) -> (Vec, Vec, Vec, Amount) { // First, if we are allowed to pick transparent value, pick them all let utxos = if transparent_only || shield_transparenent { self.get_utxos() @@ -994,60 +1129,147 @@ impl LightWallet

{ // If we are allowed only transparent funds or we've selected enough then return if transparent_only || transparent_value_selected >= target_amount { - return (vec![], utxos, transparent_value_selected); + return (vec![], vec![], utxos, transparent_value_selected); } - // Start collecting sapling funds at every allowed offset - for anchor_offset in &self.config.anchor_offset { + let mut orchard_value_selected = Amount::zero(); + let mut sapling_value_selected = Amount::zero(); + + let mut o_notes = vec![]; + + // Collect orchard notes + if allow_orchard { let keys = self.keys.read().await; + let owt = self.orchard_witnesses.read().await; + let orchard_witness_tree = owt.as_ref().unwrap(); + let mut candidate_notes = self .txns .read() .await .current .iter() - .flat_map(|(txid, tx)| tx.s_notes.iter().map(move |note| (*txid, note))) - .filter(|(_, note)| note.note.value > 0) + .flat_map(|(txid, tx)| tx.o_notes.iter().map(move |note| (*txid, note))) + .filter(|(_, note)| note.note.value().inner() > 0) .filter_map(|(txid, note)| { // Filter out notes that are already spent if note.spent.is_some() || note.unconfirmed_spent.is_some() { None } else { // Get the spending key for the selected fvk, if we have it - let extsk = keys.get_extsk_for_extfvk(¬e.extfvk); - SpendableNote::from(txid, note, *anchor_offset as usize, &extsk) + let maybe_sk = keys.get_orchard_sk_for_fvk(¬e.fvk); + if maybe_sk.is_none() || note.witness_position.is_none() { + None + } else { + let merkle_path = MerklePath::from_parts( + usize::from(note.witness_position.unwrap()) as u32, + orchard_witness_tree + .authentication_path( + note.witness_position.unwrap(), + &orchard_witness_tree.root(0).unwrap(), + ) + .unwrap() + .try_into() + .unwrap(), + ); + + Some(SpendableOrchardNote { + txid, + sk: maybe_sk.unwrap(), + note: note.note.clone(), + merkle_path, + }) + } } }) .collect::>(); - candidate_notes.sort_by(|a, b| b.note.value.cmp(&a.note.value)); + candidate_notes.sort_by(|a, b| b.note.value().inner().cmp(&a.note.value().inner())); // Select the minimum number of notes required to satisfy the target value - let notes = candidate_notes + o_notes = candidate_notes .into_iter() .scan(Amount::zero(), |running_total, spendable| { if *running_total >= (target_amount - transparent_value_selected).unwrap() { None } else { - *running_total += Amount::from_u64(spendable.note.value).unwrap(); + *running_total += Amount::from_u64(spendable.note.value().inner()).unwrap(); Some(spendable) } }) .collect::>(); - let sapling_value_selected = notes.iter().fold(Amount::zero(), |prev, sn| { - (prev + Amount::from_u64(sn.note.value).unwrap()).unwrap() + orchard_value_selected = o_notes.iter().fold(Amount::zero(), |prev, sn| { + (prev + Amount::from_u64(sn.note.value().inner()).unwrap()).unwrap() }); - if sapling_value_selected + transparent_value_selected >= Some(target_amount) { + if orchard_value_selected + transparent_value_selected >= Some(target_amount) { return ( - notes, + o_notes, + vec![], utxos, - (sapling_value_selected + transparent_value_selected).unwrap(), + (orchard_value_selected + transparent_value_selected).unwrap(), ); } } + // Start collecting sapling funds at every allowed offset + if allow_sapling { + for anchor_offset in &self.config.anchor_offset { + let keys = self.keys.read().await; + let mut candidate_notes = self + .txns + .read() + .await + .current + .iter() + .flat_map(|(txid, tx)| tx.s_notes.iter().map(move |note| (*txid, note))) + .filter(|(_, note)| note.note.value > 0) + .filter_map(|(txid, note)| { + // Filter out notes that are already spent + if note.spent.is_some() || note.unconfirmed_spent.is_some() { + None + } else { + // Get the spending key for the selected fvk, if we have it + let extsk = keys.get_extsk_for_extfvk(¬e.extfvk); + SpendableSaplingNote::from(txid, note, *anchor_offset as usize, &extsk) + } + }) + .collect::>(); + candidate_notes.sort_by(|a, b| b.note.value.cmp(&a.note.value)); + + // Select the minimum number of notes required to satisfy the target value + let s_notes = candidate_notes + .into_iter() + .scan(Amount::zero(), |running_total, spendable| { + if *running_total >= (target_amount - transparent_value_selected).unwrap() { + None + } else { + *running_total += Amount::from_u64(spendable.note.value).unwrap(); + Some(spendable) + } + }) + .collect::>(); + sapling_value_selected = s_notes.iter().fold(Amount::zero(), |prev, sn| { + (prev + Amount::from_u64(sn.note.value).unwrap()).unwrap() + }); + + if orchard_value_selected + sapling_value_selected + transparent_value_selected >= Some(target_amount) { + return ( + o_notes, + s_notes, + utxos, + (orchard_value_selected + sapling_value_selected + transparent_value_selected).unwrap(), + ); + } + } + } + // If we can't select enough, then we need to return empty handed - (vec![], vec![], Amount::zero()) + ( + vec![], + vec![], + vec![], + (orchard_value_selected + sapling_value_selected + transparent_value_selected).unwrap(), + ) } pub async fn send_to_address( @@ -1126,6 +1348,21 @@ impl LightWallet

{ }) .collect::)>, String>>()?; + // Calculate how much we're sending to each type of address + let (t_out, s_out, o_out) = recepients + .iter() + .map(|(to, value, _)| match to { + address::RecipientAddress::Unified(_) => (0, 0, value.into()), + address::RecipientAddress::Shielded(_) => (0, value.into(), 0), + address::RecipientAddress::Transparent(_) => (value.into(), 0, 0), + }) + .reduce(|(t, s, o), (t2, s2, o2)| (t + t2, s + s2, o + o2)) + .unwrap_or((0, 0, 0)); + + if s_out > 0 && o_out > 0 { + return Err("Can't send to Sapling and Orchard in same tx".to_string()); + } + // Select notes to cover the target value println!("{}: Selecting notes", now() - start_time); @@ -1136,15 +1373,18 @@ impl LightWallet

{ }; let (progress_notifier, progress_notifier_rx) = mpsc::channel(); - let mut builder = Builder::new(self.config.get_params().clone(), target_height); + + let orchard_anchor = Anchor::from(self.orchard_witnesses.read().await.as_ref().unwrap().root(0).unwrap()); + + let mut builder = Builder::new_with_orchard(self.config.get_params().clone(), target_height, orchard_anchor); builder.with_progress_notifier(progress_notifier); // Create a map from address -> sk for all taddrs, so we can spend from the // right address let address_to_sk = self.keys.read().await.get_taddr_to_sk_map(); - let (notes, utxos, selected_value) = self - .select_notes_and_utxos(target_amount.unwrap(), transparent_only, true) + let (o_notes, s_notes, utxos, selected_value) = self + .select_notes_and_utxos(target_amount.unwrap(), transparent_only, true, true, true) .await; if selected_value < target_amount.unwrap() { let e = format!( @@ -1157,9 +1397,10 @@ impl LightWallet

{ // Create the transaction println!( - "{}: Adding {} notes and {} utxos", + "{}: Adding {} o_notes {} s_notes and {} utxos", now() - start_time, - notes.len(), + o_notes.len(), + s_notes.len(), utxos.len() ); @@ -1188,14 +1429,24 @@ impl LightWallet

{ .collect::, _>>() .map_err(|e| format!("{:?}", e))?; - for selected in notes.iter() { + // Add Orchard notes + for selected in o_notes.iter() { + if let Err(e) = builder.add_orchard_spend(selected.sk, selected.note, selected.merkle_path.clone()) { + let e = format!("Error adding orchard note: {:?}", e); + error!("{}", e); + return Err(e); + } + } + + // Add Sapling notes + for selected in s_notes.iter() { if let Err(e) = builder.add_sapling_spend( selected.extsk.clone(), selected.diversifier, selected.note.clone(), selected.witness.path().unwrap(), ) { - let e = format!("Error adding note: {:?}", e); + let e = format!("Error adding sapling note: {:?}", e); error!("{}", e); return Err(e); } @@ -1204,7 +1455,7 @@ impl LightWallet

{ // If no Sapling notes were added, add the change address manually. That is, // send the change to our sapling address manually. Note that if a sapling note was spent, // the builder will automatically send change to that address - if notes.len() == 0 { + if t_out > 0 && s_notes.len() == 0 { builder.send_change_to( self.keys.read().await.zkeys[0].extfvk.fvk.ovk, self.keys.read().await.zkeys[0].zaddress.clone(), @@ -1212,8 +1463,13 @@ impl LightWallet

{ } // We'll use the first ovk to encrypt outgoing Txns - let ovk = self.keys.read().await.zkeys[0].extfvk.fvk.ovk; + let s_ovk = self.keys.read().await.zkeys[0].extfvk.fvk.ovk; + let o_ovk = self.keys.read().await.okeys[0] + .fvk() + .to_ovk(orchard::keys::Scope::External); + let mut total_z_recepients = 0u32; + let mut total_o_recepients = 0u32; for (to, value, memo) in recepients { // Compute memo if it exists let encoded_memo = match memo { @@ -1234,12 +1490,18 @@ impl LightWallet

{ println!("{}: Adding output", now() - start_time); if let Err(e) = match to { + address::RecipientAddress::Unified(to) => { + // TODO(orchard): Allow using the sapling or transparent parts of this unified address too. + let orchard_address = to.orchard().unwrap().clone(); + total_o_recepients += 1; + + builder.add_orchard_output(Some(o_ovk.clone()), orchard_address, value.into(), encoded_memo) + } address::RecipientAddress::Shielded(to) => { total_z_recepients += 1; - builder.add_sapling_output(Some(ovk), to.clone(), value, encoded_memo) + builder.add_sapling_output(Some(s_ovk), to.clone(), value, encoded_memo) } address::RecipientAddress::Transparent(to) => builder.add_transparent_output(&to, value), - address::RecipientAddress::Unified(_) => Err(transaction::builder::Error::NoChangeAddress), } { let e = format!("Error adding output: {:?}", e); error!("{}", e); @@ -1268,10 +1530,11 @@ impl LightWallet

{ }); { + // TODO(orchard): Orchard building progress let mut p = self.send_progress.write().await; p.is_send_in_progress = true; p.progress = 0; - p.total = notes.len() as u32 + total_z_recepients; + p.total = s_notes.len() as u32 + total_z_recepients + total_o_recepients; } println!("{}: Building transaction", now() - start_time); @@ -1303,9 +1566,26 @@ impl LightWallet

{ // Mark notes as spent. { - // Mark sapling notes as unconfirmed spent + // Mark sapling and orchard notes as unconfirmed spent let mut txs = self.txns.write().await; - for selected in notes { + for selected in o_notes { + let mut spent_note = txs + .current + .get_mut(&selected.txid) + .unwrap() + .o_notes + .iter_mut() + .find(|nd| { + nd.note.nullifier(&nd.fvk) + == selected + .note + .nullifier(&orchard::keys::FullViewingKey::from(&selected.sk)) + }) + .unwrap(); + spent_note.unconfirmed_spent = Some((tx.txid(), u32::from(target_height))); + } + + for selected in s_notes { let mut spent_note = txs .current .get_mut(&selected.txid) @@ -1406,7 +1686,7 @@ mod test { let amt = Amount::from_u64(10_000).unwrap(); // Reset the anchor offsets lc.wallet.config.anchor_offset = [9, 4, 2, 1, 0]; - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false, true, false).await; assert!(selected >= amt); assert_eq!(notes.len(), 1); assert_eq!(notes[0].note.value, value); @@ -1423,14 +1703,14 @@ mod test { // With min anchor_offset at 1, we can't select any notes lc.wallet.config.anchor_offset = [9, 4, 2, 1, 1]; - let (notes, utxos, _selected) = lc.wallet.select_notes_and_utxos(amt, false, false).await; + let (_, notes, utxos, _selected) = lc.wallet.select_notes_and_utxos(amt, false, false, true, false).await; assert_eq!(notes.len(), 0); assert_eq!(utxos.len(), 0); // Mine 1 block, then it should be selectable mine_random_blocks(&mut fcbl, &data, &lc, 1).await; - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false, true, false).await; assert!(selected >= amt); assert_eq!(notes.len(), 1); assert_eq!(notes[0].note.value, value); @@ -1448,7 +1728,7 @@ mod test { // Mine 15 blocks, then selecting the note should result in witness only 10 blocks deep mine_random_blocks(&mut fcbl, &data, &lc, 15).await; lc.wallet.config.anchor_offset = [9, 4, 2, 1, 1]; - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, true).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, true, true, false).await; assert!(selected >= amt); assert_eq!(notes.len(), 1); assert_eq!(notes[0].note.value, value); @@ -1465,7 +1745,7 @@ mod test { // Trying to select a large amount will fail let amt = Amount::from_u64(1_000_000).unwrap(); - let (notes, utxos, _selected) = lc.wallet.select_notes_and_utxos(amt, false, false).await; + let (_, notes, utxos, _selected) = lc.wallet.select_notes_and_utxos(amt, false, false, true, false).await; assert_eq!(notes.len(), 0); assert_eq!(utxos.len(), 0); @@ -1482,14 +1762,14 @@ mod test { // Trying to select a large amount will now succeed let amt = Amount::from_u64(value + tvalue - 10_000).unwrap(); - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, true).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, true, true, false).await; assert_eq!(selected, Amount::from_u64(value + tvalue).unwrap()); assert_eq!(notes.len(), 1); assert_eq!(utxos.len(), 1); // If we set transparent-only = true, only the utxo should be selected let amt = Amount::from_u64(tvalue - 10_000).unwrap(); - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, true, true).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, true, true, true, false).await; assert_eq!(selected, Amount::from_u64(tvalue).unwrap()); assert_eq!(notes.len(), 0); assert_eq!(utxos.len(), 1); @@ -1497,7 +1777,7 @@ mod test { // Set min confs to 5, so the sapling note will not be selected lc.wallet.config.anchor_offset = [9, 4, 4, 4, 4]; let amt = Amount::from_u64(tvalue - 10_000).unwrap(); - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, true).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, true, true, false).await; assert_eq!(selected, Amount::from_u64(tvalue).unwrap()); assert_eq!(notes.len(), 0); assert_eq!(utxos.len(), 1); @@ -1532,7 +1812,7 @@ mod test { let amt = Amount::from_u64(10_000).unwrap(); // Reset the anchor offsets lc.wallet.config.anchor_offset = [9, 4, 2, 1, 0]; - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false, true, false).await; assert!(selected >= amt); assert_eq!(notes.len(), 1); assert_eq!(notes[0].note.value, value1); @@ -1557,7 +1837,7 @@ mod test { // Now, try to select a small amount, it should prefer the older note let amt = Amount::from_u64(10_000).unwrap(); - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false, true, false).await; assert!(selected >= amt); assert_eq!(notes.len(), 1); assert_eq!(notes[0].note.value, value1); @@ -1565,7 +1845,7 @@ mod test { // Selecting a bigger amount should select both notes let amt = Amount::from_u64(value1 + value2).unwrap(); - let (notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false).await; + let (_, notes, utxos, selected) = lc.wallet.select_notes_and_utxos(amt, false, false, true, false).await; assert!(selected == amt); assert_eq!(notes.len(), 2); assert_eq!(utxos.len(), 0); diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index 90c6dbc7..b467527b 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -1,6 +1,9 @@ use crate::compact_formats::CompactBlock; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use incrementalmerkletree::Position; use orchard::keys::FullViewingKey; +use orchard::note::RandomSeed; +use orchard::value::NoteValue; use orchard::Address; use prost::Message; use std::convert::TryFrom; @@ -180,13 +183,13 @@ impl WitnessCache { pub struct OrchardNoteData { pub(super) fvk: FullViewingKey, - pub note_address: Address, - pub note_value: u64, - pub note_nullifier: orchard::note::Nullifier, + pub note: orchard::Note, - // Info needed to recreate note - pub action_bytes: Vec, + // (Block number, tx_num, output_num) + pub created_at: (u64, usize, u32), + pub witness_position: Option, + // Info needed to recreate note pub spent: Option<(TxId, u32)>, // If this note was confirmed spent // If this note was spent in a send, but has not yet been confirmed. @@ -219,10 +222,17 @@ impl OrchardNoteData { let note_value = reader.read_u64::()?; let mut rho_bytes = [0u8; 32]; reader.read_exact(&mut rho_bytes)?; - let note_nullifier = orchard::note::Nullifier::from_bytes(&rho_bytes).unwrap(); + let note_rho = orchard::note::Nullifier::from_bytes(&rho_bytes).unwrap(); + let mut note_rseed_bytes = [0u8; 32]; + reader.read_exact(&mut note_rseed_bytes)?; + let note_rseed = RandomSeed::from_bytes(note_rseed_bytes, ¬e_rho).unwrap(); - // Read the action bytes - let action_bytes = Vector::read(&mut reader, |r| r.read_u8())?; + let note = orchard::Note::from_parts(note_address, NoteValue::from_raw(note_value), note_rho, note_rseed); + + let witness_position = Optional::read(&mut reader, |r| { + let pos = r.read_u64::()?; + Ok(Position::from(pos as usize)) + })?; let spent = Optional::read(&mut reader, |r| { let mut txid_bytes = [0u8; 32]; @@ -262,10 +272,9 @@ impl OrchardNoteData { Ok(OrchardNoteData { fvk, - note_address, - note_value, - note_nullifier, - action_bytes, + note, + created_at: (0, 0, 0), + witness_position, spent, unconfirmed_spent, memo, @@ -281,12 +290,15 @@ impl OrchardNoteData { self.fvk.write(&mut writer)?; // Write the components of the note - writer.write_all(&self.note_address.to_raw_address_bytes())?; - writer.write_u64::(self.note_value)?; - writer.write_all(&self.note_nullifier.to_bytes())?; - - // Write the action bytes - Vector::write(&mut writer, &self.action_bytes, |w, b| w.write_u8(*b))?; + writer.write_all(&self.note.recipient().to_raw_address_bytes())?; + writer.write_u64::(self.note.value().inner())?; + writer.write_all(&self.note.rho().to_bytes())?; + writer.write_all(self.note.rseed().as_bytes())?; + + // We don't write the created_at, because it should be temporary + Optional::write(&mut writer, self.witness_position, |w, p| { + w.write_u64::(p.into()) + })?; Optional::write(&mut writer, self.spent, |w, (txid, h)| { w.write_all(txid.as_ref())?; @@ -740,6 +752,9 @@ pub struct WalletTx { // List of all Utxos received in this Tx. Some of these might be change notes pub utxos: Vec, + // Total value of all orchard nullifiers that were spent in this Tx + pub total_orchard_value_spent: u64, + // Total value of all the sapling nullifiers that were spent in this Tx pub total_sapling_value_spent: u64, @@ -758,7 +773,7 @@ pub struct WalletTx { impl WalletTx { pub fn serialized_version() -> u64 { - return 22; + return 23; } pub fn new_txid(txid: &Vec) -> TxId { @@ -795,6 +810,7 @@ impl WalletTx { utxos: vec![], total_transparent_value_spent: 0, total_sapling_value_spent: 0, + total_orchard_value_spent: 0, outgoing_metadata: vec![], full_tx_scanned: false, zec_price: None, @@ -822,6 +838,11 @@ impl WalletTx { let s_notes = Vector::read(&mut reader, |r| SaplingNoteData::read(r))?; let utxos = Vector::read(&mut reader, |r| Utxo::read(r))?; + let total_orchard_value_spent = if version <= 22 { + 0 + } else { + reader.read_u64::()? + }; let total_sapling_value_spent = reader.read_u64::()?; let total_transparent_value_spent = reader.read_u64::()?; @@ -873,6 +894,7 @@ impl WalletTx { s_spent_nullifiers, o_spent_nullifiers, total_sapling_value_spent, + total_orchard_value_spent, total_transparent_value_spent, outgoing_metadata, full_tx_scanned, @@ -895,6 +917,7 @@ impl WalletTx { Vector::write(&mut writer, &self.s_notes, |w, nd| nd.write(w))?; Vector::write(&mut writer, &self.utxos, |w, u| u.write(w))?; + writer.write_u64::(self.total_orchard_value_spent)?; writer.write_u64::(self.total_sapling_value_spent)?; writer.write_u64::(self.total_transparent_value_spent)?; @@ -915,7 +938,14 @@ impl WalletTx { } } -pub struct SpendableNote { +pub struct SpendableOrchardNote { + pub txid: TxId, + pub sk: orchard::keys::SpendingKey, + pub note: orchard::Note, + pub merkle_path: orchard::tree::MerklePath, +} + +pub struct SpendableSaplingNote { pub txid: TxId, pub nullifier: sapling::Nullifier, pub diversifier: Diversifier, @@ -924,7 +954,7 @@ pub struct SpendableNote { pub extsk: ExtendedSpendingKey, } -impl SpendableNote { +impl SpendableSaplingNote { pub fn from( txid: TxId, nd: &SaplingNoteData, @@ -939,7 +969,7 @@ impl SpendableNote { { let witness = nd.witnesses.get(nd.witnesses.len() - anchor_offset - 1); - witness.map(|w| SpendableNote { + witness.map(|w| SpendableSaplingNote { txid, nullifier: nd.nullifier, diversifier: nd.diversifier, diff --git a/lib/src/lightwallet/keys.rs b/lib/src/lightwallet/keys.rs index c8829aa4..a7e76a87 100644 --- a/lib/src/lightwallet/keys.rs +++ b/lib/src/lightwallet/keys.rs @@ -474,6 +474,14 @@ impl Keys

{ .collect() } + pub fn get_all_spendable_oaddresses(&self) -> Vec { + self.okeys + .iter() + .filter(|ok| ok.have_spending_key()) + .map(|ok| ok.unified_address.encode(self.config.get_network())) + .collect() + } + pub fn get_all_spendable_zaddresses(&self) -> Vec { self.zkeys .iter() @@ -502,6 +510,14 @@ impl Keys

{ .unwrap_or(false) } + pub fn get_orchard_sk_for_fvk(&self, fvk: &orchard::keys::FullViewingKey) -> Option { + self.okeys + .iter() + .find(|ok| ok.fvk == *fvk) + .map(|osk| osk.sk.clone()) + .flatten() + } + pub fn get_extsk_for_extfvk(&self, extfvk: &ExtendedFullViewingKey) -> Option { self.zkeys .iter() @@ -886,6 +902,7 @@ impl Keys

{ pub fn is_shielded_address(addr: &String, config: &LightClientConfig

) -> bool { match address::RecipientAddress::decode(&config.get_params(), addr) { Some(address::RecipientAddress::Shielded(_)) => true, + Some(address::RecipientAddress::Unified(_)) => true, _ => false, } } diff --git a/lib/src/lightwallet/wallet_txns.rs b/lib/src/lightwallet/wallet_txns.rs index 9d14a537..3f78dfca 100644 --- a/lib/src/lightwallet/wallet_txns.rs +++ b/lib/src/lightwallet/wallet_txns.rs @@ -4,6 +4,7 @@ use std::{ }; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use incrementalmerkletree::Position; use log::{error, info}; use orchard::keys::FullViewingKey; use zcash_encoding::Vector; @@ -238,7 +239,7 @@ impl WalletTxns { .unwrap_or(0) } - pub fn get_unspent_nullifiers(&self) -> Vec<(Nullifier, u64, TxId)> { + pub fn get_unspent_s_nullifiers(&self) -> Vec<(Nullifier, u64, TxId)> { self.current .iter() .flat_map(|(_, wtx)| { @@ -250,6 +251,18 @@ impl WalletTxns { .collect() } + pub fn get_unspent_o_nullifiers(&self) -> Vec<(orchard::note::Nullifier, u64, TxId)> { + self.current + .iter() + .flat_map(|(_, wtx)| { + wtx.o_notes + .iter() + .filter(|nd| nd.spent.is_none()) + .map(move |nd| (nd.note.nullifier(&nd.fvk), nd.note.value().inner(), wtx.txid.clone())) + }) + .collect() + } + pub(crate) fn get_note_witness(&self, txid: &TxId, nullifier: &Nullifier) -> Option<(WitnessCache, BlockHeight)> { self.current.get(txid).map(|wtx| { wtx.s_notes @@ -259,7 +272,24 @@ impl WalletTxns { })? } - pub(crate) fn set_note_witnesses(&mut self, txid: &TxId, nullifier: &Nullifier, witnesses: WitnessCache) { + pub(crate) fn set_o_note_witness( + &mut self, + (height, tx_num, output_num): (u64, usize, u32), + pos: Option, + ) { + self.current.iter_mut().for_each(|(_, wtx)| { + wtx.o_notes + .iter_mut() + .filter(|on| on.witness_position.is_none()) + .find(|on| { + let (h, t, p) = on.created_at; + height == h && t == tx_num && output_num == p + }) + .map(|on| on.witness_position = pos); + }); + } + + pub(crate) fn set_s_note_witnesses(&mut self, txid: &TxId, nullifier: &Nullifier, witnesses: WitnessCache) { self.current .get_mut(txid) .unwrap() @@ -299,9 +329,31 @@ impl WalletTxns { } // Will mark the nullifier of the given txid as spent. Returns the amount of the nullifier - pub fn mark_txid_nf_spent( + pub fn mark_txid_o_nf_spent( &mut self, - txid: TxId, + txid: &TxId, + nullifier: &orchard::note::Nullifier, + spent_txid: &TxId, + spent_at_height: BlockHeight, + ) -> u64 { + let mut note_data = self + .current + .get_mut(&txid) + .unwrap() + .o_notes + .iter_mut() + .find(|n| n.note.nullifier(&n.fvk) == *nullifier) + .unwrap(); + + note_data.spent = Some((spent_txid.clone(), spent_at_height.into())); + note_data.unconfirmed_spent = None; + note_data.note.value().inner() + } + + // Will mark the nullifier of the given txid as spent. Returns the amount of the nullifier + pub fn mark_txid_s_nf_spent( + &mut self, + txid: &TxId, nullifier: &Nullifier, spent_txid: &TxId, spent_at_height: BlockHeight, @@ -327,7 +379,11 @@ impl WalletTxns { self.current.get_mut(txid).map(|wtx| { wtx.s_notes.iter_mut().for_each(|n| { n.is_change = true; - }) + }); + + wtx.o_notes.iter_mut().for_each(|n| { + n.is_change = true; + }); }); } } @@ -362,8 +418,52 @@ impl WalletTxns { price.map(|p| self.current.get_mut(txid).map(|tx| tx.zec_price = Some(p))); } + pub fn add_new_o_spent( + &mut self, + txid: TxId, + height: BlockHeight, + unconfirmed: bool, + timestamp: u32, + nullifier: orchard::note::Nullifier, + value: u64, + source_txid: TxId, + ) -> Option { + // Record this Tx as having spent some funds + { + let wtx = self.get_or_create_tx(&txid, BlockHeight::from(height), unconfirmed, timestamp as u64); + + // Mark the height correctly, in case this was previously a mempool or unconfirmed tx. + wtx.block = height; + + if wtx.o_spent_nullifiers.iter().find(|nf| **nf == nullifier).is_none() { + wtx.o_spent_nullifiers.push(nullifier); + wtx.total_orchard_value_spent += value; + } + } + + // Since this Txid has spent some funds, output notes in this Tx that are sent to us are actually change. + self.check_notes_mark_change(&txid); + + // Mark the source note's nullifier as spent + if !unconfirmed { + let wtx = self.current.get_mut(&source_txid).expect("Txid should be present"); + + wtx.o_notes + .iter_mut() + .find(|n| n.note.nullifier(&n.fvk) == nullifier) + .map(|nd| { + // Record the spent height + nd.spent = Some((txid, height.into())); + nd.witness_position + }) + .flatten() + } else { + None + } + } + // Records a TxId as having spent some nullifiers from the wallet. - pub fn add_new_spent( + pub fn add_new_s_spent( &mut self, txid: TxId, height: BlockHeight, @@ -526,8 +626,8 @@ impl WalletTxns { height: BlockHeight, unconfirmed: bool, timestamp: u64, - action_bytes: Vec, note: orchard::Note, + created_at: (u64, usize, u32), fvk: &FullViewingKey, have_spending_key: bool, ) { @@ -540,16 +640,15 @@ impl WalletTxns { let note_nullifier = note.nullifier(fvk); - match wtx.o_notes.iter_mut().find(|n| n.note_nullifier == note_nullifier) { + match wtx.o_notes.iter_mut().find(|n| n.note.nullifier(fvk) == note_nullifier) { None => { let nd = OrchardNoteData { fvk: fvk.clone(), - note_address: note.recipient(), - note_value: note.value().inner(), - action_bytes, - note_nullifier, + note, spent: None, unconfirmed_spent: None, + created_at, + witness_position: None, memo: None, is_change, have_spending_key, @@ -588,7 +687,7 @@ impl WalletTxns { // Update the block height, in case this was a mempool or unconfirmed tx. wtx.block = height; - let nullifier = note.nf(&extfvk.fvk.vk, witness.position() as u64); + let nullifier = note.nf(&extfvk.fvk.vk.nk, witness.position() as u64); let witnesses = if have_spending_key { WitnessCache::new(vec![witness], u64::from(height)) } else { @@ -643,7 +742,7 @@ impl WalletTxns { self.current.get_mut(txid).map(|wtx| { wtx.o_notes .iter_mut() - .find(|n| n.note_nullifier == note_nullifier) + .find(|n| n.note.nullifier(fvk) == note_nullifier) .map(|n| n.memo = Some(memo)); }); }