Fix V1 feature flags (paritytech#117)

* add toy example and fix some feature flags

* fmt

* bn => block_number

* add a couple of comments

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
 	"bin/tx-decoder",
+	"bin/archive-demo",
 	"bin/v14-test",
 	"desub",
 	"desub-current",

bin/archive-demo/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "archive-demo"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+description.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+clap = { workspace = true, features = ["derive"] }
+desub = { workspace = true, features = ["polkadot-js"] }
+anyhow = { workspace = true }
+hex = { workspace = true }
+subxt = "0.32.1"
+tokio = { version = "1.33.0", features = ["full"] }
+pretty_env_logger = { workspace = true }

bin/archive-demo/src/main.rs

@@ -0,0 +1,60 @@
+use desub::{Chain, Decoder};
+use subxt::{
+	backend::{
+		legacy::{
+			rpc_methods::{Bytes, NumberOrHex},
+			LegacyRpcMethods,
+		},
+		rpc::{rpc_params, RpcClient},
+	},
+	config::PolkadotConfig,
+};
+
+#[tokio::main]
+async fn main() -> Result<(), anyhow::Error> {
+	pretty_env_logger::init();
+
+	// Connect to a node with an RPC client:
+	let rpc_client = RpcClient::from_url("wss://rpc.polkadot.io").await?;
+	let methods = LegacyRpcMethods::<PolkadotConfig>::new(rpc_client.clone());
+
+	let mut block_number = 1;
+	let mut decoder = Decoder::new(Chain::Polkadot);
+
+	loop {
+		// Fetch the extrinsics and spec version, which we need for decoding:
+		let hash = methods.chain_get_block_hash(Some(NumberOrHex::Number(block_number))).await?.unwrap();
+		let runtime_version = methods.state_get_runtime_version(Some(hash)).await?;
+		let spec_version = runtime_version.spec_version;
+		let block = methods.chain_get_block(Some(hash)).await?.unwrap();
+
+		// Mangle the extrinsics into the correct byte format from the RPC call which returns a Vec:
+		let ext_bytes = make_extrinsic_bytes(block.block.extrinsics);
+
+		if !decoder.has_version(spec_version) {
+			// download the relevant metadata bytes, since the decoder doesn't have it yet.
+			println!("# Downloading metadata for spec version {spec_version}");
+			let md: Bytes = rpc_client.request("state_getMetadata", rpc_params![hash]).await?;
+			decoder.register_version(spec_version, &md.0)?;
+		}
+
+		println!("# Decoding exts for block {block_number}");
+		let decoded_exts = decoder.decode_extrinsics(spec_version, &ext_bytes)?;
+
+		println!("{decoded_exts}");
+
+		// We'll decode every 10_000th block, just to make sure we span some spec versions.
+		block_number += 10_000;
+	}
+}
+
+// A hack because we get the exts back as a vec of bytes and
+// desub wants the whole block body as bytes.
+fn make_extrinsic_bytes(exts: Vec<Bytes>) -> Vec<u8> {
+	use subxt::ext::codec::Encode;
+	use subxt::utils::Encoded;
+	// The inner `Bytes` are already encoded and contain the compact length etc,
+	// so don't encode them again by wrapping them in `Encoded`.
+	let e: Vec<Encoded> = exts.into_iter().map(|ext| Encoded(ext.0)).collect();
+	e.encode()
+}

bin/tx-decoder/Cargo.toml

@@ -12,21 +12,21 @@ rust-version.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-desub = { workspace = true, features = [ "polkadot-js" ] }
+desub = { workspace = true, features = ["polkadot-js"] }
 anyhow = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 log = { workspace = true }
 
 # These are only used here, so aren't part of the workspace deps (this binary might go away):
-async-std = { version = "1.10.0", features = [ "attributes" ] }
+async-std = { version = "1.10.0", features = ["attributes"] }
 async-stream = "0.3.2"
-sqlx = { version = "0.5", features = [ "runtime-async-std-rustls", "postgres", "offline" ]}
+sqlx = { version = "0.5", features = ["runtime-async-std-rustls", "postgres", "offline"]}
 futures = "0.3.17"
 argh = "0.1.6"
-fern = { version = "0.6.0", features = [ "colored" ] }
+fern = { version = "0.6.0", features = ["colored"] }
 colored = "2.0.0"
-indicatif = { version = "0.16.2", features = [ "rayon" ] }
+indicatif = { version = "0.16.2", features = ["rayon"] }
 rayon = "1.5.1"
 parking_lot = "0.11.2"
 num_cpus = "1.13.0"

bin/tx-decoder/src/app.rs

@@ -142,13 +142,13 @@ impl<'a> AppState<'a> {
 	/// returns the previous spec version.
 	async fn register_metadata(&self, conn: &mut PgConnection, version: SpecVersion) -> Result<Option<u32>, Error> {
 		let (past, present) = past_and_present_version(conn, version.try_into()?).await?;
-		if !self.decoder.read().has_version(&present) {
+		if !self.decoder.read().has_version(present) {
 			let meta = metadata(conn, present.try_into()?).await?;
 			self.decoder.write().register_version(present, &meta)?;
 		}
 
 		if let Some(p) = past {
-			if !self.decoder.read().has_version(&p) {
+			if !self.decoder.read().has_version(p) {
 				let meta = metadata(conn, p.try_into()?).await?;
 				self.decoder.write().register_version(p, &meta)?;
 			}

desub-common/Cargo.toml

@@ -12,7 +12,7 @@ rust-version.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-serde = { workspace = true, features = [ "derive" ] }
+serde = { workspace = true, features = ["derive"] }
 
 sp-runtime = { workspace = true }
 sp-core = { workspace = true }

desub-json-resolver/Cargo.toml

@@ -18,9 +18,8 @@ log = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true, features = ["preserve_order"] }
 syn = { workspace = true, features = ["parsing", "derive"] }
-phf = { workspace = true, features = [ "macros" ] }
+phf = { workspace = true, features = ["macros"] }
 
 [features]
-default = ["default_definitions"]
-polkadot = []
-default_definitions = []
+default = ["default-definitions"]
+default-definitions = []

desub-json-resolver/src/resolver.rs

@@ -17,7 +17,7 @@
 use crate::{Extrinsics, Modules, Overrides, Result};
 use desub_legacy::{regex, RustTypeMarker, TypeDetective};
 
-#[cfg(feature = "default_definitions")]
+#[cfg(feature = "default-definitions")]
 mod default {
 	pub const DEFINITIONS: &str = include_str!("./definitions/definitions.json");
 	pub const OVERRIDES: &str = include_str!("./definitions/overrides.json");
@@ -31,6 +31,10 @@ pub struct Builder {
 }
 
 impl Builder {
+	pub fn new(mods: Modules, extrinsics: Extrinsics, overrides: Overrides) -> Self {
+		Self { mods, overrides, extrinsics }
+	}
+
 	pub fn modules(mut self, modules: Modules) -> Self {
 		self.mods = modules;
 		self
@@ -66,16 +70,7 @@ impl Builder {
 	}
 }
 
-// we need a way to construct the builder when
-// not using default features
-#[cfg(not(feature = "default_definitions"))]
-impl Builder {
-	fn new(modules: Modules, extrinsics: Extrinsics, overrides: Overrides) -> Self {
-		Self { mods, overrides, extrinsics }
-	}
-}
-
-#[cfg(feature = "default_definitions")]
+#[cfg(feature = "default-definitions")]
 impl Default for Builder {
 	fn default() -> Self {
 		Self {
@@ -85,7 +80,7 @@ impl Default for Builder {
 		}
 	}
 }
-#[cfg(feature = "default_definitions")]
+#[cfg(feature = "default-definitions")]
 impl Default for TypeResolver {
 	fn default() -> Self {
 		Builder::default().build()
@@ -101,6 +96,7 @@ pub struct TypeResolver {
 
 impl TypeResolver {
 	/// Build the builder for `TypeResolver`
+	#[cfg(feature = "default-definitions")]
 	pub fn builder() -> Builder {
 		Builder::default()
 	}

desub-legacy/src/decoder.rs

@@ -465,7 +465,7 @@ impl Decoder {
 		let (length, prefix) = Self::scale_length(data)?;
 		let meta = self.versions.get(&spec).ok_or(Error::MissingSpec(spec))?;
 		log::trace!("Decoding {} Total Extrinsics. CALLS: {:#?}", length, meta.modules_by_call_index);
-
+		log::trace!("Extrinsics bytes: {data:?}");
 		let mut state = DecodeState::new(None, None, meta, prefix, spec, data);
 		for (idx, extrinsic) in ChunkedExtrinsic::new(&data[prefix..]).enumerate() {
 			log::trace!("Extrinsic {}:{:?}", idx, extrinsic);

desub/Cargo.toml

@@ -11,17 +11,21 @@ rust-version.workspace = true
 
 [features]
 
-polkadot-js = ["desub-json-resolver", "desub-json-resolver/polkadot", "frame-metadata/legacy" ]
+polkadot-js = [
+    "desub-json-resolver",
+    "desub-json-resolver/default-definitions",
+    "frame-metadata/legacy"
+]
 
 [dependencies]
 
 desub-legacy = { workspace = true }
 desub-common = { workspace = true}
 desub-current = { workspace = true }
-desub-json-resolver = { workspace = true, optional = true }
+desub-json-resolver = { workspace = true, optional = true, default-features = true }
 
 thiserror = { workspace = true }
-frame-metadata = { workspace = true }
+frame-metadata = { workspace = true, features = ["legacy"] }
 parity-scale-codec = { workspace = true }
 serde_json = { workspace = true, features = ["preserve_order", "arbitrary_precision"] }
 

desub/src/lib.rs

@@ -120,7 +120,7 @@ impl Decoder {
 		}
 	}
 
-	pub fn has_version(&self, version: &SpecVersion) -> bool {
-		self.current_metadata.contains_key(version) || self.legacy_decoder.has_version(version)
+	pub fn has_version(&self, version: SpecVersion) -> bool {
+		self.current_metadata.contains_key(&version) || self.legacy_decoder.has_version(&version)
 	}
 }