Load KZG only once per process, exclude electra tests and add missing…

… SSZ tests.

Makefile

@@ -152,9 +152,6 @@ run-ef-tests:
 
 # Runs EF test vectors with nextest
 nextest-run-ef-tests:
-	rm -rf $(EF_TESTS)/.accessed_file_log.txt
-	cargo nextest run --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES)"
-	cargo nextest run --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES),fake_crypto"
 	./$(EF_TESTS)/check_all_files_accessed.py $(EF_TESTS)/.accessed_file_log.txt $(EF_TESTS)/consensus-spec-tests
 
 # Run the tests in the `beacon_chain` crate for all known forks.

consensus/types/src/lib.rs

@@ -134,7 +134,7 @@ pub use crate::beacon_block_body::{
 pub use crate::beacon_block_header::BeaconBlockHeader;
 pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee};
 pub use crate::beacon_state::{Error as BeaconStateError, *};
-pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList, BlobsList};
+pub use crate::blob_sidecar::{BlobIdentifier, BlobSidecar, BlobSidecarList, BlobsList};
 pub use crate::bls_to_execution_change::BlsToExecutionChange;
 pub use crate::chain_spec::{ChainSpec, Config, Domain};
 pub use crate::checkpoint::Checkpoint;
@@ -143,7 +143,7 @@ pub use crate::config_and_preset::{
 };
 pub use crate::consolidation::Consolidation;
 pub use crate::contribution_and_proof::ContributionAndProof;
-pub use crate::data_column_sidecar::{ColumnIndex, DataColumnSidecar};
+pub use crate::data_column_sidecar::{ColumnIndex, DataColumnIdentifier, DataColumnSidecar};
 pub use crate::data_column_subnet_id::DataColumnSubnetId;
 pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH};
 pub use crate::deposit_data::DepositData;

testing/ef_tests/Cargo.toml

@@ -40,3 +40,4 @@ store = { workspace = true }
 fork_choice = { workspace = true }
 execution_layer = { workspace = true }
 logging = { workspace = true }
+lazy_static = { workspace = true }

testing/ef_tests/check_all_files_accessed.py

@@ -20,6 +20,8 @@
 # following regular expressions, we will assume they are to be ignored (i.e., we are purposefully
 # *not* running the spec tests).
 excluded_paths = [
+    # TODO(das): remove once electra tests are on unstable
+    "tests/.*/electra/",
     # Eth1Block and PowBlock
     #
     # Intentionally omitted, as per https://github.com/sigp/lighthouse/issues/1835
@@ -34,7 +36,7 @@
     # Unused kzg methods
     "tests/.*/.*/kzg/compute_cells",
     "tests/.*/.*/kzg/recover_all_cells",
-    "tests/.*/.*/kzg/verify_cell_kzg_batch",
+    "tests/.*/.*/kzg/verify_cell_kzg_proof",
     # One of the EF researchers likes to pack the tarballs on a Mac
     ".*\.DS_Store.*",
     # More Mac weirdness.

testing/ef_tests/src/cases.rs

@@ -77,7 +77,9 @@ pub enum FeatureName {
 
 impl Display for FeatureName {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.write_str("eip7594")
+        match self {
+            FeatureName::Eip7594 => f.write_str("eip7594"),
+        }
     }
 }
 

testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs

@@ -36,10 +36,8 @@ impl<E: EthSpec> Case for KZGBlobToKZGCommitment<E> {
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
-        let kzg = get_kzg()?;
-
         let commitment = parse_blob::<E>(&self.input.blob).and_then(|blob| {
-            blob_to_kzg_commitment::<E>(&kzg, &blob).map_err(|e| {
+            blob_to_kzg_commitment::<E>(&KZG, &blob).map_err(|e| {
                 Error::InternalError(format!("Failed to compute kzg commitment: {:?}", e))
             })
         });

testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs

@@ -43,9 +43,8 @@ impl<E: EthSpec> Case for KZGComputeBlobKZGProof<E> {
             Ok((blob, commitment))
         };
 
-        let kzg = get_kzg()?;
         let proof = parse_input(&self.input).and_then(|(blob, commitment)| {
-            compute_blob_kzg_proof::<E>(&kzg, &blob, commitment)
+            compute_blob_kzg_proof::<E>(&KZG, &blob, commitment)
                 .map_err(|e| Error::InternalError(format!("Failed to compute kzg proof: {:?}", e)))
         });
 

testing/ef_tests/src/cases/kzg_compute_cells_and_kzg_proofs.rs

@@ -33,12 +33,11 @@ impl<E: EthSpec> Case for KZGComputeCellsAndKZGProofs<E> {
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         // TODO(das): lazy init Kzg as a static ref
-        let kzg = get_kzg()?;
         let cells_and_proofs = parse_blob::<E>(&self.input.blob).and_then(|blob| {
             let blob = KzgBlob::from_bytes(&blob).map_err(|e| {
                 Error::InternalError(format!("Failed to convert blob to kzg blob: {e:?}"))
             })?;
-            kzg.compute_cells_and_proofs(&blob).map_err(|e| {
+            KZG.compute_cells_and_proofs(&blob).map_err(|e| {
                 Error::InternalError(format!("Failed to compute cells and kzg proofs: {e:?}"))
             })
         });

testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs

@@ -50,9 +50,8 @@ impl<E: EthSpec> Case for KZGComputeKZGProof<E> {
             Ok((blob, z))
         };
 
-        let kzg = get_kzg()?;
         let proof = parse_input(&self.input).and_then(|(blob, z)| {
-            compute_kzg_proof::<E>(&kzg, &blob, z)
+            compute_kzg_proof::<E>(&KZG, &blob, z)
                 .map_err(|e| Error::InternalError(format!("Failed to compute kzg proof: {:?}", e)))
         });
 

testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs

@@ -3,15 +3,20 @@ use crate::case_result::compare_result;
 use beacon_chain::kzg_utils::validate_blob;
 use eth2_network_config::TRUSTED_SETUP_BYTES;
 use kzg::{Cell, Error as KzgError, Kzg, KzgCommitment, KzgProof, TrustedSetup};
+use lazy_static::lazy_static;
 use serde::Deserialize;
 use std::marker::PhantomData;
+use std::sync::Arc;
 use types::Blob;
 
-pub fn get_kzg() -> Result<Kzg, Error> {
-    let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES)
-        .map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e)))?;
-    Kzg::new_from_trusted_setup(trusted_setup)
-        .map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e)))
+lazy_static! {
+    pub static ref KZG: Arc<Kzg> = {
+        let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES)
+            .map_err(|e| format!("Unable to read trusted setup file: {}", e))
+            .expect("should have trusted setup");
+        let kzg = Kzg::new_from_trusted_setup(trusted_setup).expect("should create kzg");
+        Arc::new(kzg)
+    };
 }
 
 // TODO(das) `EthSpec` can be removed once KZG lib exposes the `BytesPerCell` constant.
@@ -119,9 +124,8 @@ impl<E: EthSpec> Case for KZGVerifyBlobKZGProof<E> {
             Ok((blob, commitment, proof))
         };
 
-        let kzg = get_kzg()?;
         let result = parse_input(&self.input).and_then(|(blob, commitment, proof)| {
-            match validate_blob::<E>(&kzg, &blob, commitment, proof) {
+            match validate_blob::<E>(&KZG, &blob, commitment, proof) {
                 Ok(_) => Ok(true),
                 Err(KzgError::KzgVerificationFailed) => Ok(false),
                 Err(e) => Err(Error::InternalError(format!(

testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs

@@ -57,12 +57,10 @@ impl<E: EthSpec> Case for KZGVerifyBlobKZGProofBatch<E> {
             Ok((commitments, blobs, proofs))
         };
 
-        let kzg = get_kzg()?;
-
         let result =
             parse_input(&self.input).and_then(
                 |(commitments, blobs, proofs)| match validate_blobs::<E>(
-                    &kzg,
+                    &KZG,
                     &commitments,
                     blobs.iter().collect(),
                     &proofs,

testing/ef_tests/src/cases/kzg_verify_cell_kzg_proof_batch.rs

@@ -52,13 +52,11 @@ impl<E: EthSpec> Case for KZGVerifyCellKZGProofBatch<E> {
             Ok((cells, proofs, coordinates, row_commitments))
         };
 
-        let kzg = get_kzg()?;
-
         let result =
             parse_input(&self.input).and_then(|(cells, proofs, coordinates, commitments)| {
                 let proofs: Vec<Bytes48> = proofs.iter().map(|&proof| proof.into()).collect();
                 let commitments: Vec<Bytes48> = commitments.iter().map(|&c| c.into()).collect();
-                match kzg.verify_cell_proof_batch(
+                match KZG.verify_cell_proof_batch(
                     cells.as_slice(),
                     &proofs,
                     &coordinates,

testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs

@@ -46,9 +46,8 @@ impl<E: EthSpec> Case for KZGVerifyKZGProof<E> {
             Ok((commitment, z, y, proof))
         };
 
-        let kzg = get_kzg()?;
         let result = parse_input(&self.input).and_then(|(commitment, z, y, proof)| {
-            verify_kzg_proof::<E>(&kzg, commitment, proof, z, y)
+            verify_kzg_proof::<E>(&KZG, commitment, proof, z, y)
                 .map_err(|e| Error::InternalError(format!("Failed to validate proof: {:?}", e)))
         });
 

testing/ef_tests/src/type_name.rs

@@ -1,5 +1,4 @@
 //! Mapping from types to canonical string identifiers used in testing.
-use types::blob_sidecar::BlobIdentifier;
 use types::historical_summary::HistoricalSummary;
 use types::*;
 
@@ -52,7 +51,9 @@ type_name_generic!(BeaconBlockBodyDeneb, "BeaconBlockBody");
 type_name!(BeaconBlockHeader);
 type_name_generic!(BeaconState);
 type_name!(BlobIdentifier);
+type_name!(DataColumnIdentifier);
 type_name_generic!(BlobSidecar);
+type_name_generic!(DataColumnSidecar);
 type_name!(Checkpoint);
 type_name_generic!(ContributionAndProof);
 type_name!(Deposit);

testing/ef_tests/tests/tests.rs

@@ -214,10 +214,11 @@ macro_rules! ssz_static_test_no_run {
 
 #[cfg(feature = "fake_crypto")]
 mod ssz_static {
-    use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler};
-    use types::blob_sidecar::BlobIdentifier;
+    use ef_tests::{
+        FeatureName, Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler,
+    };
     use types::historical_summary::HistoricalSummary;
-    use types::{LightClientBootstrapAltair, *};
+    use types::*;
 
     ssz_static_test!(aggregate_and_proof, AggregateAndProof<_>);
     ssz_static_test!(attestation, Attestation<_>);
@@ -516,6 +517,22 @@ mod ssz_static {
         SszStaticHandler::<HistoricalSummary, MinimalEthSpec>::capella_and_later().run();
         SszStaticHandler::<HistoricalSummary, MainnetEthSpec>::capella_and_later().run();
     }
+
+    #[test]
+    fn data_column_sidecar() {
+        SszStaticHandler::<DataColumnSidecar<MinimalEthSpec>, MinimalEthSpec>::deneb_only()
+            .run_for_feature(ForkName::Deneb, FeatureName::Eip7594);
+        SszStaticHandler::<DataColumnSidecar<MainnetEthSpec>, MainnetEthSpec>::deneb_only()
+            .run_for_feature(ForkName::Deneb, FeatureName::Eip7594);
+    }
+
+    #[test]
+    fn data_column_identifier() {
+        SszStaticHandler::<DataColumnIdentifier, MinimalEthSpec>::deneb_only()
+            .run_for_feature(ForkName::Deneb, FeatureName::Eip7594);
+        SszStaticHandler::<DataColumnIdentifier, MainnetEthSpec>::deneb_only()
+            .run_for_feature(ForkName::Deneb, FeatureName::Eip7594);
+    }
 }
 
 #[test]