Implement EIP-7688: Forward compatible consensus data structures [WIP]

Cargo.toml

@@ -121,8 +121,8 @@ error-chain = "0.12"
 ethereum-types = "0.14"
 ethereum_hashing = "0.6.0"
 ethereum_serde_utils = "0.5.2"
-ethereum_ssz = "0.5"
-ethereum_ssz_derive = "0.5"
+ethereum_ssz = { git = "https://github.com/macladson/ethereum_ssz", branch = "stable-container" }
+ethereum_ssz_derive = { git = "https://github.com/macladson/ethereum_ssz", branch = "stable-container" }
 ethers-core = "1"
 ethers-providers = { version = "1", default-features = false }
 exit-future = "0.2"
@@ -137,7 +137,7 @@ libsecp256k1 = "0.7"
 log = "0.4"
 lru = "0.12"
 maplit = "1"
-milhouse = "0.1"
+milhouse = { git = "https://github.com/macladson/milhouse", branch = "stable-container" }
 num_cpus = "1"
 parking_lot = "0.12"
 paste = "1"
@@ -163,7 +163,7 @@ slog-term = "2"
 sloggers = { version = "2", features = ["json"] }
 smallvec = { version = "1.11.2", features = ["arbitrary"] }
 snap = "1"
-ssz_types = "0.6"
+ssz_types = { git = "https://github.com/macladson/ssz_types", branch = "stable-container" }
 strum = { version = "0.24", features = ["derive"] }
 superstruct = "0.8"
 syn = "1"
@@ -177,8 +177,8 @@ tracing-appender = "0.2"
 tracing-core = "0.1"
 tracing-log = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-tree_hash = "0.6"
-tree_hash_derive = "0.6"
+tree_hash = { git = "https://github.com/macladson/tree_hash", branch = "stable-container" }
+tree_hash_derive = { git = "https://github.com/macladson/tree_hash", branch = "stable-container" }
 url = "2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 warp = { version = "0.3.7", default-features = false, features = ["tls"] }

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -3446,7 +3446,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         {
             let mut slashable_cache = self.observed_slashable.write();
             for header in blobs
-                .into_iter()
+                .iter()
                 .filter_map(|b| b.as_ref().map(|b| b.signed_block_header.clone()))
                 .unique()
             {

beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs

@@ -272,7 +272,7 @@ impl<E: EthSpec> PendingComponents<E> {
             BlockImportRequirement::AllBlobs => {
                 let num_blobs_expected = diet_executed_block.num_blobs_expected();
                 let Some(verified_blobs) = verified_blobs
-                    .into_iter()
+                    .iter()
                     .cloned()
                     .map(|b| b.map(|b| b.to_blob()))
                     .take(num_blobs_expected)

consensus/types/src/attestation.rs

@@ -5,7 +5,10 @@ use derivative::Derivative;
 use safe_arith::ArithError;
 use serde::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
-use ssz_types::BitVector;
+use ssz_types::{
+    typenum::{self, Unsigned},
+    BitVector,
+};
 use std::hash::{Hash, Hasher};
 use superstruct::superstruct;
 use test_random_derive::TestRandom;
@@ -51,6 +54,10 @@ impl From<ssz_types::Error> for Error {
         serde(bound = "E: EthSpec", deny_unknown_fields),
         arbitrary(bound = "E: EthSpec"),
     ),
+    specific_variant_attributes(Electra(tree_hash(
+        struct_behaviour = "profile",
+        max_fields = "typenum::U8"
+    ))),
     ref_attributes(derive(TreeHash), tree_hash(enum_behaviour = "transparent")),
     cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
     partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")

consensus/types/src/attester_slashing.rs

@@ -6,6 +6,10 @@ use derivative::Derivative;
 use rand::{Rng, RngCore};
 use serde::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
+use ssz_types::{
+    typenum::{self, Unsigned},
+    BitVector,
+};
 use superstruct::superstruct;
 use test_random_derive::TestRandom;
 use tree_hash_derive::TreeHash;
@@ -29,6 +33,10 @@ use tree_hash_derive::TreeHash;
         serde(bound = "E: EthSpec"),
         arbitrary(bound = "E: EthSpec")
     ),
+    specific_variant_attributes(Electra(tree_hash(
+        struct_behaviour = "profile",
+        max_fields = "typenum::U8"
+    ))),
     ref_attributes(derive(Debug))
 )]
 #[derive(

consensus/types/src/beacon_block_body.rs

@@ -57,7 +57,10 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11;
         Bellatrix(metastruct(mappings(beacon_block_body_bellatrix_fields(groups(fields))))),
         Capella(metastruct(mappings(beacon_block_body_capella_fields(groups(fields))))),
         Deneb(metastruct(mappings(beacon_block_body_deneb_fields(groups(fields))))),
-        Electra(metastruct(mappings(beacon_block_body_electra_fields(groups(fields))))),
+        Electra(
+            metastruct(mappings(beacon_block_body_electra_fields(groups(fields)))),
+            tree_hash(struct_behaviour = "profile", max_fields = "typenum::U64"),
+        ),
     ),
     cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
     partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")

consensus/types/src/beacon_state.rs

@@ -13,6 +13,10 @@ use safe_arith::{ArithError, SafeArith};
 use serde::{Deserialize, Serialize};
 use ssz::{ssz_encode, Decode, DecodeError, Encode};
 use ssz_derive::{Decode, Encode};
+use ssz_types::{
+    typenum::{self, Unsigned},
+    BitVector,
+};
 use std::hash::Hash;
 use std::{fmt, mem, sync::Arc};
 use superstruct::superstruct;
@@ -221,6 +225,8 @@ impl From<BeaconStateHash> for Hash256 {
 /// we add internal mutability to `milhouse::{List, Vector}`. See:
 ///
 /// https://github.com/sigp/milhouse/issues/43
+///
+/// This is overwritten with the Stable Container PoC.
 #[superstruct(
     variants(Base, Altair, Bellatrix, Capella, Deneb, Electra),
     variant_attributes(
@@ -312,29 +318,39 @@ impl From<BeaconStateHash> for Hash256 {
             )),
             num_fields(all()),
         )),
-        Electra(metastruct(
-            mappings(
-                map_beacon_state_electra_fields(),
-                map_beacon_state_electra_tree_list_fields(mutable, fallible, groups(tree_lists)),
-                map_beacon_state_electra_tree_list_fields_immutable(groups(tree_lists)),
+        Electra(
+            metastruct(
+                mappings(
+                    map_beacon_state_electra_fields(),
+                    map_beacon_state_electra_tree_list_fields(
+                        mutable,
+                        fallible,
+                        groups(tree_lists)
+                    ),
+                    map_beacon_state_electra_tree_list_fields_immutable(groups(tree_lists)),
+                ),
+                bimappings(bimap_beacon_state_electra_tree_list_fields(
+                    other_type = "BeaconStateElectra",
+                    self_mutable,
+                    fallible,
+                    groups(tree_lists)
+                )),
+                num_fields(all()),
             ),
-            bimappings(bimap_beacon_state_electra_tree_list_fields(
-                other_type = "BeaconStateElectra",
-                self_mutable,
-                fallible,
-                groups(tree_lists)
-            )),
-            num_fields(all()),
-        ))
+            tree_hash(struct_behaviour = "profile", max_fields = "typenum::U128")
+        ),
     ),
     cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
     partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
     map_ref_mut_into(BeaconStateRef)
 )]
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, arbitrary::Arbitrary)]
+#[derive(
+    Debug, PartialEq, Clone, Serialize, Deserialize, Encode, TreeHash, arbitrary::Arbitrary,
+)]
 #[serde(untagged)]
 #[serde(bound = "E: EthSpec")]
 #[arbitrary(bound = "E: EthSpec")]
+#[tree_hash(enum_behaviour = "transparent_stable")]
 #[ssz(enum_behaviour = "transparent")]
 pub struct BeaconState<E>
 where
@@ -663,7 +679,8 @@ impl<E: EthSpec> BeaconState<E> {
 
     /// Returns the `tree_hash_root` of the state.
     pub fn canonical_root(&mut self) -> Result<Hash256, Error> {
-        self.update_tree_hash_cache()
+        Ok(Hash256::from_slice(&self.tree_hash_root()[..]))
+        //self.update_tree_hash_cache()
     }
 
     pub fn historical_batch(&mut self) -> Result<HistoricalBatch<E>, Error> {

consensus/types/src/execution_payload.rs

@@ -38,8 +38,12 @@ pub type ConsolidationRequests<E> =
         ),
         derivative(PartialEq, Hash(bound = "E: EthSpec")),
         serde(bound = "E: EthSpec", deny_unknown_fields),
-        arbitrary(bound = "E: EthSpec")
+        arbitrary(bound = "E: EthSpec"),
     ),
+    specific_variant_attributes(Electra(tree_hash(
+        struct_behaviour = "profile",
+        max_fields = "typenum::U64"
+    ))),
     cast_error(ty = "Error", expr = "BeaconStateError::IncorrectStateVariant"),
     partial_getter_error(ty = "Error", expr = "BeaconStateError::IncorrectStateVariant"),
     map_into(FullPayload, BlindedPayload),
@@ -52,7 +56,7 @@ pub type ConsolidationRequests<E> =
 #[serde(bound = "E: EthSpec", untagged)]
 #[arbitrary(bound = "E: EthSpec")]
 #[ssz(enum_behaviour = "transparent")]
-#[tree_hash(enum_behaviour = "transparent")]
+#[tree_hash(enum_behaviour = "transparent_stable")]
 pub struct ExecutionPayload<E: EthSpec> {
     #[superstruct(getter(copy))]
     pub parent_hash: ExecutionBlockHash,

consensus/types/src/execution_payload_header.rs

@@ -25,8 +25,12 @@ use tree_hash_derive::TreeHash;
         ),
         derivative(PartialEq, Hash(bound = "E: EthSpec")),
         serde(bound = "E: EthSpec", deny_unknown_fields),
-        arbitrary(bound = "E: EthSpec")
+        arbitrary(bound = "E: EthSpec"),
     ),
+    specific_variant_attributes(Electra(tree_hash(
+        struct_behaviour = "profile",
+        max_fields = "typenum::U64"
+    ))),
     ref_attributes(
         derive(PartialEq, TreeHash, Debug),
         tree_hash(enum_behaviour = "transparent")
@@ -41,7 +45,7 @@ use tree_hash_derive::TreeHash;
 #[derivative(PartialEq, Hash(bound = "E: EthSpec"))]
 #[serde(bound = "E: EthSpec", untagged)]
 #[arbitrary(bound = "E: EthSpec")]
-#[tree_hash(enum_behaviour = "transparent")]
+#[tree_hash(enum_behaviour = "transparent_stable")]
 #[ssz(enum_behaviour = "transparent")]
 pub struct ExecutionPayloadHeader<E: EthSpec> {
     #[superstruct(getter(copy))]