From c81e87812a9449b89d6d7a31a375090466b90544 Mon Sep 17 00:00:00 2001 From: color-typea Date: Tue, 3 Jun 2025 00:30:10 +0800 Subject: [PATCH] Support VariableList longer than 2**31 on 32-bit architectures * Improves handling large VariableList for 32-bit architectures - loudly crash vs. silently overflow (and produce wrong results) * Adds feature to enable capping typenum to usize conversion to usize::MAX * Tests + github actions --- .github/workflows/test-suite.yml | 22 +++ Cargo.toml | 9 ++ src/lib.rs | 1 + src/tree_hash.rs | 118 +++++++++++---- src/typenum_helpers.rs | 42 ++++++ src/variable_list.rs | 240 +++++++++++++++++++++++++++++-- 6 files changed, 396 insertions(+), 36 deletions(-) create mode 100644 src/typenum_helpers.rs diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index f5688e5..27112ee 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -31,6 +31,28 @@ jobs: run: rustup update stable - name: Run tests run: cargo test --release + cross-test-i686: + name: cross test i686-unknown-linux-gnu + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install cross + run: cargo install cross --git https://github.com/cross-rs/cross + - name: Add i686-unknown-linux-gnu target + run: rustup target add i686-unknown-linux-gnu + - name: Run cross test for i686-unknown-linux-gnu + run: cross test --target i686-unknown-linux-gnu + cross-test-i686-overflow: + name: cross test i686-unknown-linux-gnu (typenum overflow feature) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install cross + run: cargo install cross --git https://github.com/cross-rs/cross + - name: Add i686-unknown-linux-gnu target + run: rustup target add i686-unknown-linux-gnu + - name: Run cross test for i686-unknown-linux-gnu with cap-typenum-to-usize-overflow + run: cross test --target i686-unknown-linux-gnu --features cap-typenum-to-usize-overflow coverage: name: cargo-tarpaulin runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index 4099d4a..0caa03f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,16 @@ typenum = "1.12.0" smallvec = "1.8.0" arbitrary = { version = "1.0", features = ["derive"], optional = true } itertools = "0.13.0" +ethereum_hashing = {version = "0.7.0", optional = true} [dev-dependencies] serde_json = "1.0.0" tree_hash_derive = "0.10.0" +ethereum_hashing = {version = "0.7.0"} + +[target.i686-unknown-linux-gnu] +rustflags = ["-C", "target-feature=+sse2"] + +[features] +# Very careful usage - see comment in the typenum_helpers +cap-typenum-to-usize-overflow=["dep:ethereum_hashing"] diff --git a/src/lib.rs b/src/lib.rs index a4db4be..5af0448 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,6 +41,7 @@ mod fixed_vector; pub mod serde_utils; mod tree_hash; +mod typenum_helpers; mod variable_list; pub use fixed_vector::FixedVector; diff --git a/src/tree_hash.rs b/src/tree_hash.rs index 675d0ea..1505359 100644 --- a/src/tree_hash.rs +++ b/src/tree_hash.rs @@ -1,41 +1,107 @@ +use crate::typenum_helpers::to_usize; use tree_hash::{Hash256, MerkleHasher, TreeHash, TreeHashType}; use typenum::Unsigned; -/// A helper function providing common functionality between the `TreeHash` implementations for -/// `FixedVector` and `VariableList`. -pub fn vec_tree_hash_root(vec: &[T]) -> Hash256 -where - T: TreeHash, - N: Unsigned, -{ +pub fn packing_factor() -> usize { match T::tree_hash_type() { - TreeHashType::Basic => { - let mut hasher = MerkleHasher::with_leaves( - (N::to_usize() + T::tree_hash_packing_factor() - 1) / T::tree_hash_packing_factor(), - ); + TreeHashType::Basic => T::tree_hash_packing_factor(), + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => 1, + } +} + +mod default_impl { + use super::*; + /// A helper function providing common functionality between the `TreeHash` implementations for + /// `FixedVector` and `VariableList`. + pub fn vec_tree_hash_root(vec: &[T]) -> Hash256 + where + T: TreeHash, + N: Unsigned, + { + match T::tree_hash_type() { + TreeHashType::Basic => { + let mut hasher = MerkleHasher::with_leaves( + (to_usize::() + T::tree_hash_packing_factor() - 1) + / T::tree_hash_packing_factor(), + ); + + for item in vec { + hasher + .write(&item.tree_hash_packed_encoding()) + .expect("ssz_types variable vec should not contain more elements than max"); + } - for item in vec { hasher - .write(&item.tree_hash_packed_encoding()) - .expect("ssz_types variable vec should not contain more elements than max"); + .finish() + .expect("ssz_types variable vec should not have a remaining buffer") } + TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { + let mut hasher = MerkleHasher::with_leaves(N::to_usize()); - hasher - .finish() - .expect("ssz_types variable vec should not have a remaining buffer") - } - TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => { - let mut hasher = MerkleHasher::with_leaves(N::to_usize()); + for item in vec { + hasher + .write(item.tree_hash_root().as_slice()) + .expect("ssz_types vec should not contain more elements than max"); + } - for item in vec { hasher - .write(item.tree_hash_root().as_slice()) - .expect("ssz_types vec should not contain more elements than max"); + .finish() + .expect("ssz_types vec should not have a remaining buffer") } + } + } +} + +#[cfg(feature = "cap-typenum-to-usize-overflow")] +mod arch_32x_workaround { + use super::*; + use ethereum_hashing::{hash32_concat, ZERO_HASHES}; + use tree_hash::{Hash256, TreeHash}; + use typenum::Unsigned; + + type MaxDepth = typenum::U536870912; - hasher - .finish() - .expect("ssz_types vec should not have a remaining buffer") + fn pad_to_depth( + hash: Hash256, + target_depth: usize, + current_depth: usize, + ) -> Hash256 { + let mut curhash: [u8; 32] = hash.0; + for depth in current_depth..target_depth { + curhash = hash32_concat(&curhash, ZERO_HASHES[depth].as_slice()); + } + curhash.into() + } + + fn target_tree_depth() -> usize { + let packing_factor = packing_factor::(); + let packing_factor_log2 = packing_factor.next_power_of_two().ilog2() as usize; + let tree_depth = N::to_u64().next_power_of_two().ilog2() as usize; + tree_depth - packing_factor_log2 + } + + pub fn vec_tree_hash_root(vec: &[T]) -> Hash256 { + if N::to_u64() <= MaxDepth::to_u64() { + default_impl::vec_tree_hash_root::(vec) + } else { + let main_tree_hash = default_impl::vec_tree_hash_root::(vec); + + let target_depth = target_tree_depth::(); + let current_depth = target_tree_depth::(); + + pad_to_depth::(main_tree_hash, target_depth, current_depth) } } } + +#[cfg(any( + target_pointer_width = "64", + not(feature = "cap-typenum-to-usize-overflow") +))] +pub use default_impl::vec_tree_hash_root; + +#[cfg(all( + not(target_pointer_width = "64"), + feature = "cap-typenum-to-usize-overflow" +))] +pub use arch_32x_workaround::vec_tree_hash_root; diff --git a/src/typenum_helpers.rs b/src/typenum_helpers.rs new file mode 100644 index 0000000..77eb215 --- /dev/null +++ b/src/typenum_helpers.rs @@ -0,0 +1,42 @@ +use typenum::Unsigned; + +// On x64, all typenums always fit usize +#[cfg(target_pointer_width = "64")] +pub fn to_usize() -> usize { + N::to_usize() +} + +// On x32, typenums larger starting from 2**32 do not fit usize, +#[cfg(not(target_pointer_width = "64"))] +pub fn to_usize() -> usize { + let as_usize = N::to_usize(); + let as_u64 = N::to_u64(); + // If usize == u64 representation - N still fit usize, so + // no overflow happened + if as_usize as u64 == as_u64 { + return as_usize; + } + // else we have a choice: + // Option 1. Loudly panic with as informative message as possible + #[cfg(not(feature = "cap-typenum-to-usize-overflow"))] + panic!( + "Overflow converting typenum U{} to usize (usize::MAX={})", + as_u64, + usize::MAX + ); + // Option 2. Use usize::MAX - this allows working with VariableLists "virtually larger" than the + // usize, provided the actual number of elements do not exceed usize. + // + // One example is Ethereum BeaconChain.validators field that is a VariableList<..., 2**40>, + // but actual number of validators is far less than 2**32. + // + // This option still seems sound, since if the number of elements + // actually surpass usize::MAX, the machine running this will OOM/segfault/otherwise violently + // crash the program running this, which is nearly equivalent to panic. + // + // Still, the is a double-edged sword, only apply if you can guarantee that none of the + // VariableList used in your program will have more than usize::MAX elements on the + // architecture with the smallest usize it will be even run. + #[cfg(feature = "cap-typenum-to-usize-overflow")] + usize::MAX +} diff --git a/src/variable_list.rs b/src/variable_list.rs index 5dffad1..2560742 100644 --- a/src/variable_list.rs +++ b/src/variable_list.rs @@ -1,4 +1,5 @@ use crate::tree_hash::vec_tree_hash_root; +use crate::typenum_helpers::to_usize; use crate::Error; use serde::Deserialize; use serde_derive::Serialize; @@ -79,7 +80,7 @@ impl VariableList { /// Returns `Some` if the given `vec` equals the fixed length of `Self`. Otherwise returns /// `None`. pub fn new(vec: Vec) -> Result { - if vec.len() <= N::to_usize() { + if vec.len() <= to_usize::() { Ok(Self { vec, _phantom: PhantomData, @@ -112,7 +113,7 @@ impl VariableList { /// Returns the type-level maximum length. pub fn max_len() -> usize { - N::to_usize() + to_usize::() } /// Appends `value` to the back of `self`. @@ -133,7 +134,7 @@ impl VariableList { impl From> for VariableList { fn from(mut vec: Vec) -> Self { - vec.truncate(N::to_usize()); + vec.truncate(to_usize::()); Self { vec, @@ -256,7 +257,7 @@ impl ssz::TryFromIter for VariableList { where I: IntoIterator, { - let n = N::to_usize(); + let n = to_usize::(); let clamped_n = std::cmp::min(MAX_ELEMENTS_TO_PRE_ALLOCATE, n); let iter = value.into_iter(); @@ -282,7 +283,7 @@ where } fn from_ssz_bytes(bytes: &[u8]) -> Result { - let max_len = N::to_usize(); + let max_len = to_usize::(); if bytes.is_empty() { Ok(vec![].into()) @@ -323,7 +324,7 @@ where D: serde::Deserializer<'de>, { let vec = Vec::::deserialize(deserializer)?; - if vec.len() <= N::to_usize() { + if vec.len() <= to_usize::() { Ok(VariableList { vec, _phantom: PhantomData, @@ -332,7 +333,7 @@ where Err(serde::de::Error::custom(format!( "VariableList length {} exceeds maximum length {}", vec.len(), - N::to_usize() + to_usize::() ))) } } @@ -343,7 +344,7 @@ impl<'a, T: arbitrary::Arbitrary<'a>, N: 'static + Unsigned> arbitrary::Arbitrar for VariableList { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { - let max_size = N::to_usize(); + let max_size = to_usize::(); let rand = usize::arbitrary(u)?; let size = std::cmp::min(rand, max_size); let mut vec: Vec = Vec::with_capacity(size); @@ -544,7 +545,6 @@ mod test { } } - #[test] fn large_list_pre_allocation() { use std::iter; use typenum::U1099511627776; @@ -576,7 +576,7 @@ mod test { let iter = iter::repeat(1).take(5); let wonky_iter = WonkyIterator { - hint: N::to_usize() / 2, + hint: to_usize::() / 2, iter: iter.clone(), }; @@ -587,6 +587,19 @@ mod test { ); } + #[test] + #[cfg(any(target_pointer_width = "64", feature = "cap-typenum-to-usize-overflow"))] + fn large_list_pre_allocation_test() { + large_list_pre_allocation() + } + + #[test] + #[cfg(not(any(target_pointer_width = "64", feature = "cap-typenum-to-usize-overflow")))] + #[should_panic] + fn large_list_pre_allocation_test() { + large_list_pre_allocation() + } + #[test] fn std_hash() { let x: VariableList = VariableList::from(vec![3; 16]); @@ -616,4 +629,211 @@ mod test { let result: Result, _> = serde_json::from_value(json); assert!(result.is_ok()); } + + mod large_typenums { + use crate::tree_hash::packing_factor; + + use super::*; + use ethereum_hashing::ZERO_HASHES; + fn sanity_check() { + assert_eq!(U1099511627776::to_u64(), 1099511627776u64); + } + + fn ssz_bytes_len() { + let vec: VariableList = vec![0; 4].into(); + let vec2: VariableList = vec![0; 4].into(); + assert_eq!(vec.len(), 4); + assert_eq!(vec.ssz_bytes_len(), 8); + assert_eq!(vec2.len(), 4); + assert_eq!(vec2.ssz_bytes_len(), 8); + } + + fn encode() { + let vec: VariableList = vec![0; 2].into(); + assert_eq!(vec.as_ssz_bytes(), vec![0, 0, 0, 0]); + assert_eq!( + as Encode>::ssz_fixed_len(), + 4 + ); + } + + fn encode_non_power_of_2() { + type NVal = typenum::Add1; + let vec: VariableList = vec![0; 2].into(); + assert_eq!(vec.as_ssz_bytes(), vec![0, 0, 0, 0]); + assert_eq!( as Encode>::ssz_fixed_len(), 4); + } + + fn u16_len_2power40() { + round_trip::>(vec![42; 8].into()); + round_trip::>(vec![0; 8].into()); + } + + trait CreateZero { + fn zero() -> Self; + } + + impl CreateZero for Hash256 { + fn zero() -> Self { + [0; 32].into() + } + } + + impl CreateZero for u64 { + fn zero() -> Self { + 0 + } + } + + struct TreeHashTestCase { + pub expected_hash: Hash256, + pub vec: VariableList, + } + + impl TreeHashTestCase { + fn test(&self) { + assert_eq!( + self.vec.tree_hash_root(), + tree_hash::mix_in_length(&self.expected_hash, self.vec.len()) + ); + } + pub fn zeros(vec_len: usize) -> Self { + let full_depth = N::to_u64().next_power_of_two().ilog2(); + let packing_depth_discount = packing_factor::().next_power_of_two().ilog2(); + let depth = (full_depth - packing_depth_discount) as usize; + Self { + vec: VariableList::from(vec![T::zero(); vec_len]), + expected_hash: ZERO_HASHES[depth].into(), + } + } + } + + struct AllTreeHashTests { + _phantom: PhantomData, + } + + impl AllTreeHashTests { + pub fn all_tests() { + TreeHashTestCase::::zeros(10).test(); + + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::::zeros(10).test(); + + TreeHashTestCase::::zeros(10).test(); + + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::::zeros(10).test(); + TreeHashTestCase::>::zeros(10).test(); + + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::::zeros(10).test(); + TreeHashTestCase::>::zeros(10).test(); + + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::::zeros(10).test(); + TreeHashTestCase::>::zeros(10).test(); + + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::::zeros(10).test(); + TreeHashTestCase::>::zeros(10).test(); + + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::::zeros(10).test(); + TreeHashTestCase::>::zeros(10).test(); + + // 2 ** 40 + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::::zeros(10).test(); + TreeHashTestCase::>::zeros(10).test(); + + // 2**48 + TreeHashTestCase::>::zeros(10).test(); + TreeHashTestCase::::zeros(10).test(); + // Beyond 2**48 target_pointer_width="64" arches still work ok, target_pointer_width="32" fail due to ethereum_hashing::ZEROHASHES running out of elements + } + } + + #[cfg(any(target_pointer_width = "64", feature = "cap-typenum-to-usize-overflow"))] + mod arch64_bit_or_capping { + #[test] + fn sanity_check() { + super::sanity_check() + } + + #[test] + fn ssz_bytes_len() { + super::ssz_bytes_len(); + } + + #[test] + fn encode() { + super::encode(); + } + + #[test] + fn encode_non_power_of_2() { + super::encode_non_power_of_2(); + } + + #[test] + fn u16_len_2power40() { + super::u16_len_2power40() + } + + #[test] + fn tree_hash_tests_hash256() { + super::AllTreeHashTests::::all_tests(); + } + + #[test] + fn tree_hash_tests_u64() { + super::AllTreeHashTests::::all_tests(); + } + } + + #[cfg(not(any(target_pointer_width = "64", feature = "cap-typenum-to-usize-overflow")))] + mod arch32_bit_no_capping { + #[test] + fn sanity_check() { + super::sanity_check() + } + + #[test] + #[should_panic()] + fn ssz_bytes_len() { + super::ssz_bytes_len(); + } + + #[test] + #[should_panic()] + fn encode() { + super::encode(); + } + + #[test] + #[should_panic()] + fn encode_non_power_of_2() { + super::encode_non_power_of_2(); + } + + #[test] + #[should_panic()] + fn u16_len_2power40() { + super::u16_len_2power40() + } + + #[test] + #[should_panic()] + fn tree_hash_tests_hash256() { + super::AllTreeHashTests::::all_tests(); + } + + #[test] + #[should_panic()] + fn tree_hash_tests_u64() { + super::AllTreeHashTests::::all_tests(); + } + } + } }