Skip to content

Improve VPCLMULQDQ to use 512-bit wide registers #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Improve VPCLMULQDQ to use 512-bit wide registers #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2e0fbff
Calculate 256-byte coefficients
onethumb Jun 7, 2025
c2f30c7
Use 4x512-bit registers instead of 2x256-bit
onethumb Jun 7, 2025
41b7600
Update benchmarks in README
onethumb Jun 7, 2025
6eaf13e
Merge branch 'main' into implement-vpclmulqdq-512-bits
onethumb Jun 7, 2025
75c2812
Merge branch 'main' into implement-vpclmulqdq-512-bits
onethumb Jun 7, 2025
8ae96e2
Increase benchmark sample size to 1000
onethumb Jun 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,12 +297,13 @@ AKA `crc32` in many, but not all, implementations.

### CRC-64/NVME

| Arch | Brand | CPU | System | Target | 1KiB (GiB/s) | 1MiB (GiB/s) |
|:--------|:------|:----------------|:---------------------|:----------------|-------------:|-------------:|
| x86_64 | Intel | Sapphire Rapids | EC2 c7i.metal-24xl | avx2_vpclmulqdq | ~22.9 | ~56.4 |
| x86_64 | AMD | Genoa | EC2 c7a.metal-48xl | avx2_vpclmulqdq | ~19.8 | ~27.4 |
| aarch64 | AWS | Graviton4 | EC2 c8g.metal-48xl | neon_pclmulqdq | ~20.0 | ~37.6 |
| aarch64 | Apple | M3 Ultra | Mac Studio (32 core) | neon_pclmulqdq | ~49.8 | ~71.9 |
| Arch | Brand | CPU | System | Target | 1KiB (GiB/s) | 1MiB (GiB/s) |
|:--------|:------|:----------------|:---------------------|:--------------------|-------------:|-------------:|
| x86_64 | Intel | Sapphire Rapids | EC2 c7i.metal-24xl | avx512_vpclmulqdq | ~24.9 | ~109.7 |
| x86_64 | AMD | Genoa | EC2 c7a.metal-48xl | avx512_vpclmulqdq | ~24.4 | ~54.6 |
| aarch64 | AWS | Graviton4 | EC2 c8g.metal-48xl | neon_pclmulqdq_eor3 | ~18.7 | ~36.8 |
| aarch64 | AWS | Graviton2 | EC2 c6g.metal | neon_pclmulqdq | ~9.8 | ~15.9 |
| aarch64 | Apple | M3 Ultra | Mac Studio (32 core) | neon_pclmulqdq_eor3 | ~49.5 | ~71.9 |

## Other CRC widths

Expand Down
28 changes: 26 additions & 2 deletions benches/benchmark.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,28 @@ fn random_data(size: i32) -> Vec<u8> {
buf
}

fn create_aligned_data(input: &[u8]) -> Vec<u8> {
// Size of our target alignment structure
let align_size = std::mem::size_of::<[[u64; 4]; 2]>(); // 64 bytes

// Create a vector with padding to ensure we can find a properly aligned position
let mut padded = Vec::with_capacity(input.len() + align_size);

// Fill with zeros initially to reach needed capacity
padded.resize(input.len() + align_size, 0);

// Find the first address that satisfies our alignment
let start_addr = padded.as_ptr() as usize;
let align_offset = (align_size - (start_addr % align_size)) % align_size;

// Copy the input into the aligned position
let aligned_start = &mut padded[align_offset..];
aligned_start[..input.len()].copy_from_slice(input);

// Return the exact slice we need
aligned_start[..input.len()].to_vec()
}

#[inline(always)]
fn bench_crc32(c: &mut Criterion) {
let mut group = c.benchmark_group("CRC-32");
Expand All @@ -65,7 +87,7 @@ fn bench_crc32(c: &mut Criterion) {
);

for (size_name, size) in SIZES {
let buf = random_data(*size);
let buf = create_aligned_data(&*random_data(*size));

let (part1, rest) = buf.split_at(buf.len() / 4);
let (part2, rest) = rest.split_at(rest.len() / 3);
Expand All @@ -78,6 +100,7 @@ fn bench_crc32(c: &mut Criterion) {
let alg_suffix = algorithm_name_parts.next();

group.throughput(Throughput::Bytes(*size as u64));
group.sample_size(1000);

let bench_name = [alg_suffix.unwrap(), "(checksum)"].join(" ");

Expand Down Expand Up @@ -108,7 +131,7 @@ fn bench_crc64(c: &mut Criterion) {
let mut group = c.benchmark_group("CRC-64");

for (size_name, size) in SIZES {
let buf = random_data(*size);
let buf = create_aligned_data(&*random_data(*size));

let (part1, rest) = buf.split_at(buf.len() / 4);
let (part2, rest) = rest.split_at(rest.len() / 3);
Expand All @@ -121,6 +144,7 @@ fn bench_crc64(c: &mut Criterion) {
let alg_suffix = algorithm_name_parts.next();

group.throughput(Throughput::Bytes(*size as u64));
group.sample_size(1000);

let bench_name = [alg_suffix.unwrap(), "(checksum)"].join(" ");

Expand Down
18 changes: 10 additions & 8 deletions src/algorithm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ unsafe fn process_by_strategy<T: ArchOps, W: EnhancedCrcWidth>(
data: &[u8],
state: &mut CrcState<T::Vector>,
reflector: Reflector<T::Vector>,
keys: [u64; 21],
keys: [u64; 23],
ops: &T,
) -> W::Value
where
Expand Down Expand Up @@ -129,7 +129,7 @@ unsafe fn process_large_aligned<T: ArchOps, W: EnhancedCrcWidth>(
bytes: &[u8],
state: &mut CrcState<T::Vector>,
reflector: Reflector<T::Vector>,
keys: [u64; 21],
keys: [u64; 23],
ops: &T,
) -> W::Value
where
Expand All @@ -149,7 +149,9 @@ where
}

// try to use the enhanced SIMD implementation first, fall back to non-enhanced if necessary
if !ops.process_enhanced_simd_blocks::<W>(state, first, rest, &reflector, keys) {
if rest.is_empty()
|| !ops.process_enhanced_simd_blocks::<W>(state, first, rest, &reflector, keys)
{
process_simd_chunks::<T, W>(state, first, rest, &reflector, keys, ops);
}

Expand Down Expand Up @@ -181,7 +183,7 @@ unsafe fn process_simd_chunks<T: ArchOps, W: EnhancedCrcWidth>(
first: &[T::Vector; 8],
rest: &[[T::Vector; 8]],
reflector: &Reflector<T::Vector>,
keys: [u64; 21],
keys: [u64; 23],
ops: &T,
) where
T::Vector: Copy,
Expand Down Expand Up @@ -260,7 +262,7 @@ unsafe fn process_exactly_16<T: ArchOps, W: EnhancedCrcWidth>(
data: &[u8],
state: &mut CrcState<T::Vector>,
reflector: &Reflector<T::Vector>,
keys: [u64; 21],
keys: [u64; 23],
ops: &T,
) -> W::Value
where
Expand Down Expand Up @@ -360,7 +362,7 @@ unsafe fn process_17_to_31<T: ArchOps, W: EnhancedCrcWidth>(
data: &[u8],
state: &mut CrcState<T::Vector>,
reflector: &Reflector<T::Vector>,
keys: [u64; 21],
keys: [u64; 23],
ops: &T,
) -> W::Value
where
Expand Down Expand Up @@ -399,7 +401,7 @@ unsafe fn process_32_to_255<T: ArchOps, W: EnhancedCrcWidth>(
data: &[u8],
state: &mut CrcState<T::Vector>,
reflector: &Reflector<T::Vector>,
keys: [u64; 21],
keys: [u64; 23],
ops: &T,
) -> W::Value
where
Expand Down Expand Up @@ -461,7 +463,7 @@ unsafe fn get_last_two_xmms<T: ArchOps, W: EnhancedCrcWidth>(
data: &[u8],
remaining_len: usize,
current_state: T::Vector,
keys: [u64; 21],
keys: [u64; 23],
reflector: &Reflector<T::Vector>,
reflected: bool,
ops: &T,
Expand Down
104 changes: 92 additions & 12 deletions src/arch/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ use crate::arch::aarch64::AArch64Ops;
use crate::arch::x86::X86Ops;

#[cfg(all(target_arch = "x86_64", feature = "vpclmulqdq"))]
use crate::arch::vpclmulqdq::VpclmulqdqOps;
use crate::arch::vpclmulqdq::Vpclmulqdq512Ops;

pub(crate) mod aarch64;
mod aarch64;
mod software;
mod vpclmulqdq;
pub(crate) mod x86;
mod x86;

/// Main entry point that dispatches to the appropriate architecture
///
Expand Down Expand Up @@ -61,14 +61,16 @@ pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: CrcParams) -> u64
use std::arch::is_x86_feature_detected;

if bytes.len() >= 256 && is_x86_feature_detected!("vpclmulqdq") {
let ops = vpclmulqdq::VpclmulqdqOps::new();
let ops = Vpclmulqdq512Ops::new();

return match params.width {
64 => algorithm::update::<VpclmulqdqOps, Width64>(state, bytes, params, &ops),
32 => {
algorithm::update::<VpclmulqdqOps, Width32>(state as u32, bytes, params, &ops)
as u64
}
64 => algorithm::update::<Vpclmulqdq512Ops, Width64>(state, bytes, params, &ops),
32 => algorithm::update::<Vpclmulqdq512Ops, Width32>(
state as u32,
bytes,
params,
&ops,
) as u64,
_ => panic!("Unsupported CRC width: {}", params.width),
};
}
Expand All @@ -90,11 +92,18 @@ pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: CrcParams) -> u64
}

pub fn get_target() -> String {
#[cfg(target_arch = "aarch64")]
#[cfg(all(target_arch = "aarch64", target_feature = "sha3"))]
return "internal-aarch64-neon-eor3".to_string();

#[cfg(all(target_arch = "aarch64", not(target_feature = "sha3")))]
return "internal-aarch64-neon".to_string();

#[cfg(all(target_arch = "x86_64", feature = "vpclmulqdq"))]
return "internal-x86_64-avx512-vpclmulqdq".to_string();
{
if is_x86_feature_detected!("vpclmulqdq") {
return "internal-x86_64-avx512-vpclmulqdq".to_string();
}
}

#[allow(unreachable_code)]
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Expand All @@ -110,6 +119,7 @@ mod tests {
use crate::crc32::consts::CRC32_BZIP2;
use crate::crc64::consts::CRC64_NVME;
use crate::test::consts::{TEST_256_BYTES_STRING, TEST_ALL_CONFIGS, TEST_CHECK_STRING};
use crate::test::create_aligned_data;
use rand::{rng, Rng};

#[test]
Expand All @@ -132,6 +142,76 @@ mod tests {
}
}

#[test]
fn test_256_string() {
for config in TEST_ALL_CONFIGS {
let actual = unsafe {
update(
config.get_init(),
&*create_aligned_data(TEST_256_BYTES_STRING),
*config.get_params(),
) ^ config.get_xorout()
};

assert_eq!(
actual,
config.checksum_with_reference(TEST_256_BYTES_STRING),
"Mismatch CRC, {}, expected {:#x}, got {:#x}",
config.get_name(),
config.get_check(),
actual
);
}
}

#[test]
fn test_512_string() {
let test_string = b"12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234561234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456";

for config in TEST_ALL_CONFIGS {
let actual = unsafe {
update(
config.get_init(),
&*create_aligned_data(test_string),
*config.get_params(),
) ^ config.get_xorout()
};

assert_eq!(
actual,
config.checksum_with_reference(test_string),
"Mismatch CRC, {}, expected {:#x}, got {:#x}",
config.get_name(),
config.get_check(),
actual
);
}
}

#[test]
fn test_1024_string() {
let test_string = b"1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345612345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234561234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456";

for config in TEST_ALL_CONFIGS {
let actual = unsafe {
update(
config.get_init(),
&*create_aligned_data(test_string),
*config.get_params(),
) ^ config.get_xorout()
};

assert_eq!(
actual,
config.checksum_with_reference(test_string),
"Mismatch CRC, {}, expected {:#x}, got {:#x}",
config.get_name(),
config.get_check(),
actual
);
}
}

// CRC-64/NVME is a special flower in that Rust's crc library doesn't support it yet, so we have
// tested values to check against.
#[test]
Expand Down Expand Up @@ -296,7 +376,7 @@ mod tests {
assert_eq!(
actual,
expected,
"\nFailed for {} with length {}\\nGot: {:016x}\nExpected: {:016x}",
"\nFailed for {} with length {}\nGot: {:016x}\nExpected: {:016x}",
config.get_name(),
len,
actual,
Expand Down
Loading