[PATCH] lib/crc: arm: Support partial blocks in NEON crc64_nvme
From: Shreesh Adiga
Date: Sun Nov 22 2026 - 03:21:33 EST
Provide vqtbl1q_u8() for AArch32 using two vtbl2_u8() lookups, the same
emulation used by lib/raid/raid6/arm/recov_neon_inner.c, and derive the
partial block blend mask with an arithmetic shift right by 7 instead of
the AArch64-only vcgezq_s8(). This lets fold_partial_block() be shared
by both architectures and removes the AArch32-specific requirement that
crc64_nvme_neon() be called with a length that is a multiple of 16.
The AArch32 dispatch code therefore no longer rounds chunks down to a
multiple of 16 bytes. Each chunk is now either 4 KiB or the whole
remaining length, so the tail is processed by NEON instead of falling
back to the generic table-based implementation.
Also document why the arm64 PMULL helpers use inline asm instead of the
intrinsics, and turn the 4-way fold loop into a do-while loop, since
callers guarantee len >= 128.
AArch32 was tested for correctness only (KUnit under QEMU), as no
AArch32-capable hardware was available for benchmarking.
Suggested-by: Eric Biggers <ebiggers@xxxxxxxxxx>
Signed-off-by: Shreesh Adiga <16567adigashreesh@xxxxxxxxx>
---
lib/crc/arm/crc64-neon.h | 16 ++++++++++++++++
lib/crc/arm/crc64.h | 4 ++--
lib/crc/arm64/crc64-neon.h | 12 ++++++++++++
lib/crc/crc64-neon.c | 24 ++++++++++++++----------
4 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/lib/crc/arm/crc64-neon.h b/lib/crc/arm/crc64-neon.h
index 645f553220ff..0d0071a92071 100644
--- a/lib/crc/arm/crc64-neon.h
+++ b/lib/crc/arm/crc64-neon.h
@@ -32,3 +32,19 @@ static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b)
return result;
}
+
+/*
+ * AArch32 does not provide this intrinsic natively because it does not
+ * implement the underlying instruction. AArch32 only provides a 64-bit
+ * wide vtbl.8 instruction, so use that instead.
+ */
+static inline uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b)
+{
+ union {
+ uint8x16_t val;
+ uint8x8x2_t pair;
+ } __a = { a };
+
+ return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)),
+ vtbl2_u8(__a.pair, vget_high_u8(b)));
+}
diff --git a/lib/crc/arm/crc64.h b/lib/crc/arm/crc64.h
index de274288af61..ce0ffe590596 100644
--- a/lib/crc/arm/crc64.h
+++ b/lib/crc/arm/crc64.h
@@ -13,10 +13,10 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
{
- if (len >= 128 && static_branch_likely(&have_pmull) &&
+ if (static_branch_likely(&have_pmull) && len >= 128 &&
likely(may_use_simd())) {
do {
- size_t chunk = min_t(size_t, len & ~15, SZ_4K);
+ size_t chunk = (len < SZ_4K + 128) ? len : SZ_4K;
scoped_ksimd()
crc = crc64_nvme_neon(crc, p, chunk);
diff --git a/lib/crc/arm64/crc64-neon.h b/lib/crc/arm64/crc64-neon.h
index 700a3a8619f4..d872f4d4e2e7 100644
--- a/lib/crc/arm64/crc64-neon.h
+++ b/lib/crc/arm64/crc64-neon.h
@@ -1,5 +1,17 @@
// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * The PMULL helpers below use inline asm instead of the vmull_p64() and
+ * vmull_high_p64() intrinsics. vmull_p64() takes poly64_t scalars, so the low
+ * lanes have to be extracted with vgetq_lane_u64(). For the fold constants,
+ * compilers fold that extraction into a 64-bit immediate: recent GCC and Clang
+ * build it in a GPR with mov/movk and transfer it with fmov (Clang also
+ * rebuilds the high half used by pmull2 with mov/movk and dup), instead of
+ * loading the constant pair with a single ldr. With asm operating on whole
+ * vector registers, the constants are loaded from .rodata and pmull reads
+ * the low lanes in place.
+ */
+
static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b)
{
uint64x2_t r;
diff --git a/lib/crc/crc64-neon.c b/lib/crc/crc64-neon.c
index 03b46d0abcc4..5b39d6ae1e9c 100644
--- a/lib/crc/crc64-neon.c
+++ b/lib/crc/crc64-neon.c
@@ -32,8 +32,15 @@ static inline uint64x2_t fold_single_round(uint64x2_t fold, uint64x2_t temp,
return temp ^ temp1 ^ temp2;
}
-#ifdef CONFIG_ARM64
-/* vqtbl1q_u8 is not available on AArch32. */
+/*
+ * Fold in the final 1 <= len <= 15 bytes. With A = fold, the remaining
+ * polynomial is A*x^(8*len) + B. Rewrite it as C1*x^128 + C2, where
+ * C1 = floor(A / x^(128 - 8*len)) and C2 = (A*x^(8*len) mod x^128) + B,
+ * then fold C1 into C2. t1 selects the len highest-degree bytes of A to
+ * form C1; ~t1 selects the remaining bytes to form A*x^(8*len) mod x^128,
+ * since tbl returns 0 for any index >= 16. The sign of t1 then selects
+ * which bytes of C2 come from B.
+ */
static inline uint64x2_t fold_partial_block(uint64x2_t fold, uint64x2_t consts,
const u8 *p, size_t len)
{
@@ -52,16 +59,15 @@ static inline uint64x2_t fold_partial_block(uint64x2_t fold, uint64x2_t consts,
a = vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(fold), t1));
t2 = vmvnq_u8(t1);
t2 = vqtbl1q_u8(vreinterpretq_u8_u64(fold), t2);
- t1 = vcgezq_s8(vreinterpretq_s8_u8(t1));
- b = vreinterpretq_u64_u8(vbslq_u8(t1, last16, t2));
+ t1 = vreinterpretq_u8_s8(vshrq_n_s8(vreinterpretq_s8_u8(t1), 7));
+ b = vreinterpretq_u64_u8(vbslq_u8(t1, t2, last16));
return fold_single_round(a, b, consts);
}
-#endif
u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
{
- /* Assumes len >= 64. AArch32 also assumes len is multiple of 16. */
+ /* Assumes len >= 128. */
uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
uint64x2_t fold_consts_4x = vld1q_u64(fold_consts_4x_val);
uint64x2_t fold_consts_2x = vld1q_u64(fold_consts_2x_val);
@@ -76,7 +82,7 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
p += 64;
len -= 64;
- while (len >= 64) {
+ do {
uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p + 0));
uint64x2_t t1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
uint64x2_t t2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
@@ -88,7 +94,7 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
v3 = fold_single_round(v3, t3, fold_consts_4x);
p += 64;
len -= 64;
- }
+ } while (len >= 64);
v0 = fold_single_round(v0, v2, fold_consts_2x);
v1 = fold_single_round(v1, v3, fold_consts_2x);
@@ -113,10 +119,8 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
len -= 16;
}
-#ifdef CONFIG_ARM64
if (len)
v0 = fold_partial_block(v0, fold_consts, p, len);
-#endif
/* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
--
2.56.1