mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-05-16 03:11:11 -04:00
71e59795c9
Remove the "ghash-neon" crypto_shash algorithm. Move the corresponding assembly code into lib/crypto/, and wire it up to the GHASH library. This makes the GHASH library be optimized on arm (though only with NEON, not PMULL; for now the goal is just parity with crypto_shash). It greatly reduces the amount of arm-specific glue code that is needed, and it fixes the issue where this optimization was disabled by default. To integrate the assembly code correctly with the library, make the following tweaks: - Change the type of 'blocks' from int to size_t. - Change the types of 'dg' and 'h' to polyval_elem. Note that this simply reflects the format that the code was already using, at least on little endian CPUs. For big endian CPUs, add byte-swaps. - Remove the 'head' argument, which is no longer needed. Acked-by: Ard Biesheuvel <ardb@kernel.org> Link: https://lore.kernel.org/r/20260319061723.1140720-8-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
44 lines
1.1 KiB
C
44 lines
1.1 KiB
C
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
|
/*
|
|
* GHASH, arm optimized
|
|
*
|
|
* Copyright 2026 Google LLC
|
|
*/
|
|
|
|
#include <asm/hwcap.h>
|
|
#include <asm/neon.h>
|
|
#include <asm/simd.h>
|
|
|
|
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
|
|
|
|
void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
|
|
const u8 *src, const struct polyval_elem *h);
|
|
|
|
#define ghash_blocks_arch ghash_blocks_arch
|
|
static void ghash_blocks_arch(struct polyval_elem *acc,
|
|
const struct ghash_key *key,
|
|
const u8 *data, size_t nblocks)
|
|
{
|
|
if (static_branch_likely(&have_neon) && may_use_simd()) {
|
|
do {
|
|
/* Allow rescheduling every 4 KiB. */
|
|
size_t n =
|
|
min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE);
|
|
|
|
scoped_ksimd()
|
|
pmull_ghash_update_p8(n, acc, data, &key->h);
|
|
data += n * GHASH_BLOCK_SIZE;
|
|
nblocks -= n;
|
|
} while (nblocks);
|
|
} else {
|
|
ghash_blocks_generic(acc, &key->h, data, nblocks);
|
|
}
|
|
}
|
|
|
|
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
|
|
static void gf128hash_mod_init_arch(void)
|
|
{
|
|
if (elf_hwcap & HWCAP_NEON)
|
|
static_branch_enable(&have_neon);
|
|
}
|