From e49a3eac9207e9575337f70feeb29430f6f16bb7 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 13 Jun 2025 11:48:14 -0700 Subject: [PATCH 01/67] lib/crypto: Explicitly include Fix build warnings with W=1 that started appearing after commit a934a57a42f6 ("scripts/misc-check: check missing #include when W=1"). While at it, also sort the include lists alphabetically. (Keep asm/irqflags.h last, as otherwise it doesn't build on alpha.) This handles all of lib/crypto/, but not arch/*/lib/crypto/. The exports in arch/*/lib/crypto/ will go away when the code is properly integrated into lib/crypto/ as planned. Link: https://lore.kernel.org/r/20250613184814.50173-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/aes.c | 1 + lib/crypto/aescfb.c | 7 +++---- lib/crypto/aesgcm.c | 5 ++--- lib/crypto/arc4.c | 1 + lib/crypto/blake2s-generic.c | 9 +++++---- lib/crypto/blake2s.c | 9 +++++---- lib/crypto/chacha.c | 8 ++++---- lib/crypto/chacha20poly1305.c | 8 ++++---- lib/crypto/curve25519-generic.c | 1 + lib/crypto/des.c | 7 +++---- lib/crypto/gf128mul.c | 1 + lib/crypto/libchacha.c | 7 +++---- lib/crypto/memneq.c | 3 ++- lib/crypto/mpi/mpi-add.c | 2 ++ lib/crypto/mpi/mpi-bit.c | 2 ++ lib/crypto/mpi/mpi-cmp.c | 2 ++ lib/crypto/mpi/mpi-mul.c | 2 ++ lib/crypto/mpi/mpi-pow.c | 2 ++ lib/crypto/mpi/mpi-sub-ui.c | 2 ++ lib/crypto/mpi/mpicoder.c | 3 ++- lib/crypto/mpi/mpiutil.c | 2 ++ lib/crypto/poly1305-donna32.c | 3 ++- lib/crypto/poly1305-donna64.c | 3 ++- lib/crypto/poly1305-generic.c | 1 + lib/crypto/poly1305.c | 1 + lib/crypto/sha1.c | 10 +++++----- lib/crypto/sha256-generic.c | 1 + lib/crypto/sha256.c | 1 + lib/crypto/sm3.c | 1 + lib/crypto/utils.c | 3 ++- 30 files changed, 67 insertions(+), 41 deletions(-) diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index eafe14d021f5..b57fda3460f1 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -5,6 +5,7 @@ #include #include +#include #include #include diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 2f09ae92ffa0..0f294c8cbf3c 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -5,11 +5,10 @@ * Copyright 2023 Google LLC */ -#include - -#include #include - +#include +#include +#include #include static void aescfb_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst, diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index faa4dee9bb1b..ac0b2fcfd606 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -5,12 +5,11 @@ * Copyright 2022 Google LLC */ -#include - #include #include #include - +#include +#include #include static void aesgcm_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst, diff --git a/lib/crypto/arc4.c b/lib/crypto/arc4.c index 838812d18216..4e950e1e66d0 100644 --- a/lib/crypto/arc4.c +++ b/lib/crypto/arc4.c @@ -8,6 +8,7 @@ */ #include +#include #include int arc4_setkey(struct arc4_ctx *ctx, const u8 *in_key, unsigned int key_len) diff --git a/lib/crypto/blake2s-generic.c b/lib/crypto/blake2s-generic.c index 09682136b57c..9828176a2efe 100644 --- a/lib/crypto/blake2s-generic.c +++ b/lib/crypto/blake2s-generic.c @@ -9,11 +9,12 @@ */ #include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include static const u8 blake2s_sigma[10][16] = { diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c index b0f9a678300b..f6ec68c3dcda 100644 --- a/lib/crypto/blake2s.c +++ b/lib/crypto/blake2s.c @@ -9,12 +9,13 @@ */ #include -#include -#include +#include +#include +#include #include #include -#include -#include +#include +#include static inline void blake2s_set_lastblock(struct blake2s_state *state) { diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index ced87dd31a97..77f68de71066 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -5,13 +5,13 @@ * Copyright (C) 2015 Martin Willi */ -#include -#include -#include +#include #include +#include +#include +#include #include #include -#include static void chacha_permute(struct chacha_state *state, int nrounds) { diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index e29eed49a5a1..0b49d6aedefd 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -7,16 +7,16 @@ * Information: https://tools.ietf.org/html/rfc8439 */ -#include #include +#include #include #include - -#include -#include +#include #include +#include #include #include +#include static void chacha_load_key(u32 *k, const u8 *in) { diff --git a/lib/crypto/curve25519-generic.c b/lib/crypto/curve25519-generic.c index de7c99172fa2..f8aa70c9f559 100644 --- a/lib/crypto/curve25519-generic.c +++ b/lib/crypto/curve25519-generic.c @@ -10,6 +10,7 @@ */ #include +#include #include const u8 curve25519_null_point[CURVE25519_KEY_SIZE] __aligned(32) = { 0 }; diff --git a/lib/crypto/des.c b/lib/crypto/des.c index d3423b34a8e9..a906070136dc 100644 --- a/lib/crypto/des.c +++ b/lib/crypto/des.c @@ -7,21 +7,20 @@ * Copyright (c) 2005 Dag Arne Osvik */ +#include +#include #include #include #include #include +#include #include #include #include #include #include - #include -#include -#include - #define ROL(x, r) ((x) = rol32((x), (r))) #define ROR(x, r) ((x) = ror32((x), (r))) diff --git a/lib/crypto/gf128mul.c b/lib/crypto/gf128mul.c index fbe72cb3453a..2a34590fe3f1 100644 --- a/lib/crypto/gf128mul.c +++ b/lib/crypto/gf128mul.c @@ -49,6 +49,7 @@ */ #include +#include #include #include #include diff --git a/lib/crypto/libchacha.c b/lib/crypto/libchacha.c index ebcca381e248..26862ad90a96 100644 --- a/lib/crypto/libchacha.c +++ b/lib/crypto/libchacha.c @@ -5,12 +5,11 @@ * Copyright (C) 2015 Martin Willi */ -#include -#include -#include - #include // for crypto_xor_cpy #include +#include +#include +#include void chacha_crypt_generic(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c index a2afd10349c9..44daacb8cb51 100644 --- a/lib/crypto/memneq.c +++ b/lib/crypto/memneq.c @@ -59,9 +59,10 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include #include +#include #include +#include /* Generic path for arbitrary size */ static inline unsigned long diff --git a/lib/crypto/mpi/mpi-add.c b/lib/crypto/mpi/mpi-add.c index 3015140d4860..c0375c1672fa 100644 --- a/lib/crypto/mpi/mpi-add.c +++ b/lib/crypto/mpi/mpi-add.c @@ -11,6 +11,8 @@ * to avoid revealing of sensitive data due to paging etc. */ +#include + #include "mpi-internal.h" int mpi_add(MPI w, MPI u, MPI v) diff --git a/lib/crypto/mpi/mpi-bit.c b/lib/crypto/mpi/mpi-bit.c index 934d81311360..b3d0e7ddbc03 100644 --- a/lib/crypto/mpi/mpi-bit.c +++ b/lib/crypto/mpi/mpi-bit.c @@ -18,6 +18,8 @@ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA */ +#include + #include "mpi-internal.h" #include "longlong.h" diff --git a/lib/crypto/mpi/mpi-cmp.c b/lib/crypto/mpi/mpi-cmp.c index ceaebe181cd7..b42929296bce 100644 --- a/lib/crypto/mpi/mpi-cmp.c +++ b/lib/crypto/mpi/mpi-cmp.c @@ -18,6 +18,8 @@ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA */ +#include + #include "mpi-internal.h" int mpi_cmp_ui(MPI u, unsigned long v) diff --git a/lib/crypto/mpi/mpi-mul.c b/lib/crypto/mpi/mpi-mul.c index 7e6ff1ce3e9b..d79f186ad90b 100644 --- a/lib/crypto/mpi/mpi-mul.c +++ b/lib/crypto/mpi/mpi-mul.c @@ -11,6 +11,8 @@ * to avoid revealing of sensitive data due to paging etc. */ +#include + #include "mpi-internal.h" int mpi_mul(MPI w, MPI u, MPI v) diff --git a/lib/crypto/mpi/mpi-pow.c b/lib/crypto/mpi/mpi-pow.c index 67fbd4c2503d..9e695a3bda3a 100644 --- a/lib/crypto/mpi/mpi-pow.c +++ b/lib/crypto/mpi/mpi-pow.c @@ -13,8 +13,10 @@ * however I decided to publish this code under the plain GPL. */ +#include #include #include + #include "mpi-internal.h" #include "longlong.h" diff --git a/lib/crypto/mpi/mpi-sub-ui.c b/lib/crypto/mpi/mpi-sub-ui.c index b41b082b5f3e..0edcdbd24833 100644 --- a/lib/crypto/mpi/mpi-sub-ui.c +++ b/lib/crypto/mpi/mpi-sub-ui.c @@ -32,6 +32,8 @@ * see https://www.gnu.org/licenses/. */ +#include + #include "mpi-internal.h" int mpi_sub_ui(MPI w, MPI u, unsigned long vval) diff --git a/lib/crypto/mpi/mpicoder.c b/lib/crypto/mpi/mpicoder.c index dde01030807d..47f6939599b3 100644 --- a/lib/crypto/mpi/mpicoder.c +++ b/lib/crypto/mpi/mpicoder.c @@ -19,8 +19,9 @@ */ #include -#include #include +#include +#include #include #include #include "mpi-internal.h" diff --git a/lib/crypto/mpi/mpiutil.c b/lib/crypto/mpi/mpiutil.c index 979ece5a81d2..7f2db830f404 100644 --- a/lib/crypto/mpi/mpiutil.c +++ b/lib/crypto/mpi/mpiutil.c @@ -18,6 +18,8 @@ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA */ +#include + #include "mpi-internal.h" /**************** diff --git a/lib/crypto/poly1305-donna32.c b/lib/crypto/poly1305-donna32.c index 0a4a2d99e365..b66131b3f6d4 100644 --- a/lib/crypto/poly1305-donna32.c +++ b/lib/crypto/poly1305-donna32.c @@ -6,9 +6,10 @@ * public domain. */ +#include +#include #include #include -#include void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[POLY1305_BLOCK_SIZE]) diff --git a/lib/crypto/poly1305-donna64.c b/lib/crypto/poly1305-donna64.c index 530287531b2e..8a72a5a84944 100644 --- a/lib/crypto/poly1305-donna64.c +++ b/lib/crypto/poly1305-donna64.c @@ -6,9 +6,10 @@ * public domain. */ +#include +#include #include #include -#include void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[POLY1305_BLOCK_SIZE]) diff --git a/lib/crypto/poly1305-generic.c b/lib/crypto/poly1305-generic.c index a73f700fa1fb..71a16c5c538b 100644 --- a/lib/crypto/poly1305-generic.c +++ b/lib/crypto/poly1305-generic.c @@ -8,6 +8,7 @@ */ #include +#include #include #include diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 5f2f2af3b59f..a6dc182b6c22 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index ebb60519ae93..6d809c3088be 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -6,12 +6,12 @@ * This was based on the git SHA1 implementation. */ -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include /* diff --git a/lib/crypto/sha256-generic.c b/lib/crypto/sha256-generic.c index a16ad4f25ebb..2968d95d0403 100644 --- a/lib/crypto/sha256-generic.c +++ b/lib/crypto/sha256-generic.c @@ -12,6 +12,7 @@ */ #include +#include #include #include #include diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 107e5162507a..6bfa4ae8dfb5 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c index efff0e267d84..c6b9ad8a3ac6 100644 --- a/lib/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include diff --git a/lib/crypto/utils.c b/lib/crypto/utils.c index 87da2a6dd161..dec381d5e906 100644 --- a/lib/crypto/utils.c +++ b/lib/crypto/utils.c @@ -5,9 +5,10 @@ * Copyright (c) 2006 Herbert Xu */ -#include #include +#include #include +#include /* * XOR @len bytes from @src1 and @src2 together, writing the result to @dst From e0fca177556c4419819876ac5a947b0844115d56 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:05 -0700 Subject: [PATCH 02/67] crypto: sha512 - Rename conflicting symbols Rename existing functions and structs in architecture-optimized SHA-512 code that had names conflicting with the upcoming library interface which will be added to : sha384_init, sha512_init, sha512_update, sha384, and sha512. Note: all affected code will be superseded by later commits that migrate the arch-optimized SHA-512 code into the library. This commit simply keeps the kernel building for the initial introduction of the library. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/crypto/sha512-glue.c | 8 ++++---- arch/s390/crypto/sha512_s390.c | 8 ++++---- arch/sparc/crypto/sha512_glue.c | 14 +++++++------- arch/x86/crypto/sha512_ssse3_glue.c | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index 15aa9d8b7b2c..a78e184c100f 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -27,8 +27,8 @@ static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, sha512_blocks_arch(sst->state, src, blocks); } -static int sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha512_update_arm64(struct shash_desc *desc, const u8 *data, + unsigned int len) { return sha512_base_do_update_blocks(desc, data, len, sha512_arm64_transform); @@ -44,7 +44,7 @@ static int sha512_finup(struct shash_desc *desc, const u8 *data, static struct shash_alg algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, - .update = sha512_update, + .update = sha512_update_arm64, .finup = sha512_finup, .descsize = SHA512_STATE_SIZE, .base.cra_name = "sha512", @@ -57,7 +57,7 @@ static struct shash_alg algs[] = { { }, { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, - .update = sha512_update, + .update = sha512_update_arm64, .finup = sha512_finup, .descsize = SHA512_STATE_SIZE, .base.cra_name = "sha384", diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 33711a29618c..e8bb172dbed7 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -17,7 +17,7 @@ #include "sha.h" -static int sha512_init(struct shash_desc *desc) +static int sha512_init_s390(struct shash_desc *desc) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); @@ -62,7 +62,7 @@ static int sha512_import(struct shash_desc *desc, const void *in) static struct shash_alg sha512_alg = { .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_init, + .init = sha512_init_s390, .update = s390_sha_update_blocks, .finup = s390_sha_finup, .export = sha512_export, @@ -82,7 +82,7 @@ static struct shash_alg sha512_alg = { MODULE_ALIAS_CRYPTO("sha512"); -static int sha384_init(struct shash_desc *desc) +static int sha384_init_s390(struct shash_desc *desc) { struct s390_sha_ctx *ctx = shash_desc_ctx(desc); @@ -103,7 +103,7 @@ static int sha384_init(struct shash_desc *desc) static struct shash_alg sha384_alg = { .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_init, + .init = sha384_init_s390, .update = s390_sha_update_blocks, .finup = s390_sha_finup, .export = sha512_export, diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index 47b9277b6877..fb81c3290c8c 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -40,7 +40,7 @@ static int sha512_sparc64_finup(struct shash_desc *desc, const u8 *src, return sha512_base_finish(desc, out); } -static struct shash_alg sha512 = { +static struct shash_alg sha512_alg = { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_sparc64_update, @@ -55,7 +55,7 @@ static struct shash_alg sha512 = { } }; -static struct shash_alg sha384 = { +static struct shash_alg sha384_alg = { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_sparc64_update, @@ -87,13 +87,13 @@ static bool __init sparc64_has_sha512_opcode(void) static int __init sha512_sparc64_mod_init(void) { if (sparc64_has_sha512_opcode()) { - int ret = crypto_register_shash(&sha384); + int ret = crypto_register_shash(&sha384_alg); if (ret < 0) return ret; - ret = crypto_register_shash(&sha512); + ret = crypto_register_shash(&sha512_alg); if (ret < 0) { - crypto_unregister_shash(&sha384); + crypto_unregister_shash(&sha384_alg); return ret; } @@ -106,8 +106,8 @@ static int __init sha512_sparc64_mod_init(void) static void __exit sha512_sparc64_mod_fini(void) { - crypto_unregister_shash(&sha384); - crypto_unregister_shash(&sha512); + crypto_unregister_shash(&sha384_alg); + crypto_unregister_shash(&sha512_alg); } module_init(sha512_sparc64_mod_init); diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index 067684c54395..97744b7d2381 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -38,8 +38,8 @@ asmlinkage void sha512_transform_ssse3(struct sha512_state *state, const u8 *data, int blocks); -static int sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len, sha512_block_fn *sha512_xform) +static int sha512_update_x86(struct shash_desc *desc, const u8 *data, + unsigned int len, sha512_block_fn *sha512_xform) { int remain; @@ -69,7 +69,7 @@ static int sha512_finup(struct shash_desc *desc, const u8 *data, static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha512_update(desc, data, len, sha512_transform_ssse3); + return sha512_update_x86(desc, data, len, sha512_transform_ssse3); } static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, @@ -141,7 +141,7 @@ static bool avx_usable(void) static int sha512_avx_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha512_update(desc, data, len, sha512_transform_avx); + return sha512_update_x86(desc, data, len, sha512_transform_avx); } static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, @@ -203,7 +203,7 @@ asmlinkage void sha512_transform_rorx(struct sha512_state *state, static int sha512_avx2_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha512_update(desc, data, len, sha512_transform_rorx); + return sha512_update_x86(desc, data, len, sha512_transform_rorx); } static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, From b693c703accb08cbd52f0b94d810d6abbca3bfb9 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:06 -0700 Subject: [PATCH 03/67] lib/crypto: sha512: Add support for SHA-384 and SHA-512 Add basic support for SHA-384 and SHA-512 to lib/crypto/. Various in-kernel users will be able to use this instead of the old-school crypto API, which is harder to use and has more overhead. The basic support added by this commit consists of the API and its documentation, backed by a C implementation of the algorithms. sha512_block_generic() is derived from crypto/sha512_generic.c. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 128 ++++++++++++++++++++ lib/crypto/Kconfig | 10 ++ lib/crypto/Makefile | 6 + lib/crypto/sha512.c | 265 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 409 insertions(+) create mode 100644 lib/crypto/sha512.c diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 4912572578dc..f2a6e84b2840 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -129,4 +129,132 @@ static inline void sha224_init(struct sha256_state *sctx) /* Simply use sha256_update as it is equivalent to sha224_update. */ void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); +/* State for the SHA-512 (and SHA-384) compression function */ +struct sha512_block_state { + u64 h[8]; +}; + +/* + * Context structure, shared by SHA-384 and SHA-512. The sha384_ctx and + * sha512_ctx structs wrap this one so that the API has proper typing and + * doesn't allow mixing the SHA-384 and SHA-512 functions arbitrarily. + */ +struct __sha512_ctx { + struct sha512_block_state state; + u64 bytecount_lo; + u64 bytecount_hi; + u8 buf[SHA512_BLOCK_SIZE] __aligned(__alignof__(__be64)); +}; +void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len); + +/** + * struct sha384_ctx - Context for hashing a message with SHA-384 + * @ctx: private + */ +struct sha384_ctx { + struct __sha512_ctx ctx; +}; + +/** + * sha384_init() - Initialize a SHA-384 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha384() instead. + * + * Context: Any context. + */ +void sha384_init(struct sha384_ctx *ctx); + +/** + * sha384_update() - Update a SHA-384 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void sha384_update(struct sha384_ctx *ctx, + const u8 *data, size_t len) +{ + __sha512_update(&ctx->ctx, data, len); +} + +/** + * sha384_final() - Finish computing a SHA-384 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-384 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha384_final(struct sha384_ctx *ctx, u8 out[SHA384_DIGEST_SIZE]); + +/** + * sha384() - Compute SHA-384 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-384 message digest + * + * Context: Any context. + */ +void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE]); + +/** + * struct sha512_ctx - Context for hashing a message with SHA-512 + * @ctx: private + */ +struct sha512_ctx { + struct __sha512_ctx ctx; +}; + +/** + * sha512_init() - Initialize a SHA-512 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha512() instead. + * + * Context: Any context. + */ +void sha512_init(struct sha512_ctx *ctx); + +/** + * sha512_update() - Update a SHA-512 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void sha512_update(struct sha512_ctx *ctx, + const u8 *data, size_t len) +{ + __sha512_update(&ctx->ctx, data, len); +} + +/** + * sha512_final() - Finish computing a SHA-512 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-512 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha512_final(struct sha512_ctx *ctx, u8 out[SHA512_DIGEST_SIZE]); + +/** + * sha512() - Compute SHA-512 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-512 message digest + * + * Context: Any context. + */ +void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]); + #endif /* _CRYPTO_SHA2_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 1ec1466108cc..2d295c0e0f79 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -167,6 +167,16 @@ config CRYPTO_LIB_SHA256_GENERIC for SIMD implementations. If no arch specific implementation is enabled, this implementation serves the users of CRYPTO_LIB_SHA256. +config CRYPTO_LIB_SHA512 + tristate + help + The SHA-384 and SHA-512 library functions. Select this if your module + uses any of these functions from . + +config CRYPTO_LIB_SHA512_ARCH + bool + depends on CRYPTO_LIB_SHA512 && !UML + config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index b0c0f8aea269..f6b6f370451e 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -64,6 +64,12 @@ libsha256-y := sha256.o obj-$(CONFIG_CRYPTO_LIB_SHA256_GENERIC) += libsha256-generic.o libsha256-generic-y := sha256-generic.o +obj-$(CONFIG_CRYPTO_LIB_SHA512) += libsha512.o +libsha512-y := sha512.o +ifeq ($(CONFIG_CRYPTO_LIB_SHA512_ARCH),y) +CFLAGS_sha512.o += -I$(src)/$(SRCARCH) +endif # CONFIG_CRYPTO_LIB_SHA512_ARCH + obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c new file mode 100644 index 000000000000..536b71481b1c --- /dev/null +++ b/lib/crypto/sha512.c @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-384 and SHA-512 library functions + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2003 Kyle McMartin + * Copyright 2025 Google LLC + */ + +#include +#include +#include +#include +#include +#include +#include + +static const struct sha512_block_state sha384_iv = { + .h = { + SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3, + SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7, + }, +}; + +static const struct sha512_block_state sha512_iv = { + .h = { + SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3, + SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7, + }, +}; + +static const u64 sha512_K[80] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, + 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL, + 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, + 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL, + 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, + 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL, + 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, + 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL, + 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, + 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, + 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, + 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL, + 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, + 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) +#define Maj(x, y, z) (((x) & (y)) | ((z) & ((x) | (y)))) +#define e0(x) (ror64((x), 28) ^ ror64((x), 34) ^ ror64((x), 39)) +#define e1(x) (ror64((x), 14) ^ ror64((x), 18) ^ ror64((x), 41)) +#define s0(x) (ror64((x), 1) ^ ror64((x), 8) ^ ((x) >> 7)) +#define s1(x) (ror64((x), 19) ^ ror64((x), 61) ^ ((x) >> 6)) + +static void sha512_block_generic(struct sha512_block_state *state, + const u8 *data) +{ + u64 a = state->h[0]; + u64 b = state->h[1]; + u64 c = state->h[2]; + u64 d = state->h[3]; + u64 e = state->h[4]; + u64 f = state->h[5]; + u64 g = state->h[6]; + u64 h = state->h[7]; + u64 t1, t2; + u64 W[16]; + + for (int j = 0; j < 16; j++) + W[j] = get_unaligned_be64(data + j * sizeof(u64)); + + for (int i = 0; i < 80; i += 8) { + if ((i & 15) == 0 && i != 0) { + for (int j = 0; j < 16; j++) { + W[j & 15] += s1(W[(j - 2) & 15]) + + W[(j - 7) & 15] + + s0(W[(j - 15) & 15]); + } + } + t1 = h + e1(e) + Ch(e, f, g) + sha512_K[i] + W[(i & 15)]; + t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; + t1 = g + e1(d) + Ch(d, e, f) + sha512_K[i+1] + W[(i & 15) + 1]; + t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; + t1 = f + e1(c) + Ch(c, d, e) + sha512_K[i+2] + W[(i & 15) + 2]; + t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; + t1 = e + e1(b) + Ch(b, c, d) + sha512_K[i+3] + W[(i & 15) + 3]; + t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; + t1 = d + e1(a) + Ch(a, b, c) + sha512_K[i+4] + W[(i & 15) + 4]; + t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; + t1 = c + e1(h) + Ch(h, a, b) + sha512_K[i+5] + W[(i & 15) + 5]; + t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; + t1 = b + e1(g) + Ch(g, h, a) + sha512_K[i+6] + W[(i & 15) + 6]; + t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; + t1 = a + e1(f) + Ch(f, g, h) + sha512_K[i+7] + W[(i & 15) + 7]; + t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; + } + + state->h[0] += a; + state->h[1] += b; + state->h[2] += c; + state->h[3] += d; + state->h[4] += e; + state->h[5] += f; + state->h[6] += g; + state->h[7] += h; +} + +static void __maybe_unused +sha512_blocks_generic(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + do { + sha512_block_generic(state, data); + data += SHA512_BLOCK_SIZE; + } while (--nblocks); +} + +#ifdef CONFIG_CRYPTO_LIB_SHA512_ARCH +#include "sha512.h" /* $(SRCARCH)/sha512.h */ +#else +#define sha512_blocks sha512_blocks_generic +#endif + +static void __sha512_init(struct __sha512_ctx *ctx, + const struct sha512_block_state *iv, + u64 initial_bytecount) +{ + ctx->state = *iv; + ctx->bytecount_lo = initial_bytecount; + ctx->bytecount_hi = 0; +} + +void sha384_init(struct sha384_ctx *ctx) +{ + __sha512_init(&ctx->ctx, &sha384_iv, 0); +} +EXPORT_SYMBOL_GPL(sha384_init); + +void sha512_init(struct sha512_ctx *ctx) +{ + __sha512_init(&ctx->ctx, &sha512_iv, 0); +} +EXPORT_SYMBOL_GPL(sha512_init); + +void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len) +{ + size_t partial = ctx->bytecount_lo % SHA512_BLOCK_SIZE; + + if (check_add_overflow(ctx->bytecount_lo, len, &ctx->bytecount_lo)) + ctx->bytecount_hi++; + + if (partial + len >= SHA512_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l = SHA512_BLOCK_SIZE - partial; + + memcpy(&ctx->buf[partial], data, l); + data += l; + len -= l; + + sha512_blocks(&ctx->state, ctx->buf, 1); + } + + nblocks = len / SHA512_BLOCK_SIZE; + len %= SHA512_BLOCK_SIZE; + + if (nblocks) { + sha512_blocks(&ctx->state, data, nblocks); + data += nblocks * SHA512_BLOCK_SIZE; + } + partial = 0; + } + if (len) + memcpy(&ctx->buf[partial], data, len); +} +EXPORT_SYMBOL_GPL(__sha512_update); + +static void __sha512_final(struct __sha512_ctx *ctx, + u8 *out, size_t digest_size) +{ + u64 bitcount_hi = (ctx->bytecount_hi << 3) | (ctx->bytecount_lo >> 61); + u64 bitcount_lo = ctx->bytecount_lo << 3; + size_t partial = ctx->bytecount_lo % SHA512_BLOCK_SIZE; + + ctx->buf[partial++] = 0x80; + if (partial > SHA512_BLOCK_SIZE - 16) { + memset(&ctx->buf[partial], 0, SHA512_BLOCK_SIZE - partial); + sha512_blocks(&ctx->state, ctx->buf, 1); + partial = 0; + } + memset(&ctx->buf[partial], 0, SHA512_BLOCK_SIZE - 16 - partial); + *(__be64 *)&ctx->buf[SHA512_BLOCK_SIZE - 16] = cpu_to_be64(bitcount_hi); + *(__be64 *)&ctx->buf[SHA512_BLOCK_SIZE - 8] = cpu_to_be64(bitcount_lo); + sha512_blocks(&ctx->state, ctx->buf, 1); + + for (size_t i = 0; i < digest_size; i += 8) + put_unaligned_be64(ctx->state.h[i / 8], out + i); +} + +void sha384_final(struct sha384_ctx *ctx, u8 out[SHA384_DIGEST_SIZE]) +{ + __sha512_final(&ctx->ctx, out, SHA384_DIGEST_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sha384_final); + +void sha512_final(struct sha512_ctx *ctx, u8 out[SHA512_DIGEST_SIZE]) +{ + __sha512_final(&ctx->ctx, out, SHA512_DIGEST_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sha512_final); + +void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE]) +{ + struct sha384_ctx ctx; + + sha384_init(&ctx); + sha384_update(&ctx, data, len); + sha384_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sha384); + +void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]) +{ + struct sha512_ctx ctx; + + sha512_init(&ctx); + sha512_update(&ctx, data, len); + sha512_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sha512); + +#ifdef sha512_mod_init_arch +static int __init sha512_mod_init(void) +{ + sha512_mod_init_arch(); + return 0; +} +subsys_initcall(sha512_mod_init); + +static void __exit sha512_mod_exit(void) +{ +} +module_exit(sha512_mod_exit); +#endif + +MODULE_DESCRIPTION("SHA-384 and SHA-512 library functions"); +MODULE_LICENSE("GPL"); From 23e8b4371dbd5907d633262f36903144a378a114 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:07 -0700 Subject: [PATCH 04/67] lib/crypto: sha512: Add HMAC-SHA384 and HMAC-SHA512 support Since HMAC support is commonly needed and is fairly simple, include it as a first-class citizen of the SHA-512 library. The API supports both incremental and one-shot computation, and either preparing the key ahead of time or just using a raw key. The implementation is much more streamlined than crypto/hmac.c. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 222 ++++++++++++++++++++++++++++++++++++++++++ lib/crypto/Kconfig | 5 +- lib/crypto/sha512.c | 141 ++++++++++++++++++++++++++- 3 files changed, 364 insertions(+), 4 deletions(-) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index f2a6e84b2840..296ce9d468bf 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -147,6 +147,22 @@ struct __sha512_ctx { }; void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len); +/* + * HMAC key and message context structs, shared by HMAC-SHA384 and HMAC-SHA512. + * The hmac_sha384_* and hmac_sha512_* structs wrap this one so that the API has + * proper typing and doesn't allow mixing the functions arbitrarily. + */ +struct __hmac_sha512_key { + struct sha512_block_state istate; + struct sha512_block_state ostate; +}; +struct __hmac_sha512_ctx { + struct __sha512_ctx sha_ctx; + struct sha512_block_state ostate; +}; +void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx, + const struct __hmac_sha512_key *key); + /** * struct sha384_ctx - Context for hashing a message with SHA-384 * @ctx: private @@ -202,6 +218,109 @@ void sha384_final(struct sha384_ctx *ctx, u8 out[SHA384_DIGEST_SIZE]); */ void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE]); +/** + * struct hmac_sha384_key - Prepared key for HMAC-SHA384 + * @key: private + */ +struct hmac_sha384_key { + struct __hmac_sha512_key key; +}; + +/** + * struct hmac_sha384_ctx - Context for computing HMAC-SHA384 of a message + * @ctx: private + */ +struct hmac_sha384_ctx { + struct __hmac_sha512_ctx ctx; +}; + +/** + * hmac_sha384_preparekey() - Prepare a key for HMAC-SHA384 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA384 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha384_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha384_preparekey(struct hmac_sha384_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha384_init() - Initialize an HMAC-SHA384 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha384() instead. + * + * Context: Any context. + */ +static inline void hmac_sha384_init(struct hmac_sha384_ctx *ctx, + const struct hmac_sha384_key *key) +{ + __hmac_sha512_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha384_update() - Update an HMAC-SHA384 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha384_update(struct hmac_sha384_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha512_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha384_final() - Finish computing an HMAC-SHA384 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA384 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha384_final(struct hmac_sha384_ctx *ctx, u8 out[SHA384_DIGEST_SIZE]); + +/** + * hmac_sha384() - Compute HMAC-SHA384 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA384 value + * + * If you're using the key only once, consider using hmac_sha384_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha384(const struct hmac_sha384_key *key, + const u8 *data, size_t data_len, u8 out[SHA384_DIGEST_SIZE]); + +/** + * hmac_sha384_usingrawkey() - Compute HMAC-SHA384 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA384 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA384 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha384_preparekey() followed by multiple calls to hmac_sha384() instead. + * + * Context: Any context. + */ +void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA384_DIGEST_SIZE]); + /** * struct sha512_ctx - Context for hashing a message with SHA-512 * @ctx: private @@ -257,4 +376,107 @@ void sha512_final(struct sha512_ctx *ctx, u8 out[SHA512_DIGEST_SIZE]); */ void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]); +/** + * struct hmac_sha512_key - Prepared key for HMAC-SHA512 + * @key: private + */ +struct hmac_sha512_key { + struct __hmac_sha512_key key; +}; + +/** + * struct hmac_sha512_ctx - Context for computing HMAC-SHA512 of a message + * @ctx: private + */ +struct hmac_sha512_ctx { + struct __hmac_sha512_ctx ctx; +}; + +/** + * hmac_sha512_preparekey() - Prepare a key for HMAC-SHA512 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA512 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha512_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha512_preparekey(struct hmac_sha512_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha512_init() - Initialize an HMAC-SHA512 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha512() instead. + * + * Context: Any context. + */ +static inline void hmac_sha512_init(struct hmac_sha512_ctx *ctx, + const struct hmac_sha512_key *key) +{ + __hmac_sha512_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha512_update() - Update an HMAC-SHA512 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha512_update(struct hmac_sha512_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha512_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha512_final() - Finish computing an HMAC-SHA512 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA512 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha512_final(struct hmac_sha512_ctx *ctx, u8 out[SHA512_DIGEST_SIZE]); + +/** + * hmac_sha512() - Compute HMAC-SHA512 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA512 value + * + * If you're using the key only once, consider using hmac_sha512_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha512(const struct hmac_sha512_key *key, + const u8 *data, size_t data_len, u8 out[SHA512_DIGEST_SIZE]); + +/** + * hmac_sha512_usingrawkey() - Compute HMAC-SHA512 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA512 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA512 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha512_preparekey() followed by multiple calls to hmac_sha512() instead. + * + * Context: Any context. + */ +void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA512_DIGEST_SIZE]); + #endif /* _CRYPTO_SHA2_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 2d295c0e0f79..d1bee3787eb3 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -170,8 +170,9 @@ config CRYPTO_LIB_SHA256_GENERIC config CRYPTO_LIB_SHA512 tristate help - The SHA-384 and SHA-512 library functions. Select this if your module - uses any of these functions from . + The SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions. + Select this if your module uses any of these functions from + . config CRYPTO_LIB_SHA512_ARCH bool diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c index 536b71481b1c..d514721491ca 100644 --- a/lib/crypto/sha512.c +++ b/lib/crypto/sha512.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * SHA-384 and SHA-512 library functions + * SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions * * Copyright (c) Jean-Luc Cooke * Copyright (c) Andrew McDonald @@ -8,6 +8,7 @@ * Copyright 2025 Google LLC */ +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include static const struct sha512_block_state sha384_iv = { .h = { @@ -247,6 +249,141 @@ void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]) } EXPORT_SYMBOL_GPL(sha512); +static void __hmac_sha512_preparekey(struct __hmac_sha512_key *key, + const u8 *raw_key, size_t raw_key_len, + const struct sha512_block_state *iv) +{ + union { + u8 b[SHA512_BLOCK_SIZE]; + unsigned long w[SHA512_BLOCK_SIZE / sizeof(unsigned long)]; + } derived_key = { 0 }; + + if (unlikely(raw_key_len > SHA512_BLOCK_SIZE)) { + if (iv == &sha384_iv) + sha384(raw_key, raw_key_len, derived_key.b); + else + sha512(raw_key, raw_key_len, derived_key.b); + } else { + memcpy(derived_key.b, raw_key, raw_key_len); + } + + for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) + derived_key.w[i] ^= REPEAT_BYTE(HMAC_IPAD_VALUE); + key->istate = *iv; + sha512_blocks(&key->istate, derived_key.b, 1); + + for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) + derived_key.w[i] ^= REPEAT_BYTE(HMAC_OPAD_VALUE ^ + HMAC_IPAD_VALUE); + key->ostate = *iv; + sha512_blocks(&key->ostate, derived_key.b, 1); + + memzero_explicit(&derived_key, sizeof(derived_key)); +} + +void hmac_sha384_preparekey(struct hmac_sha384_key *key, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha512_preparekey(&key->key, raw_key, raw_key_len, &sha384_iv); +} +EXPORT_SYMBOL_GPL(hmac_sha384_preparekey); + +void hmac_sha512_preparekey(struct hmac_sha512_key *key, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha512_preparekey(&key->key, raw_key, raw_key_len, &sha512_iv); +} +EXPORT_SYMBOL_GPL(hmac_sha512_preparekey); + +void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx, + const struct __hmac_sha512_key *key) +{ + __sha512_init(&ctx->sha_ctx, &key->istate, SHA512_BLOCK_SIZE); + ctx->ostate = key->ostate; +} +EXPORT_SYMBOL_GPL(__hmac_sha512_init); + +static void __hmac_sha512_final(struct __hmac_sha512_ctx *ctx, + u8 *out, size_t digest_size) +{ + /* Generate the padded input for the outer hash in ctx->sha_ctx.buf. */ + __sha512_final(&ctx->sha_ctx, ctx->sha_ctx.buf, digest_size); + memset(&ctx->sha_ctx.buf[digest_size], 0, + SHA512_BLOCK_SIZE - digest_size); + ctx->sha_ctx.buf[digest_size] = 0x80; + *(__be32 *)&ctx->sha_ctx.buf[SHA512_BLOCK_SIZE - 4] = + cpu_to_be32(8 * (SHA512_BLOCK_SIZE + digest_size)); + + /* Compute the outer hash, which gives the HMAC value. */ + sha512_blocks(&ctx->ostate, ctx->sha_ctx.buf, 1); + for (size_t i = 0; i < digest_size; i += 8) + put_unaligned_be64(ctx->ostate.h[i / 8], out + i); + + memzero_explicit(ctx, sizeof(*ctx)); +} + +void hmac_sha384_final(struct hmac_sha384_ctx *ctx, + u8 out[SHA384_DIGEST_SIZE]) +{ + __hmac_sha512_final(&ctx->ctx, out, SHA384_DIGEST_SIZE); +} +EXPORT_SYMBOL_GPL(hmac_sha384_final); + +void hmac_sha512_final(struct hmac_sha512_ctx *ctx, + u8 out[SHA512_DIGEST_SIZE]) +{ + __hmac_sha512_final(&ctx->ctx, out, SHA512_DIGEST_SIZE); +} +EXPORT_SYMBOL_GPL(hmac_sha512_final); + +void hmac_sha384(const struct hmac_sha384_key *key, + const u8 *data, size_t data_len, u8 out[SHA384_DIGEST_SIZE]) +{ + struct hmac_sha384_ctx ctx; + + hmac_sha384_init(&ctx, key); + hmac_sha384_update(&ctx, data, data_len); + hmac_sha384_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(hmac_sha384); + +void hmac_sha512(const struct hmac_sha512_key *key, + const u8 *data, size_t data_len, u8 out[SHA512_DIGEST_SIZE]) +{ + struct hmac_sha512_ctx ctx; + + hmac_sha512_init(&ctx, key); + hmac_sha512_update(&ctx, data, data_len); + hmac_sha512_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(hmac_sha512); + +void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA384_DIGEST_SIZE]) +{ + struct hmac_sha384_key key; + + hmac_sha384_preparekey(&key, raw_key, raw_key_len); + hmac_sha384(&key, data, data_len, out); + + memzero_explicit(&key, sizeof(key)); +} +EXPORT_SYMBOL_GPL(hmac_sha384_usingrawkey); + +void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA512_DIGEST_SIZE]) +{ + struct hmac_sha512_key key; + + hmac_sha512_preparekey(&key, raw_key, raw_key_len); + hmac_sha512(&key, data, data_len, out); + + memzero_explicit(&key, sizeof(key)); +} +EXPORT_SYMBOL_GPL(hmac_sha512_usingrawkey); + #ifdef sha512_mod_init_arch static int __init sha512_mod_init(void) { @@ -261,5 +398,5 @@ static void __exit sha512_mod_exit(void) module_exit(sha512_mod_exit); #endif -MODULE_DESCRIPTION("SHA-384 and SHA-512 library functions"); +MODULE_DESCRIPTION("SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions"); MODULE_LICENSE("GPL"); From 485deceec03997eabd884d4da89217af73d1ab8d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:08 -0700 Subject: [PATCH 05/67] crypto: riscv/sha512 - Stop depending on sha512_generic_block_fn sha512_generic_block_fn() will no longer be available when the SHA-512 support in the old-school crypto API is changed to just wrap the SHA-512 library. Replace the use of sha512_generic_block_fn() in sha512-riscv64-glue.c with temporary code that uses the library's __sha512_update(). This is just a temporary workaround to keep the kernel building and functional at each commit; this code gets superseded when the RISC-V optimized SHA-512 is migrated to lib/crypto/ anyway. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/riscv/crypto/Kconfig | 1 + arch/riscv/crypto/sha512-riscv64-glue.c | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index cd9b776602f8..53e4e1eacf55 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -31,6 +31,7 @@ config CRYPTO_GHASH_RISCV64 config CRYPTO_SHA512_RISCV64 tristate "Hash functions: SHA-384 and SHA-512" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + select CRYPTO_LIB_SHA512 select CRYPTO_SHA512 help SHA-384 and SHA-512 secure hash algorithm (FIPS 180) diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sha512-riscv64-glue.c index 4634fca78ae2..b3dbc71de07b 100644 --- a/arch/riscv/crypto/sha512-riscv64-glue.c +++ b/arch/riscv/crypto/sha512-riscv64-glue.c @@ -38,7 +38,13 @@ static void sha512_block(struct sha512_state *state, const u8 *data, sha512_transform_zvknhb_zvkb(state, data, num_blocks); kernel_vector_end(); } else { - sha512_generic_block_fn(state, data, num_blocks); + struct __sha512_ctx ctx = {}; + + static_assert(sizeof(ctx.state) == sizeof(state->state)); + memcpy(&ctx.state, state->state, sizeof(ctx.state)); + __sha512_update(&ctx, data, + (size_t)num_blocks * SHA512_BLOCK_SIZE); + memcpy(state->state, &ctx.state, sizeof(state->state)); } } From 469acaa12502e05eefd439693361fe4b851a4fd5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:09 -0700 Subject: [PATCH 06/67] crypto: sha512 - Replace sha512_generic with wrapper around SHA-512 library Delete crypto/sha512_generic.c, which provided "generic" SHA-384 and SHA-512 crypto_shash algorithms. Replace it with crypto/sha512.c which provides SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 crypto_shash algorithms using the corresponding library functions. This is a prerequisite for migrating all the arch-optimized SHA-512 code (which is almost 3000 lines) to lib/crypto/ rather than duplicating it. Since the replacement crypto_shash algorithms are implemented using the (potentially arch-optimized) library functions, give them cra_driver_names ending with "-lib" rather than "-generic". Update crypto/testmgr.c and one odd driver to take this change in driver name into account. Besides these cases which are accounted for, there are no known cases where the cra_driver_name was being depended on. This change does mean that the abstract partial block handling code in crypto/shash.c, which got added in 6.16, no longer gets used. But that's fine; the library has to implement the partial block handling anyway, and it's better to do it in the library since the block size and other properties of the algorithm are all fixed at compile time there, resulting in more streamlined code. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 4 +- crypto/Makefile | 2 +- crypto/sha512.c | 258 ++++++++++++++++++++++++++ crypto/sha512_generic.c | 217 ---------------------- crypto/testmgr.c | 10 + drivers/crypto/starfive/jh7110-hash.c | 8 +- include/crypto/sha512_base.h | 3 - 7 files changed, 276 insertions(+), 226 deletions(-) create mode 100644 crypto/sha512.c delete mode 100644 crypto/sha512_generic.c diff --git a/crypto/Kconfig b/crypto/Kconfig index e1cfd0d4cc8f..cb40a9b46972 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1002,8 +1002,10 @@ config CRYPTO_SHA256 config CRYPTO_SHA512 tristate "SHA-384 and SHA-512" select CRYPTO_HASH + select CRYPTO_LIB_SHA512 help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC 10118-3) + SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC + 10118-3), including HMAC support. config CRYPTO_SHA3 tristate "SHA-3" diff --git a/crypto/Makefile b/crypto/Makefile index 017df3a2e4bb..271c77462cec 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -78,7 +78,7 @@ obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) += sha256.o CFLAGS_sha256.o += -DARCH=$(ARCH) -obj-$(CONFIG_CRYPTO_SHA512) += sha512_generic.o +obj-$(CONFIG_CRYPTO_SHA512) += sha512.o obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o diff --git a/crypto/sha512.c b/crypto/sha512.c new file mode 100644 index 000000000000..d1e5400fe590 --- /dev/null +++ b/crypto/sha512.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Crypto API support for SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2003 Kyle McMartin + * Copyright 2025 Google LLC + */ +#include +#include +#include +#include + +/* SHA-384 */ + +const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] = { + 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38, + 0x4c, 0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a, + 0x21, 0xfd, 0xb7, 0x11, 0x14, 0xbe, 0x07, 0x43, + 0x4c, 0x0c, 0xc7, 0xbf, 0x63, 0xf6, 0xe1, 0xda, + 0x27, 0x4e, 0xde, 0xbf, 0xe7, 0x6f, 0x65, 0xfb, + 0xd5, 0x1a, 0xd2, 0xf1, 0x48, 0x98, 0xb9, 0x5b +}; +EXPORT_SYMBOL_GPL(sha384_zero_message_hash); + +#define SHA384_CTX(desc) ((struct sha384_ctx *)shash_desc_ctx(desc)) + +static int crypto_sha384_init(struct shash_desc *desc) +{ + sha384_init(SHA384_CTX(desc)); + return 0; +} + +static int crypto_sha384_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sha384_update(SHA384_CTX(desc), data, len); + return 0; +} + +static int crypto_sha384_final(struct shash_desc *desc, u8 *out) +{ + sha384_final(SHA384_CTX(desc), out); + return 0; +} + +static int crypto_sha384_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sha384(data, len, out); + return 0; +} + +/* SHA-512 */ + +const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE] = { + 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd, + 0xf1, 0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07, + 0xd6, 0x20, 0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc, + 0x83, 0xf4, 0xa9, 0x21, 0xd3, 0x6c, 0xe9, 0xce, + 0x47, 0xd0, 0xd1, 0x3c, 0x5d, 0x85, 0xf2, 0xb0, + 0xff, 0x83, 0x18, 0xd2, 0x87, 0x7e, 0xec, 0x2f, + 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41, 0x7a, 0x81, + 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda, 0x3e +}; +EXPORT_SYMBOL_GPL(sha512_zero_message_hash); + +#define SHA512_CTX(desc) ((struct sha512_ctx *)shash_desc_ctx(desc)) + +static int crypto_sha512_init(struct shash_desc *desc) +{ + sha512_init(SHA512_CTX(desc)); + return 0; +} + +static int crypto_sha512_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sha512_update(SHA512_CTX(desc), data, len); + return 0; +} + +static int crypto_sha512_final(struct shash_desc *desc, u8 *out) +{ + sha512_final(SHA512_CTX(desc), out); + return 0; +} + +static int crypto_sha512_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sha512(data, len, out); + return 0; +} + +/* HMAC-SHA384 */ + +#define HMAC_SHA384_KEY(tfm) ((struct hmac_sha384_key *)crypto_shash_ctx(tfm)) +#define HMAC_SHA384_CTX(desc) ((struct hmac_sha384_ctx *)shash_desc_ctx(desc)) + +static int crypto_hmac_sha384_setkey(struct crypto_shash *tfm, + const u8 *raw_key, unsigned int keylen) +{ + hmac_sha384_preparekey(HMAC_SHA384_KEY(tfm), raw_key, keylen); + return 0; +} + +static int crypto_hmac_sha384_init(struct shash_desc *desc) +{ + hmac_sha384_init(HMAC_SHA384_CTX(desc), HMAC_SHA384_KEY(desc->tfm)); + return 0; +} + +static int crypto_hmac_sha384_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + hmac_sha384_update(HMAC_SHA384_CTX(desc), data, len); + return 0; +} + +static int crypto_hmac_sha384_final(struct shash_desc *desc, u8 *out) +{ + hmac_sha384_final(HMAC_SHA384_CTX(desc), out); + return 0; +} + +static int crypto_hmac_sha384_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, + u8 *out) +{ + hmac_sha384(HMAC_SHA384_KEY(desc->tfm), data, len, out); + return 0; +} + +/* HMAC-SHA512 */ + +#define HMAC_SHA512_KEY(tfm) ((struct hmac_sha512_key *)crypto_shash_ctx(tfm)) +#define HMAC_SHA512_CTX(desc) ((struct hmac_sha512_ctx *)shash_desc_ctx(desc)) + +static int crypto_hmac_sha512_setkey(struct crypto_shash *tfm, + const u8 *raw_key, unsigned int keylen) +{ + hmac_sha512_preparekey(HMAC_SHA512_KEY(tfm), raw_key, keylen); + return 0; +} + +static int crypto_hmac_sha512_init(struct shash_desc *desc) +{ + hmac_sha512_init(HMAC_SHA512_CTX(desc), HMAC_SHA512_KEY(desc->tfm)); + return 0; +} + +static int crypto_hmac_sha512_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + hmac_sha512_update(HMAC_SHA512_CTX(desc), data, len); + return 0; +} + +static int crypto_hmac_sha512_final(struct shash_desc *desc, u8 *out) +{ + hmac_sha512_final(HMAC_SHA512_CTX(desc), out); + return 0; +} + +static int crypto_hmac_sha512_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, + u8 *out) +{ + hmac_sha512(HMAC_SHA512_KEY(desc->tfm), data, len, out); + return 0; +} + +/* Algorithm definitions */ + +static struct shash_alg algs[] = { + { + .base.cra_name = "sha384", + .base.cra_driver_name = "sha384-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SHA384_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA384_DIGEST_SIZE, + .init = crypto_sha384_init, + .update = crypto_sha384_update, + .final = crypto_sha384_final, + .digest = crypto_sha384_digest, + .descsize = sizeof(struct sha384_ctx), + }, + { + .base.cra_name = "sha512", + .base.cra_driver_name = "sha512-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SHA512_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA512_DIGEST_SIZE, + .init = crypto_sha512_init, + .update = crypto_sha512_update, + .final = crypto_sha512_final, + .digest = crypto_sha512_digest, + .descsize = sizeof(struct sha512_ctx), + }, + { + .base.cra_name = "hmac(sha384)", + .base.cra_driver_name = "hmac-sha384-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SHA384_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct hmac_sha384_key), + .base.cra_module = THIS_MODULE, + .digestsize = SHA384_DIGEST_SIZE, + .setkey = crypto_hmac_sha384_setkey, + .init = crypto_hmac_sha384_init, + .update = crypto_hmac_sha384_update, + .final = crypto_hmac_sha384_final, + .digest = crypto_hmac_sha384_digest, + .descsize = sizeof(struct hmac_sha384_ctx), + }, + { + .base.cra_name = "hmac(sha512)", + .base.cra_driver_name = "hmac-sha512-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SHA512_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct hmac_sha512_key), + .base.cra_module = THIS_MODULE, + .digestsize = SHA512_DIGEST_SIZE, + .setkey = crypto_hmac_sha512_setkey, + .init = crypto_hmac_sha512_init, + .update = crypto_hmac_sha512_update, + .final = crypto_hmac_sha512_final, + .digest = crypto_hmac_sha512_digest, + .descsize = sizeof(struct hmac_sha512_ctx), + }, +}; + +static int __init crypto_sha512_mod_init(void) +{ + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} +module_init(crypto_sha512_mod_init); + +static void __exit crypto_sha512_mod_exit(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} +module_exit(crypto_sha512_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Crypto API support for SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512"); + +MODULE_ALIAS_CRYPTO("sha384"); +MODULE_ALIAS_CRYPTO("sha384-lib"); +MODULE_ALIAS_CRYPTO("sha512"); +MODULE_ALIAS_CRYPTO("sha512-lib"); +MODULE_ALIAS_CRYPTO("hmac(sha384)"); +MODULE_ALIAS_CRYPTO("hmac-sha384-lib"); +MODULE_ALIAS_CRYPTO("hmac(sha512)"); +MODULE_ALIAS_CRYPTO("hmac-sha512-lib"); diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c deleted file mode 100644 index 7368173f545e..000000000000 --- a/crypto/sha512_generic.c +++ /dev/null @@ -1,217 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* SHA-512 code by Jean-Luc Cooke - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2003 Kyle McMartin - */ -#include -#include -#include -#include -#include -#include - -const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] = { - 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38, - 0x4c, 0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a, - 0x21, 0xfd, 0xb7, 0x11, 0x14, 0xbe, 0x07, 0x43, - 0x4c, 0x0c, 0xc7, 0xbf, 0x63, 0xf6, 0xe1, 0xda, - 0x27, 0x4e, 0xde, 0xbf, 0xe7, 0x6f, 0x65, 0xfb, - 0xd5, 0x1a, 0xd2, 0xf1, 0x48, 0x98, 0xb9, 0x5b -}; -EXPORT_SYMBOL_GPL(sha384_zero_message_hash); - -const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE] = { - 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd, - 0xf1, 0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07, - 0xd6, 0x20, 0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc, - 0x83, 0xf4, 0xa9, 0x21, 0xd3, 0x6c, 0xe9, 0xce, - 0x47, 0xd0, 0xd1, 0x3c, 0x5d, 0x85, 0xf2, 0xb0, - 0xff, 0x83, 0x18, 0xd2, 0x87, 0x7e, 0xec, 0x2f, - 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41, 0x7a, 0x81, - 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda, 0x3e -}; -EXPORT_SYMBOL_GPL(sha512_zero_message_hash); - -static inline u64 Ch(u64 x, u64 y, u64 z) -{ - return z ^ (x & (y ^ z)); -} - -static inline u64 Maj(u64 x, u64 y, u64 z) -{ - return (x & y) | (z & (x | y)); -} - -static const u64 sha512_K[80] = { - 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, - 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, - 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL, - 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, - 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, - 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, - 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL, - 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, - 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, - 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, - 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL, - 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, - 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, - 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, - 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL, - 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, - 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, - 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, - 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, - 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, - 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, - 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, - 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL, - 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, - 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, - 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, - 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, -}; - -#define e0(x) (ror64(x,28) ^ ror64(x,34) ^ ror64(x,39)) -#define e1(x) (ror64(x,14) ^ ror64(x,18) ^ ror64(x,41)) -#define s0(x) (ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7)) -#define s1(x) (ror64(x,19) ^ ror64(x,61) ^ (x >> 6)) - -static inline void LOAD_OP(int I, u64 *W, const u8 *input) -{ - W[I] = get_unaligned_be64((__u64 *)input + I); -} - -static inline void BLEND_OP(int I, u64 *W) -{ - W[I & 15] += s1(W[(I-2) & 15]) + W[(I-7) & 15] + s0(W[(I-15) & 15]); -} - -static void -sha512_transform(u64 *state, const u8 *input) -{ - u64 a, b, c, d, e, f, g, h, t1, t2; - - int i; - u64 W[16]; - - /* load the state into our registers */ - a=state[0]; b=state[1]; c=state[2]; d=state[3]; - e=state[4]; f=state[5]; g=state[6]; h=state[7]; - - /* now iterate */ - for (i=0; i<80; i+=8) { - if (!(i & 8)) { - int j; - - if (i < 16) { - /* load the input */ - for (j = 0; j < 16; j++) - LOAD_OP(i + j, W, input); - } else { - for (j = 0; j < 16; j++) { - BLEND_OP(i + j, W); - } - } - } - - t1 = h + e1(e) + Ch(e,f,g) + sha512_K[i ] + W[(i & 15)]; - t2 = e0(a) + Maj(a,b,c); d+=t1; h=t1+t2; - t1 = g + e1(d) + Ch(d,e,f) + sha512_K[i+1] + W[(i & 15) + 1]; - t2 = e0(h) + Maj(h,a,b); c+=t1; g=t1+t2; - t1 = f + e1(c) + Ch(c,d,e) + sha512_K[i+2] + W[(i & 15) + 2]; - t2 = e0(g) + Maj(g,h,a); b+=t1; f=t1+t2; - t1 = e + e1(b) + Ch(b,c,d) + sha512_K[i+3] + W[(i & 15) + 3]; - t2 = e0(f) + Maj(f,g,h); a+=t1; e=t1+t2; - t1 = d + e1(a) + Ch(a,b,c) + sha512_K[i+4] + W[(i & 15) + 4]; - t2 = e0(e) + Maj(e,f,g); h+=t1; d=t1+t2; - t1 = c + e1(h) + Ch(h,a,b) + sha512_K[i+5] + W[(i & 15) + 5]; - t2 = e0(d) + Maj(d,e,f); g+=t1; c=t1+t2; - t1 = b + e1(g) + Ch(g,h,a) + sha512_K[i+6] + W[(i & 15) + 6]; - t2 = e0(c) + Maj(c,d,e); f+=t1; b=t1+t2; - t1 = a + e1(f) + Ch(f,g,h) + sha512_K[i+7] + W[(i & 15) + 7]; - t2 = e0(b) + Maj(b,c,d); e+=t1; a=t1+t2; - } - - state[0] += a; state[1] += b; state[2] += c; state[3] += d; - state[4] += e; state[5] += f; state[6] += g; state[7] += h; -} - -void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, - int blocks) -{ - do { - sha512_transform(sst->state, src); - src += SHA512_BLOCK_SIZE; - } while (--blocks); -} -EXPORT_SYMBOL_GPL(sha512_generic_block_fn); - -static int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, - sha512_generic_block_fn); -} - -static int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) -{ - sha512_base_do_finup(desc, data, len, sha512_generic_block_fn); - return sha512_base_finish(desc, hash); -} - -static struct shash_alg sha512_algs[2] = { { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_base_init, - .update = crypto_sha512_update, - .finup = crypto_sha512_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name = "sha512-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_base_init, - .update = crypto_sha512_update, - .finup = crypto_sha512_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name = "sha384-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init sha512_generic_mod_init(void) -{ - return crypto_register_shashes(sha512_algs, ARRAY_SIZE(sha512_algs)); -} - -static void __exit sha512_generic_mod_fini(void) -{ - crypto_unregister_shashes(sha512_algs, ARRAY_SIZE(sha512_algs)); -} - -module_init(sha512_generic_mod_init); -module_exit(sha512_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-512 and SHA-384 Secure Hash Algorithms"); - -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha384-generic"); -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha512-generic"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 32f753d6c430..9d8b11ea4af7 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4315,12 +4315,14 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "authenc(hmac(sha384),cbc(des))", + .generic_driver = "authenc(hmac-sha384-lib,cbc(des-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha384_des_cbc_tv_temp) } }, { .alg = "authenc(hmac(sha384),cbc(des3_ede))", + .generic_driver = "authenc(hmac-sha384-lib,cbc(des3_ede-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp) @@ -4331,6 +4333,7 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "authenc(hmac(sha384),cts(cbc(aes)))", + .generic_driver = "authenc(hmac-sha384-lib,cts(cbc(aes-generic)))", .test = alg_test_aead, .suite = { .aead = __VECS(krb5_test_aes256_cts_hmac_sha384_192) @@ -4341,6 +4344,7 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "authenc(hmac(sha512),cbc(aes))", + .generic_driver = "authenc(hmac-sha512-lib,cbc(aes-generic))", .fips_allowed = 1, .test = alg_test_aead, .suite = { @@ -4348,12 +4352,14 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "authenc(hmac(sha512),cbc(des))", + .generic_driver = "authenc(hmac-sha512-lib,cbc(des-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha512_des_cbc_tv_temp) } }, { .alg = "authenc(hmac(sha512),cbc(des3_ede))", + .generic_driver = "authenc(hmac-sha512-lib,cbc(des3_ede-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp) @@ -5157,6 +5163,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "hmac(sha384)", + .generic_driver = "hmac-sha384-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { @@ -5164,6 +5171,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "hmac(sha512)", + .generic_driver = "hmac-sha512-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { @@ -5493,6 +5501,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "sha384", + .generic_driver = "sha384-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { @@ -5500,6 +5509,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "sha512", + .generic_driver = "sha512-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index 2c60a1047bc3..4abbff07412f 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -505,13 +505,13 @@ static int starfive_sha256_init_tfm(struct crypto_ahash *hash) static int starfive_sha384_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha384-generic", + return starfive_hash_init_tfm(hash, "sha384-lib", STARFIVE_HASH_SHA384, 0); } static int starfive_sha512_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha512-generic", + return starfive_hash_init_tfm(hash, "sha512-lib", STARFIVE_HASH_SHA512, 0); } @@ -535,13 +535,13 @@ static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash) static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha384-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha384-lib", STARFIVE_HASH_SHA384, 1); } static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha512-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha512-lib", STARFIVE_HASH_SHA512, 1); } diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index aa814bab442d..d1361b3eb70b 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -114,7 +114,4 @@ static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) return 0; } -void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, - int blocks); - #endif /* _CRYPTO_SHA512_BASE_H */ From 4bc7f7b687a2a1506cdc457bc4f6d29a81794a08 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:10 -0700 Subject: [PATCH 07/67] crypto: sha512 - Use same state format as legacy drivers Make the export and import functions for the sha384, sha512, hmac(sha384), and hmac(sha512) shash algorithms use the same format as the padlock-sha and nx-sha512 drivers, as required by Herbert. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-7-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/sha512.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/crypto/sha512.c b/crypto/sha512.c index d1e5400fe590..fb1c520978ef 100644 --- a/crypto/sha512.c +++ b/crypto/sha512.c @@ -12,6 +12,44 @@ #include #include +/* + * Export and import functions. crypto_shash wants a particular format that + * matches that used by some legacy drivers. It currently is the same as the + * library SHA context, except the value in bytecount_lo must be block-aligned + * and the remainder must be stored in an extra u8 appended to the struct. + */ + +#define SHA512_SHASH_STATE_SIZE 209 +static_assert(offsetof(struct __sha512_ctx, state) == 0); +static_assert(offsetof(struct __sha512_ctx, bytecount_lo) == 64); +static_assert(offsetof(struct __sha512_ctx, bytecount_hi) == 72); +static_assert(offsetof(struct __sha512_ctx, buf) == 80); +static_assert(sizeof(struct __sha512_ctx) + 1 == SHA512_SHASH_STATE_SIZE); + +static int __crypto_sha512_export(const struct __sha512_ctx *ctx0, void *out) +{ + struct __sha512_ctx ctx = *ctx0; + unsigned int partial; + u8 *p = out; + + partial = ctx.bytecount_lo % SHA512_BLOCK_SIZE; + ctx.bytecount_lo -= partial; + memcpy(p, &ctx, sizeof(ctx)); + p += sizeof(ctx); + *p = partial; + return 0; +} + +static int __crypto_sha512_import(struct __sha512_ctx *ctx, const void *in) +{ + const u8 *p = in; + + memcpy(ctx, p, sizeof(*ctx)); + p += sizeof(*ctx); + ctx->bytecount_lo += *p; + return 0; +} + /* SHA-384 */ const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] = { @@ -52,6 +90,16 @@ static int crypto_sha384_digest(struct shash_desc *desc, return 0; } +static int crypto_sha384_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha512_export(&SHA384_CTX(desc)->ctx, out); +} + +static int crypto_sha384_import(struct shash_desc *desc, const void *in) +{ + return __crypto_sha512_import(&SHA384_CTX(desc)->ctx, in); +} + /* SHA-512 */ const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE] = { @@ -94,6 +142,16 @@ static int crypto_sha512_digest(struct shash_desc *desc, return 0; } +static int crypto_sha512_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha512_export(&SHA512_CTX(desc)->ctx, out); +} + +static int crypto_sha512_import(struct shash_desc *desc, const void *in) +{ + return __crypto_sha512_import(&SHA512_CTX(desc)->ctx, in); +} + /* HMAC-SHA384 */ #define HMAC_SHA384_KEY(tfm) ((struct hmac_sha384_key *)crypto_shash_ctx(tfm)) @@ -133,6 +191,19 @@ static int crypto_hmac_sha384_digest(struct shash_desc *desc, return 0; } +static int crypto_hmac_sha384_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha512_export(&HMAC_SHA384_CTX(desc)->ctx.sha_ctx, out); +} + +static int crypto_hmac_sha384_import(struct shash_desc *desc, const void *in) +{ + struct hmac_sha384_ctx *ctx = HMAC_SHA384_CTX(desc); + + ctx->ctx.ostate = HMAC_SHA384_KEY(desc->tfm)->key.ostate; + return __crypto_sha512_import(&ctx->ctx.sha_ctx, in); +} + /* HMAC-SHA512 */ #define HMAC_SHA512_KEY(tfm) ((struct hmac_sha512_key *)crypto_shash_ctx(tfm)) @@ -172,6 +243,19 @@ static int crypto_hmac_sha512_digest(struct shash_desc *desc, return 0; } +static int crypto_hmac_sha512_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha512_export(&HMAC_SHA512_CTX(desc)->ctx.sha_ctx, out); +} + +static int crypto_hmac_sha512_import(struct shash_desc *desc, const void *in) +{ + struct hmac_sha512_ctx *ctx = HMAC_SHA512_CTX(desc); + + ctx->ctx.ostate = HMAC_SHA512_KEY(desc->tfm)->key.ostate; + return __crypto_sha512_import(&ctx->ctx.sha_ctx, in); +} + /* Algorithm definitions */ static struct shash_alg algs[] = { @@ -186,7 +270,10 @@ static struct shash_alg algs[] = { .update = crypto_sha384_update, .final = crypto_sha384_final, .digest = crypto_sha384_digest, + .export = crypto_sha384_export, + .import = crypto_sha384_import, .descsize = sizeof(struct sha384_ctx), + .statesize = SHA512_SHASH_STATE_SIZE, }, { .base.cra_name = "sha512", @@ -199,7 +286,10 @@ static struct shash_alg algs[] = { .update = crypto_sha512_update, .final = crypto_sha512_final, .digest = crypto_sha512_digest, + .export = crypto_sha512_export, + .import = crypto_sha512_import, .descsize = sizeof(struct sha512_ctx), + .statesize = SHA512_SHASH_STATE_SIZE, }, { .base.cra_name = "hmac(sha384)", @@ -214,7 +304,10 @@ static struct shash_alg algs[] = { .update = crypto_hmac_sha384_update, .final = crypto_hmac_sha384_final, .digest = crypto_hmac_sha384_digest, + .export = crypto_hmac_sha384_export, + .import = crypto_hmac_sha384_import, .descsize = sizeof(struct hmac_sha384_ctx), + .statesize = SHA512_SHASH_STATE_SIZE, }, { .base.cra_name = "hmac(sha512)", @@ -229,7 +322,10 @@ static struct shash_alg algs[] = { .update = crypto_hmac_sha512_update, .final = crypto_hmac_sha512_final, .digest = crypto_hmac_sha512_digest, + .export = crypto_hmac_sha512_export, + .import = crypto_hmac_sha512_import, .descsize = sizeof(struct hmac_sha512_ctx), + .statesize = SHA512_SHASH_STATE_SIZE, }, }; From 24c91b62ac50a798ceabb3482efbca3e0e88a2db Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:11 -0700 Subject: [PATCH 08/67] lib/crypto: arm/sha512: Migrate optimized SHA-512 code to library Instead of exposing the arm-optimized SHA-512 code via arm-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be arm-optimized, and it fixes the longstanding issue where the arm-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-8-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm/configs/exynos_defconfig | 1 - arch/arm/configs/milbeaut_m10v_defconfig | 1 - arch/arm/configs/multi_v7_defconfig | 1 - arch/arm/configs/omap2plus_defconfig | 1 - arch/arm/configs/pxa_defconfig | 1 - arch/arm/crypto/Kconfig | 10 -- arch/arm/crypto/Makefile | 15 --- arch/arm/crypto/sha512-glue.c | 110 ------------------ arch/arm/crypto/sha512-neon-glue.c | 75 ------------ arch/arm/crypto/sha512.h | 3 - lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 14 +++ lib/crypto/arm/.gitignore | 2 + .../crypto => lib/crypto/arm}/sha512-armv4.pl | 0 lib/crypto/arm/sha512.h | 38 ++++++ 15 files changed, 55 insertions(+), 218 deletions(-) delete mode 100644 arch/arm/crypto/sha512-glue.c delete mode 100644 arch/arm/crypto/sha512-neon-glue.c delete mode 100644 arch/arm/crypto/sha512.h create mode 100644 lib/crypto/arm/.gitignore rename {arch/arm/crypto => lib/crypto/arm}/sha512-armv4.pl (100%) create mode 100644 lib/crypto/arm/sha512.h diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index f71af368674c..d58e30069304 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -364,7 +364,6 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_SHA1_ARM_NEON=m -CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_CHACHA20_NEON=m CONFIG_CRYPTO_DEV_EXYNOS_RNG=y diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 242e7d5a3f68..8ebf8bd872fe 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -100,7 +100,6 @@ CONFIG_CRYPTO_SEQIV=m CONFIG_CRYPTO_GHASH_ARM_CE=m CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA1_ARM_CE=m -CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_AES_ARM_CE=m diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index 50c170b4619f..3fd07e864ca8 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -1282,7 +1282,6 @@ CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_GHASH_ARM_CE=m CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA1_ARM_CE=m -CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_AES_ARM_CE=m diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index 9f9780c8e62a..530dfb8338c9 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -705,7 +705,6 @@ CONFIG_SECURITY=y CONFIG_CRYPTO_MICHAEL_MIC=y CONFIG_CRYPTO_GHASH_ARM_CE=m CONFIG_CRYPTO_SHA1_ARM_NEON=m -CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_CHACHA20_NEON=m diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index ff29c5b0e9c9..eaa44574d4a6 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -659,7 +659,6 @@ CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_DEFLATE=y CONFIG_CRYPTO_LZO=y CONFIG_CRYPTO_SHA1_ARM=m -CONFIG_CRYPTO_SHA512_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_FONTS=y CONFIG_FONT_8x8=y diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 7efb9a8596e4..a18f97f1597c 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -93,16 +93,6 @@ config CRYPTO_SHA1_ARM_CE Architecture: arm using ARMv8 Crypto Extensions -config CRYPTO_SHA512_ARM - tristate "Hash functions: SHA-384 and SHA-512 (NEON)" - select CRYPTO_HASH - depends on !CPU_V7M - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: arm using - - NEON (Advanced SIMD) extensions - config CRYPTO_AES_ARM tristate "Ciphers: AES" select CRYPTO_ALGAPI diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 8479137c6e80..78a4042d8761 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o -obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o obj-$(CONFIG_CRYPTO_CURVE25519_NEON) += curve25519-neon.o @@ -20,23 +19,9 @@ aes-arm-y := aes-cipher-core.o aes-cipher-glue.o aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o sha1-arm-y := sha1-armv4-large.o sha1_glue.o sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o -sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha512-neon-glue.o -sha512-arm-y := sha512-core.o sha512-glue.o $(sha512-arm-neon-y) blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o curve25519-neon-y := curve25519-core.o curve25519-glue.o - -quiet_cmd_perl = PERL $@ - cmd_perl = $(PERL) $(<) > $(@) - -$(obj)/%-core.S: $(src)/%-armv4.pl - $(call cmd,perl) - -clean-files += sha512-core.S - -aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 - -AFLAGS_sha512-core.o += $(aflags-thumb2-y) diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c deleted file mode 100644 index f8a6480889b1..000000000000 --- a/arch/arm/crypto/sha512-glue.c +++ /dev/null @@ -1,110 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sha512-glue.c - accelerated SHA-384/512 for ARM - * - * Copyright (C) 2015 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "sha512.h" - -MODULE_DESCRIPTION("Accelerated SHA-384/SHA-512 secure hash for ARM"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); - -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha384-arm"); -MODULE_ALIAS_CRYPTO("sha512-arm"); - -asmlinkage void sha512_block_data_order(struct sha512_state *state, - u8 const *src, int blocks); - -static int sha512_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, - sha512_block_data_order); -} - -static int sha512_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, data, len, sha512_block_data_order); - return sha512_base_finish(desc, out); -} - -static struct shash_alg sha512_arm_algs[] = { { - .init = sha384_base_init, - .update = sha512_arm_update, - .finup = sha512_arm_finup, - .descsize = SHA512_STATE_SIZE, - .digestsize = SHA384_DIGEST_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name = "sha384-arm", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .init = sha512_base_init, - .update = sha512_arm_update, - .finup = sha512_arm_finup, - .descsize = SHA512_STATE_SIZE, - .digestsize = SHA512_DIGEST_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name = "sha512-arm", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init sha512_arm_mod_init(void) -{ - int err; - - err = crypto_register_shashes(sha512_arm_algs, - ARRAY_SIZE(sha512_arm_algs)); - if (err) - return err; - - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) { - err = crypto_register_shashes(sha512_neon_algs, - ARRAY_SIZE(sha512_neon_algs)); - if (err) - goto err_unregister; - } - return 0; - -err_unregister: - crypto_unregister_shashes(sha512_arm_algs, - ARRAY_SIZE(sha512_arm_algs)); - - return err; -} - -static void __exit sha512_arm_mod_fini(void) -{ - crypto_unregister_shashes(sha512_arm_algs, - ARRAY_SIZE(sha512_arm_algs)); - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) - crypto_unregister_shashes(sha512_neon_algs, - ARRAY_SIZE(sha512_neon_algs)); -} - -module_init(sha512_arm_mod_init); -module_exit(sha512_arm_mod_fini); diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c deleted file mode 100644 index bd528077fefb..000000000000 --- a/arch/arm/crypto/sha512-neon-glue.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sha512-neon-glue.c - accelerated SHA-384/512 for ARM NEON - * - * Copyright (C) 2015 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include - -#include "sha512.h" - -MODULE_ALIAS_CRYPTO("sha384-neon"); -MODULE_ALIAS_CRYPTO("sha512-neon"); - -asmlinkage void sha512_block_data_order_neon(struct sha512_state *state, - const u8 *src, int blocks); - -static int sha512_neon_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - kernel_neon_begin(); - remain = sha512_base_do_update_blocks(desc, data, len, - sha512_block_data_order_neon); - kernel_neon_end(); - return remain; -} - -static int sha512_neon_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_neon_begin(); - sha512_base_do_finup(desc, data, len, sha512_block_data_order_neon); - kernel_neon_end(); - return sha512_base_finish(desc, out); -} - -struct shash_alg sha512_neon_algs[] = { { - .init = sha384_base_init, - .update = sha512_neon_update, - .finup = sha512_neon_finup, - .descsize = SHA512_STATE_SIZE, - .digestsize = SHA384_DIGEST_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name = "sha384-neon", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - - } -}, { - .init = sha512_base_init, - .update = sha512_neon_update, - .finup = sha512_neon_finup, - .descsize = SHA512_STATE_SIZE, - .digestsize = SHA512_DIGEST_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name = "sha512-neon", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; diff --git a/arch/arm/crypto/sha512.h b/arch/arm/crypto/sha512.h deleted file mode 100644 index eeaee52cda69..000000000000 --- a/arch/arm/crypto/sha512.h +++ /dev/null @@ -1,3 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -extern struct shash_alg sha512_neon_algs[2]; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index d1bee3787eb3..dac6356ba0aa 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -177,6 +177,7 @@ config CRYPTO_LIB_SHA512 config CRYPTO_LIB_SHA512_ARCH bool depends on CRYPTO_LIB_SHA512 && !UML + default y if ARM && !CPU_V7M config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index f6b6f370451e..67008a1612c6 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 +aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $(<) > $(@) + obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o libcryptoutils-y := memneq.o utils.o @@ -68,6 +73,15 @@ obj-$(CONFIG_CRYPTO_LIB_SHA512) += libsha512.o libsha512-y := sha512.o ifeq ($(CONFIG_CRYPTO_LIB_SHA512_ARCH),y) CFLAGS_sha512.o += -I$(src)/$(SRCARCH) + +ifeq ($(CONFIG_ARM),y) +libsha512-y += arm/sha512-core.o +$(obj)/arm/sha512-core.S: $(src)/arm/sha512-armv4.pl + $(call cmd,perlasm) +clean-files += arm/sha512-core.S +AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y) +endif + endif # CONFIG_CRYPTO_LIB_SHA512_ARCH obj-$(CONFIG_MPILIB) += mpi/ diff --git a/lib/crypto/arm/.gitignore b/lib/crypto/arm/.gitignore new file mode 100644 index 000000000000..670a4d97b568 --- /dev/null +++ b/lib/crypto/arm/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +sha512-core.S diff --git a/arch/arm/crypto/sha512-armv4.pl b/lib/crypto/arm/sha512-armv4.pl similarity index 100% rename from arch/arm/crypto/sha512-armv4.pl rename to lib/crypto/arm/sha512-armv4.pl diff --git a/lib/crypto/arm/sha512.h b/lib/crypto/arm/sha512.h new file mode 100644 index 000000000000..f147b6490d6c --- /dev/null +++ b/lib/crypto/arm/sha512.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * arm32-optimized SHA-512 block function + * + * Copyright 2025 Google LLC + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); + +asmlinkage void sha512_block_data_order(struct sha512_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage void sha512_block_data_order_neon(struct sha512_block_state *state, + const u8 *data, size_t nblocks); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_neon) && likely(crypto_simd_usable())) { + kernel_neon_begin(); + sha512_block_data_order_neon(state, data, nblocks); + kernel_neon_end(); + } else { + sha512_block_data_order(state, data, nblocks); + } +} + +#ifdef CONFIG_KERNEL_MODE_NEON +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (cpu_has_neon()) + static_branch_enable(&have_neon); +} +#endif /* CONFIG_KERNEL_MODE_NEON */ From 60e3f1e9b7a57567c2f3b3ae013e3e292cf6d115 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:12 -0700 Subject: [PATCH 09/67] lib/crypto: arm64/sha512: Migrate optimized SHA-512 code to library Instead of exposing the arm64-optimized SHA-512 code via arm64-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be arm64-optimized, and it fixes the longstanding issue where the arm64-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly functions from int or 'unsigned int' to size_t. Update the ARMv8 CE assembly function accordingly. The scalar assembly function actually already treated it as size_t. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-9-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/configs/defconfig | 1 - arch/arm64/crypto/Kconfig | 19 ---- arch/arm64/crypto/Makefile | 14 --- arch/arm64/crypto/sha512-ce-glue.c | 96 ------------------- arch/arm64/crypto/sha512-glue.c | 83 ---------------- lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 10 ++ lib/crypto/arm64/.gitignore | 2 + .../crypto/arm64}/sha512-ce-core.S | 10 +- lib/crypto/arm64/sha512.h | 46 +++++++++ 10 files changed, 64 insertions(+), 218 deletions(-) delete mode 100644 arch/arm64/crypto/sha512-ce-glue.c delete mode 100644 arch/arm64/crypto/sha512-glue.c create mode 100644 lib/crypto/arm64/.gitignore rename {arch/arm64/crypto => lib/crypto/arm64}/sha512-ce-core.S (97%) create mode 100644 lib/crypto/arm64/sha512.h diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 897fc686e6a9..b612b78b3b09 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1744,7 +1744,6 @@ CONFIG_CRYPTO_ANSI_CPRNG=y CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_GHASH_ARM64_CE=y CONFIG_CRYPTO_SHA1_ARM64_CE=y -CONFIG_CRYPTO_SHA512_ARM64_CE=m CONFIG_CRYPTO_SHA3_ARM64=m CONFIG_CRYPTO_SM3_ARM64_CE=m CONFIG_CRYPTO_AES_ARM64_CE_BLK=y diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index c44b0f202a1f..a9ead99f72c2 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -36,25 +36,6 @@ config CRYPTO_SHA1_ARM64_CE Architecture: arm64 using: - ARMv8 Crypto Extensions -config CRYPTO_SHA512_ARM64 - tristate "Hash functions: SHA-384 and SHA-512" - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: arm64 - -config CRYPTO_SHA512_ARM64_CE - tristate "Hash functions: SHA-384 and SHA-512 (ARMv8 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_SHA512_ARM64 - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: arm64 using: - - ARMv8 Crypto Extensions - config CRYPTO_SHA3_ARM64 tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index c231c980c514..228101f125d5 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -8,9 +8,6 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM64_CE) += sha1-ce.o sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o -obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) += sha512-ce.o -sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o - obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o @@ -53,9 +50,6 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o aes-neon-blk-y := aes-glue-neon.o aes-neon.o -obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o -sha512-arm64-y := sha512-glue.o sha512-core.o - obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o @@ -64,11 +58,3 @@ aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o - -quiet_cmd_perlasm = PERLASM $@ - cmd_perlasm = $(PERL) $(<) void $(@) - -$(obj)/sha512-core.S: $(src)/../lib/crypto/sha2-armv8.pl - $(call cmd,perlasm) - -clean-files += sha512-core.S diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c deleted file mode 100644 index 6fb3001fa2c9..000000000000 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * sha512-ce-glue.c - SHA-384/SHA-512 using ARMv8 Crypto Extensions - * - * Copyright (C) 2018 Linaro Ltd - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include -#include -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha512"); - -asmlinkage int __sha512_ce_transform(struct sha512_state *sst, u8 const *src, - int blocks); - -static void sha512_ce_transform(struct sha512_state *sst, u8 const *src, - int blocks) -{ - do { - int rem; - - kernel_neon_begin(); - rem = __sha512_ce_transform(sst, src, blocks); - kernel_neon_end(); - src += (blocks - rem) * SHA512_BLOCK_SIZE; - blocks = rem; - } while (blocks); -} - -static int sha512_ce_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, - sha512_ce_transform); -} - -static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, data, len, sha512_ce_transform); - return sha512_base_finish(desc, out); -} - -static struct shash_alg algs[] = { { - .init = sha384_base_init, - .update = sha512_ce_update, - .finup = sha512_ce_finup, - .descsize = SHA512_STATE_SIZE, - .digestsize = SHA384_DIGEST_SIZE, - .base.cra_name = "sha384", - .base.cra_driver_name = "sha384-ce", - .base.cra_priority = 200, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA512_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}, { - .init = sha512_base_init, - .update = sha512_ce_update, - .finup = sha512_ce_finup, - .descsize = SHA512_STATE_SIZE, - .digestsize = SHA512_DIGEST_SIZE, - .base.cra_name = "sha512", - .base.cra_driver_name = "sha512-ce", - .base.cra_priority = 200, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA512_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -} }; - -static int __init sha512_ce_mod_init(void) -{ - return crypto_register_shashes(algs, ARRAY_SIZE(algs)); -} - -static void __exit sha512_ce_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_cpu_feature_match(SHA512, sha512_ce_mod_init); -module_exit(sha512_ce_mod_fini); diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c deleted file mode 100644 index a78e184c100f..000000000000 --- a/arch/arm64/crypto/sha512-glue.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Linux/arm64 port of the OpenSSL SHA512 implementation for AArch64 - * - * Copyright (c) 2016 Linaro Ltd. - */ - -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash for arm64"); -MODULE_AUTHOR("Andy Polyakov "); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha512"); - -asmlinkage void sha512_blocks_arch(u64 *digest, const void *data, - unsigned int num_blks); - -static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, - int blocks) -{ - sha512_blocks_arch(sst->state, src, blocks); -} - -static int sha512_update_arm64(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, - sha512_arm64_transform); -} - -static int sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, data, len, sha512_arm64_transform); - return sha512_base_finish(desc, out); -} - -static struct shash_alg algs[] = { { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_base_init, - .update = sha512_update_arm64, - .finup = sha512_finup, - .descsize = SHA512_STATE_SIZE, - .base.cra_name = "sha512", - .base.cra_driver_name = "sha512-arm64", - .base.cra_priority = 150, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA512_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -}, { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_base_init, - .update = sha512_update_arm64, - .finup = sha512_finup, - .descsize = SHA512_STATE_SIZE, - .base.cra_name = "sha384", - .base.cra_driver_name = "sha384-arm64", - .base.cra_priority = 150, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA384_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, -} }; - -static int __init sha512_mod_init(void) -{ - return crypto_register_shashes(algs, ARRAY_SIZE(algs)); -} - -static void __exit sha512_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_init(sha512_mod_init); -module_exit(sha512_mod_fini); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index dac6356ba0aa..26413f679fab 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -178,6 +178,7 @@ config CRYPTO_LIB_SHA512_ARCH bool depends on CRYPTO_LIB_SHA512 && !UML default y if ARM && !CPU_V7M + default y if ARM64 config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 67008a1612c6..22269ab06d70 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -5,6 +5,9 @@ aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $(<) > $(@) +quiet_cmd_perlasm_with_args = PERLASM $@ + cmd_perlasm_with_args = $(PERL) $(<) void $(@) + obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o libcryptoutils-y := memneq.o utils.o @@ -82,6 +85,13 @@ clean-files += arm/sha512-core.S AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y) endif +ifeq ($(CONFIG_ARM64),y) +libsha512-y += arm64/sha512-core.o +$(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.pl + $(call cmd,perlasm_with_args) +clean-files += arm64/sha512-core.S +libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o +endif endif # CONFIG_CRYPTO_LIB_SHA512_ARCH obj-$(CONFIG_MPILIB) += mpi/ diff --git a/lib/crypto/arm64/.gitignore b/lib/crypto/arm64/.gitignore new file mode 100644 index 000000000000..670a4d97b568 --- /dev/null +++ b/lib/crypto/arm64/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +sha512-core.S diff --git a/arch/arm64/crypto/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S similarity index 97% rename from arch/arm64/crypto/sha512-ce-core.S rename to lib/crypto/arm64/sha512-ce-core.S index 91ef68b15fcc..7d870a435ea3 100644 --- a/arch/arm64/crypto/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -102,8 +102,8 @@ .endm /* - * int __sha512_ce_transform(struct sha512_state *sst, u8 const *src, - * int blocks) + * size_t __sha512_ce_transform(struct sha512_block_state *state, + * const u8 *data, size_t nblocks); */ .text SYM_FUNC_START(__sha512_ce_transform) @@ -117,7 +117,7 @@ SYM_FUNC_START(__sha512_ce_transform) /* load input */ 0: ld1 {v12.2d-v15.2d}, [x1], #64 ld1 {v16.2d-v19.2d}, [x1], #64 - sub w2, w2, #1 + sub x2, x2, #1 CPU_LE( rev64 v12.16b, v12.16b ) CPU_LE( rev64 v13.16b, v13.16b ) @@ -197,10 +197,10 @@ CPU_LE( rev64 v19.16b, v19.16b ) cond_yield 3f, x4, x5 /* handled all input blocks? */ - cbnz w2, 0b + cbnz x2, 0b /* store new state */ 3: st1 {v8.2d-v11.2d}, [x0] - mov w0, w2 + mov x0, x2 ret SYM_FUNC_END(__sha512_ce_transform) diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h new file mode 100644 index 000000000000..eae14f9752e0 --- /dev/null +++ b/lib/crypto/arm64/sha512.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * arm64-optimized SHA-512 block function + * + * Copyright 2025 Google LLC + */ + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); + +asmlinkage void sha512_blocks_arch(struct sha512_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_sha512_insns) && + likely(crypto_simd_usable())) { + do { + size_t rem; + + kernel_neon_begin(); + rem = __sha512_ce_transform(state, data, nblocks); + kernel_neon_end(); + data += (nblocks - rem) * SHA512_BLOCK_SIZE; + nblocks = rem; + } while (nblocks); + } else { + sha512_blocks_arch(state, data, nblocks); + } +} + +#ifdef CONFIG_KERNEL_MODE_NEON +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (cpu_have_named_feature(SHA512)) + static_branch_enable(&have_sha512_insns); +} +#endif /* CONFIG_KERNEL_MODE_NEON */ From ecac3068ffc28324070e96c46d171b5431215c0d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:13 -0700 Subject: [PATCH 10/67] mips: cavium-octeon: Move octeon-crypto.h into asm directory Since arch/mips/cavium-octeon/crypto/octeon-crypto.h is now needed outside of its directory, move it to arch/mips/include/asm/octeon/crypto.h so that it can be included as . Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-10-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/mips/cavium-octeon/crypto/octeon-crypto.c | 3 +-- arch/mips/cavium-octeon/crypto/octeon-md5.c | 3 +-- arch/mips/cavium-octeon/crypto/octeon-sha1.c | 3 +-- arch/mips/cavium-octeon/crypto/octeon-sha256.c | 3 +-- arch/mips/cavium-octeon/crypto/octeon-sha512.c | 3 +-- .../crypto/octeon-crypto.h => include/asm/octeon/crypto.h} | 0 6 files changed, 5 insertions(+), 10 deletions(-) rename arch/mips/{cavium-octeon/crypto/octeon-crypto.h => include/asm/octeon/crypto.h} (100%) diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.c b/arch/mips/cavium-octeon/crypto/octeon-crypto.c index cfb4a146cf17..0ff8559391f5 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.c +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.c @@ -7,12 +7,11 @@ */ #include +#include #include #include #include -#include "octeon-crypto.h" - /** * Enable access to Octeon's COP2 crypto hardware for kernel use. Wrap any * crypto operations in calls to octeon_crypto_enable/disable in order to make diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c index fbc84eb7fedf..a8ce831e2ceb 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-md5.c +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -19,6 +19,7 @@ * any later version. */ +#include #include #include #include @@ -27,8 +28,6 @@ #include #include -#include "octeon-crypto.h" - struct octeon_md5_state { __le32 hash[MD5_HASH_WORDS]; u64 byte_count; diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index e70f21a473da..e4a369a7764f 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -13,6 +13,7 @@ * Copyright (c) Jean-Francois Dive */ +#include #include #include #include @@ -21,8 +22,6 @@ #include #include -#include "octeon-crypto.h" - /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index f93faaf1f4af..c20038239cb6 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -12,13 +12,12 @@ * SHA224 Support Copyright 2007 Intel Corporation */ +#include #include #include #include #include -#include "octeon-crypto.h" - /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 215311053db3..53de74f642db 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -13,6 +13,7 @@ * Copyright (c) 2003 Kyle McMartin */ +#include #include #include #include @@ -20,8 +21,6 @@ #include #include -#include "octeon-crypto.h" - /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/include/asm/octeon/crypto.h similarity index 100% rename from arch/mips/cavium-octeon/crypto/octeon-crypto.h rename to arch/mips/include/asm/octeon/crypto.h From 7117739ad2b40298776f428382ebce525adc7f7d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:14 -0700 Subject: [PATCH 11/67] lib/crypto: mips/sha512: Migrate optimized SHA-512 code to library Instead of exposing the mips-optimized SHA-512 code via mips-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be mips-optimized, and it fixes the longstanding issue where the mips-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. Note: to see the diff from arch/mips/cavium-octeon/crypto/octeon-sha512.c to lib/crypto/mips/sha512.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-11-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/mips/cavium-octeon/crypto/Makefile | 1 - .../mips/cavium-octeon/crypto/octeon-sha512.c | 166 ------------------ arch/mips/configs/cavium_octeon_defconfig | 1 - arch/mips/crypto/Kconfig | 10 -- lib/crypto/Kconfig | 1 + lib/crypto/mips/sha512.h | 74 ++++++++ 6 files changed, 75 insertions(+), 178 deletions(-) delete mode 100644 arch/mips/cavium-octeon/crypto/octeon-sha512.c create mode 100644 lib/crypto/mips/sha512.h diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-octeon/crypto/Makefile index db26c73fa0ed..168b19ef7ce8 100644 --- a/arch/mips/cavium-octeon/crypto/Makefile +++ b/arch/mips/cavium-octeon/crypto/Makefile @@ -8,4 +8,3 @@ obj-y += octeon-crypto.o obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o obj-$(CONFIG_CRYPTO_SHA1_OCTEON) += octeon-sha1.o obj-$(CONFIG_CRYPTO_SHA256_OCTEON) += octeon-sha256.o -obj-$(CONFIG_CRYPTO_SHA512_OCTEON) += octeon-sha512.o diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c deleted file mode 100644 index 53de74f642db..000000000000 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ /dev/null @@ -1,166 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * SHA-512 and SHA-384 Secure Hash Algorithm. - * - * Adapted for OCTEON by Aaro Koskinen . - * - * Based on crypto/sha512_generic.c, which is: - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2003 Kyle McMartin - */ - -#include -#include -#include -#include -#include -#include -#include - -/* - * We pass everything as 64-bit. OCTEON can handle misaligned data. - */ - -static void octeon_sha512_store_hash(struct sha512_state *sctx) -{ - write_octeon_64bit_hash_sha512(sctx->state[0], 0); - write_octeon_64bit_hash_sha512(sctx->state[1], 1); - write_octeon_64bit_hash_sha512(sctx->state[2], 2); - write_octeon_64bit_hash_sha512(sctx->state[3], 3); - write_octeon_64bit_hash_sha512(sctx->state[4], 4); - write_octeon_64bit_hash_sha512(sctx->state[5], 5); - write_octeon_64bit_hash_sha512(sctx->state[6], 6); - write_octeon_64bit_hash_sha512(sctx->state[7], 7); -} - -static void octeon_sha512_read_hash(struct sha512_state *sctx) -{ - sctx->state[0] = read_octeon_64bit_hash_sha512(0); - sctx->state[1] = read_octeon_64bit_hash_sha512(1); - sctx->state[2] = read_octeon_64bit_hash_sha512(2); - sctx->state[3] = read_octeon_64bit_hash_sha512(3); - sctx->state[4] = read_octeon_64bit_hash_sha512(4); - sctx->state[5] = read_octeon_64bit_hash_sha512(5); - sctx->state[6] = read_octeon_64bit_hash_sha512(6); - sctx->state[7] = read_octeon_64bit_hash_sha512(7); -} - -static void octeon_sha512_transform(struct sha512_state *sctx, - const u8 *src, int blocks) -{ - do { - const u64 *block = (const u64 *)src; - - write_octeon_64bit_block_sha512(block[0], 0); - write_octeon_64bit_block_sha512(block[1], 1); - write_octeon_64bit_block_sha512(block[2], 2); - write_octeon_64bit_block_sha512(block[3], 3); - write_octeon_64bit_block_sha512(block[4], 4); - write_octeon_64bit_block_sha512(block[5], 5); - write_octeon_64bit_block_sha512(block[6], 6); - write_octeon_64bit_block_sha512(block[7], 7); - write_octeon_64bit_block_sha512(block[8], 8); - write_octeon_64bit_block_sha512(block[9], 9); - write_octeon_64bit_block_sha512(block[10], 10); - write_octeon_64bit_block_sha512(block[11], 11); - write_octeon_64bit_block_sha512(block[12], 12); - write_octeon_64bit_block_sha512(block[13], 13); - write_octeon_64bit_block_sha512(block[14], 14); - octeon_sha512_start(block[15]); - - src += SHA512_BLOCK_SIZE; - } while (--blocks); -} - -static int octeon_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - struct octeon_cop2_state state; - unsigned long flags; - int remain; - - flags = octeon_crypto_enable(&state); - octeon_sha512_store_hash(sctx); - - remain = sha512_base_do_update_blocks(desc, data, len, - octeon_sha512_transform); - - octeon_sha512_read_hash(sctx); - octeon_crypto_disable(&state, flags); - return remain; -} - -static int octeon_sha512_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *hash) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - struct octeon_cop2_state state; - unsigned long flags; - - flags = octeon_crypto_enable(&state); - octeon_sha512_store_hash(sctx); - - sha512_base_do_finup(desc, src, len, octeon_sha512_transform); - - octeon_sha512_read_hash(sctx); - octeon_crypto_disable(&state, flags); - return sha512_base_finish(desc, hash); -} - -static struct shash_alg octeon_sha512_algs[2] = { { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_base_init, - .update = octeon_sha512_update, - .finup = octeon_sha512_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name= "octeon-sha512", - .cra_priority = OCTEON_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_base_init, - .update = octeon_sha512_update, - .finup = octeon_sha512_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name= "octeon-sha384", - .cra_priority = OCTEON_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init octeon_sha512_mod_init(void) -{ - if (!octeon_has_crypto()) - return -ENOTSUPP; - return crypto_register_shashes(octeon_sha512_algs, - ARRAY_SIZE(octeon_sha512_algs)); -} - -static void __exit octeon_sha512_mod_fini(void) -{ - crypto_unregister_shashes(octeon_sha512_algs, - ARRAY_SIZE(octeon_sha512_algs)); -} - -module_init(octeon_sha512_mod_init); -module_exit(octeon_sha512_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-512 and SHA-384 Secure Hash Algorithms (OCTEON)"); -MODULE_AUTHOR("Aaro Koskinen "); diff --git a/arch/mips/configs/cavium_octeon_defconfig b/arch/mips/configs/cavium_octeon_defconfig index 88ae0aa85364..effdfb2bb738 100644 --- a/arch/mips/configs/cavium_octeon_defconfig +++ b/arch/mips/configs/cavium_octeon_defconfig @@ -157,7 +157,6 @@ CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5_OCTEON=y CONFIG_CRYPTO_SHA1_OCTEON=m -CONFIG_CRYPTO_SHA512_OCTEON=m CONFIG_CRYPTO_DES=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_FS=y diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 6bf073ae7613..51a76a5ee3b1 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -22,14 +22,4 @@ config CRYPTO_SHA1_OCTEON Architecture: mips OCTEON -config CRYPTO_SHA512_OCTEON - tristate "Hash functions: SHA-384 and SHA-512 (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: mips OCTEON using crypto instructions, when available - endmenu diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 26413f679fab..303ea15e3e90 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -179,6 +179,7 @@ config CRYPTO_LIB_SHA512_ARCH depends on CRYPTO_LIB_SHA512 && !UML default y if ARM && !CPU_V7M default y if ARM64 + default y if MIPS && CPU_CAVIUM_OCTEON config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/mips/sha512.h b/lib/crypto/mips/sha512.h new file mode 100644 index 000000000000..b3ffbc1e8ca8 --- /dev/null +++ b/lib/crypto/mips/sha512.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Cryptographic API. + * + * SHA-512 and SHA-384 Secure Hash Algorithm. + * + * Adapted for OCTEON by Aaro Koskinen . + * + * Based on crypto/sha512_generic.c, which is: + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2003 Kyle McMartin + */ + +#include +#include + +/* + * We pass everything as 64-bit. OCTEON can handle misaligned data. + */ + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + struct octeon_cop2_state cop2_state; + unsigned long flags; + + if (!octeon_has_crypto()) + return sha512_blocks_generic(state, data, nblocks); + + flags = octeon_crypto_enable(&cop2_state); + write_octeon_64bit_hash_sha512(state->h[0], 0); + write_octeon_64bit_hash_sha512(state->h[1], 1); + write_octeon_64bit_hash_sha512(state->h[2], 2); + write_octeon_64bit_hash_sha512(state->h[3], 3); + write_octeon_64bit_hash_sha512(state->h[4], 4); + write_octeon_64bit_hash_sha512(state->h[5], 5); + write_octeon_64bit_hash_sha512(state->h[6], 6); + write_octeon_64bit_hash_sha512(state->h[7], 7); + + do { + const u64 *block = (const u64 *)data; + + write_octeon_64bit_block_sha512(block[0], 0); + write_octeon_64bit_block_sha512(block[1], 1); + write_octeon_64bit_block_sha512(block[2], 2); + write_octeon_64bit_block_sha512(block[3], 3); + write_octeon_64bit_block_sha512(block[4], 4); + write_octeon_64bit_block_sha512(block[5], 5); + write_octeon_64bit_block_sha512(block[6], 6); + write_octeon_64bit_block_sha512(block[7], 7); + write_octeon_64bit_block_sha512(block[8], 8); + write_octeon_64bit_block_sha512(block[9], 9); + write_octeon_64bit_block_sha512(block[10], 10); + write_octeon_64bit_block_sha512(block[11], 11); + write_octeon_64bit_block_sha512(block[12], 12); + write_octeon_64bit_block_sha512(block[13], 13); + write_octeon_64bit_block_sha512(block[14], 14); + octeon_sha512_start(block[15]); + + data += SHA512_BLOCK_SIZE; + } while (--nblocks); + + state->h[0] = read_octeon_64bit_hash_sha512(0); + state->h[1] = read_octeon_64bit_hash_sha512(1); + state->h[2] = read_octeon_64bit_hash_sha512(2); + state->h[3] = read_octeon_64bit_hash_sha512(3); + state->h[4] = read_octeon_64bit_hash_sha512(4); + state->h[5] = read_octeon_64bit_hash_sha512(5); + state->h[6] = read_octeon_64bit_hash_sha512(6); + state->h[7] = read_octeon_64bit_hash_sha512(7); + octeon_crypto_disable(&cop2_state, flags); +} From b59059a22c5a717284b05385de5c0cbff2879859 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:15 -0700 Subject: [PATCH 12/67] lib/crypto: riscv/sha512: Migrate optimized SHA-512 code to library Instead of exposing the riscv-optimized SHA-512 code via riscv-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be riscv-optimized, and it fixes the longstanding issue where the riscv-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly function from int to size_t. The assembly function actually already treated it as size_t. Note: to see the diff from arch/riscv/crypto/sha512-riscv64-glue.c to lib/crypto/riscv/sha512.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-12-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/riscv/crypto/Kconfig | 12 -- arch/riscv/crypto/Makefile | 3 - arch/riscv/crypto/sha512-riscv64-glue.c | 130 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 2 + .../riscv}/sha512-riscv64-zvknhb-zvkb.S | 4 +- lib/crypto/riscv/sha512.h | 41 ++++++ 7 files changed, 46 insertions(+), 147 deletions(-) delete mode 100644 arch/riscv/crypto/sha512-riscv64-glue.c rename {arch/riscv/crypto => lib/crypto/riscv}/sha512-riscv64-zvknhb-zvkb.S (98%) create mode 100644 lib/crypto/riscv/sha512.h diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 53e4e1eacf55..a75d6325607b 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -28,18 +28,6 @@ config CRYPTO_GHASH_RISCV64 Architecture: riscv64 using: - Zvkg vector crypto extension -config CRYPTO_SHA512_RISCV64 - tristate "Hash functions: SHA-384 and SHA-512" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_LIB_SHA512 - select CRYPTO_SHA512 - help - SHA-384 and SHA-512 secure hash algorithm (FIPS 180) - - Architecture: riscv64 using: - - Zvknhb vector crypto extension - - Zvkb vector crypto extension - config CRYPTO_SM3_RISCV64 tristate "Hash functions: SM3 (ShangMi 3)" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index e10e8257734e..183495a95cc0 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -7,9 +7,6 @@ aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o -obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o -sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o - obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sha512-riscv64-glue.c deleted file mode 100644 index b3dbc71de07b..000000000000 --- a/arch/riscv/crypto/sha512-riscv64-glue.c +++ /dev/null @@ -1,130 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SHA-512 and SHA-384 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih - */ - -#include -#include -#include -#include -#include -#include -#include - -/* - * Note: the asm function only uses the 'state' field of struct sha512_state. - * It is assumed to be the first field. - */ -asmlinkage void sha512_transform_zvknhb_zvkb( - struct sha512_state *state, const u8 *data, int num_blocks); - -static void sha512_block(struct sha512_state *state, const u8 *data, - int num_blocks) -{ - /* - * Ensure struct sha512_state begins directly with the SHA-512 - * 512-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sha512_transform_zvknhb_zvkb(state, data, num_blocks); - kernel_vector_end(); - } else { - struct __sha512_ctx ctx = {}; - - static_assert(sizeof(ctx.state) == sizeof(state->state)); - memcpy(&ctx.state, state->state, sizeof(ctx.state)); - __sha512_update(&ctx, data, - (size_t)num_blocks * SHA512_BLOCK_SIZE); - memcpy(state->state, &ctx.state, sizeof(state->state)); - } -} - -static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, sha512_block); -} - -static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, data, len, sha512_block); - return sha512_base_finish(desc, out); -} - -static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_base_init(desc) ?: - riscv64_sha512_finup(desc, data, len, out); -} - -static struct shash_alg riscv64_sha512_algs[] = { - { - .init = sha512_base_init, - .update = riscv64_sha512_update, - .finup = riscv64_sha512_finup, - .digest = riscv64_sha512_digest, - .descsize = SHA512_STATE_SIZE, - .digestsize = SHA512_DIGEST_SIZE, - .base = { - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_name = "sha512", - .cra_driver_name = "sha512-riscv64-zvknhb-zvkb", - .cra_module = THIS_MODULE, - }, - }, { - .init = sha384_base_init, - .update = riscv64_sha512_update, - .finup = riscv64_sha512_finup, - .descsize = SHA512_STATE_SIZE, - .digestsize = SHA384_DIGEST_SIZE, - .base = { - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_name = "sha384", - .cra_driver_name = "sha384-riscv64-zvknhb-zvkb", - .cra_module = THIS_MODULE, - }, - }, -}; - -static int __init riscv64_sha512_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKNHB) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shashes(riscv64_sha512_algs, - ARRAY_SIZE(riscv64_sha512_algs)); - - return -ENODEV; -} - -static void __exit riscv64_sha512_mod_exit(void) -{ - crypto_unregister_shashes(riscv64_sha512_algs, - ARRAY_SIZE(riscv64_sha512_algs)); -} - -module_init(riscv64_sha512_mod_init); -module_exit(riscv64_sha512_mod_exit); - -MODULE_DESCRIPTION("SHA-512 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha384"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 303ea15e3e90..656f2c08c46c 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -180,6 +180,7 @@ config CRYPTO_LIB_SHA512_ARCH default y if ARM && !CPU_V7M default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON + default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 22269ab06d70..bc42464a279b 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -92,6 +92,8 @@ $(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.pl clean-files += arm64/sha512-core.S libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o endif + +libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o endif # CONFIG_CRYPTO_LIB_SHA512_ARCH obj-$(CONFIG_MPILIB) += mpi/ diff --git a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S similarity index 98% rename from arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S rename to lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S index 89f4a10d12dd..b41eebf60546 100644 --- a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S +++ b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S @@ -93,8 +93,8 @@ sha512_4rounds \last, W3, W0, W1, W2 .endm -// void sha512_transform_zvknhb_zvkb(u64 state[8], const u8 *data, -// int num_blocks); +// void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state, +// const u8 *data, size_t nblocks); SYM_FUNC_START(sha512_transform_zvknhb_zvkb) // Setup mask for the vmerge to replace the first word (idx==0) in diff --git a/lib/crypto/riscv/sha512.h b/lib/crypto/riscv/sha512.h new file mode 100644 index 000000000000..9d0abede322f --- /dev/null +++ b/lib/crypto/riscv/sha512.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-512 and SHA-384 using the RISC-V vector crypto extensions + * + * Copyright (C) 2023 VRULL GmbH + * Author: Heiko Stuebner + * + * Copyright (C) 2023 SiFive, Inc. + * Author: Jerry Shih + */ + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); + +asmlinkage void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state, + const u8 *data, size_t nblocks); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_extensions) && + likely(crypto_simd_usable())) { + kernel_vector_begin(); + sha512_transform_zvknhb_zvkb(state, data, nblocks); + kernel_vector_end(); + } else { + sha512_blocks_generic(state, data, nblocks); + } +} + +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKNHB) && + riscv_isa_extension_available(NULL, ZVKB) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_extensions); +} From b7b366087e0f1645f867077a14bab617516d0f57 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:16 -0700 Subject: [PATCH 13/67] lib/crypto: s390/sha512: Migrate optimized SHA-512 code to library Instead of exposing the s390-optimized SHA-512 code via s390-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be s390-optimized, and it fixes the longstanding issue where the s390-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/s390/configs/debug_defconfig | 1 - arch/s390/configs/defconfig | 1 - arch/s390/crypto/Kconfig | 10 -- arch/s390/crypto/Makefile | 1 - arch/s390/crypto/sha512_s390.c | 151 ------------------------------ lib/crypto/Kconfig | 1 + lib/crypto/s390/sha512.h | 28 ++++++ 7 files changed, 29 insertions(+), 164 deletions(-) delete mode 100644 arch/s390/crypto/sha512_s390.c create mode 100644 lib/crypto/s390/sha512.h diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 8ecad727497e..ef313c30b375 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -804,7 +804,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_SHA512_S390=m CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA3_256_S390=m CONFIG_CRYPTO_SHA3_512_S390=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index c13a77765162..b6fa341bb03b 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -791,7 +791,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_SHA512_S390=m CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA3_256_S390=m CONFIG_CRYPTO_SHA3_512_S390=m diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index e2c27588b21a..4557514fbac3 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -2,16 +2,6 @@ menu "Accelerated Cryptographic Algorithms for CPU (s390)" -config CRYPTO_SHA512_S390 - tristate "Hash functions: SHA-384 and SHA-512" - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: s390 - - It is available as of z10. - config CRYPTO_SHA1_S390 tristate "Hash functions: SHA-1" select CRYPTO_HASH diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 21757d86cd49..473d64c0982a 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -4,7 +4,6 @@ # obj-$(CONFIG_CRYPTO_SHA1_S390) += sha1_s390.o sha_common.o -obj-$(CONFIG_CRYPTO_SHA512_S390) += sha512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_256_S390) += sha3_256_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_512_S390) += sha3_512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c deleted file mode 100644 index e8bb172dbed7..000000000000 --- a/arch/s390/crypto/sha512_s390.c +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Cryptographic API. - * - * s390 implementation of the SHA512 and SHA38 Secure Hash Algorithm. - * - * Copyright IBM Corp. 2007 - * Author(s): Jan Glauber (jang@de.ibm.com) - */ -#include -#include -#include -#include -#include -#include -#include - -#include "sha.h" - -static int sha512_init_s390(struct shash_desc *desc) -{ - struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - - ctx->sha512.state[0] = SHA512_H0; - ctx->sha512.state[1] = SHA512_H1; - ctx->sha512.state[2] = SHA512_H2; - ctx->sha512.state[3] = SHA512_H3; - ctx->sha512.state[4] = SHA512_H4; - ctx->sha512.state[5] = SHA512_H5; - ctx->sha512.state[6] = SHA512_H6; - ctx->sha512.state[7] = SHA512_H7; - ctx->count = 0; - ctx->sha512.count_hi = 0; - ctx->func = CPACF_KIMD_SHA_512; - - return 0; -} - -static int sha512_export(struct shash_desc *desc, void *out) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - struct sha512_state *octx = out; - - octx->count[0] = sctx->count; - octx->count[1] = sctx->sha512.count_hi; - memcpy(octx->state, sctx->state, sizeof(octx->state)); - return 0; -} - -static int sha512_import(struct shash_desc *desc, const void *in) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha512_state *ictx = in; - - sctx->count = ictx->count[0]; - sctx->sha512.count_hi = ictx->count[1]; - - memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - sctx->func = CPACF_KIMD_SHA_512; - return 0; -} - -static struct shash_alg sha512_alg = { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_init_s390, - .update = s390_sha_update_blocks, - .finup = s390_sha_finup, - .export = sha512_export, - .import = sha512_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name= "sha512-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -MODULE_ALIAS_CRYPTO("sha512"); - -static int sha384_init_s390(struct shash_desc *desc) -{ - struct s390_sha_ctx *ctx = shash_desc_ctx(desc); - - ctx->sha512.state[0] = SHA384_H0; - ctx->sha512.state[1] = SHA384_H1; - ctx->sha512.state[2] = SHA384_H2; - ctx->sha512.state[3] = SHA384_H3; - ctx->sha512.state[4] = SHA384_H4; - ctx->sha512.state[5] = SHA384_H5; - ctx->sha512.state[6] = SHA384_H6; - ctx->sha512.state[7] = SHA384_H7; - ctx->count = 0; - ctx->sha512.count_hi = 0; - ctx->func = CPACF_KIMD_SHA_512; - - return 0; -} - -static struct shash_alg sha384_alg = { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_init_s390, - .update = s390_sha_update_blocks, - .finup = s390_sha_finup, - .export = sha512_export, - .import = sha512_import, - .descsize = sizeof(struct s390_sha_ctx), - .statesize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name= "sha384-s390", - .cra_priority = 300, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_ctxsize = sizeof(struct s390_sha_ctx), - .cra_module = THIS_MODULE, - } -}; - -MODULE_ALIAS_CRYPTO("sha384"); - -static int __init init(void) -{ - int ret; - - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_512)) - return -ENODEV; - if ((ret = crypto_register_shash(&sha512_alg)) < 0) - goto out; - if ((ret = crypto_register_shash(&sha384_alg)) < 0) - crypto_unregister_shash(&sha512_alg); -out: - return ret; -} - -static void __exit fini(void) -{ - crypto_unregister_shash(&sha512_alg); - crypto_unregister_shash(&sha384_alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, init); -module_exit(fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA512 and SHA-384 Secure Hash Algorithm"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 656f2c08c46c..6f1386b8d79d 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -181,6 +181,7 @@ config CRYPTO_LIB_SHA512_ARCH default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + default y if S390 config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/s390/sha512.h b/lib/crypto/s390/sha512.h new file mode 100644 index 000000000000..24744651550c --- /dev/null +++ b/lib/crypto/s390/sha512.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-512 optimized using the CP Assist for Cryptographic Functions (CPACF) + * + * Copyright 2025 Google LLC + */ +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha512); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_cpacf_sha512)) + cpacf_kimd(CPACF_KIMD_SHA_512, state, data, + nblocks * SHA512_BLOCK_SIZE); + else + sha512_blocks_generic(state, data, nblocks); +} + +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (cpu_have_feature(S390_CPU_FEATURE_MSA) && + cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_512)) + static_branch_enable(&have_cpacf_sha512); +} From 02b35bab7e6c5bb2a843828316d528216b8cedc8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:17 -0700 Subject: [PATCH 14/67] lib/crypto: sparc/sha512: Migrate optimized SHA-512 code to library Instead of exposing the sparc-optimized SHA-512 code via sparc-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be sparc-optimized, and it fixes the longstanding issue where the sparc-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly function from int to size_t. The assembly function actually already treated it as size_t. Note: to see the diff from arch/sparc/crypto/sha512_glue.c to lib/crypto/sparc/sha512.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-14-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/sparc/crypto/Kconfig | 10 -- arch/sparc/crypto/Makefile | 2 - arch/sparc/crypto/sha512_glue.c | 122 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 1 + lib/crypto/sparc/sha512.h | 42 ++++++ .../crypto => lib/crypto/sparc}/sha512_asm.S | 0 7 files changed, 44 insertions(+), 134 deletions(-) delete mode 100644 arch/sparc/crypto/sha512_glue.c create mode 100644 lib/crypto/sparc/sha512.h rename {arch/sparc/crypto => lib/crypto/sparc}/sha512_asm.S (100%) diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index a6ba319c42dc..9d8da9aef3a4 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -36,16 +36,6 @@ config CRYPTO_SHA1_SPARC64 Architecture: sparc64 -config CRYPTO_SHA512_SPARC64 - tristate "Hash functions: SHA-384 and SHA-512" - depends on SPARC64 - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: sparc64 using crypto instructions, when available - config CRYPTO_AES_SPARC64 tristate "Ciphers: AES, modes: ECB, CBC, CTR" depends on SPARC64 diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile index 701c39edb0d7..99a7e8fd13bc 100644 --- a/arch/sparc/crypto/Makefile +++ b/arch/sparc/crypto/Makefile @@ -4,7 +4,6 @@ # obj-$(CONFIG_CRYPTO_SHA1_SPARC64) += sha1-sparc64.o -obj-$(CONFIG_CRYPTO_SHA512_SPARC64) += sha512-sparc64.o obj-$(CONFIG_CRYPTO_MD5_SPARC64) += md5-sparc64.o obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o @@ -12,7 +11,6 @@ obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o sha1-sparc64-y := sha1_asm.o sha1_glue.o -sha512-sparc64-y := sha512_asm.o sha512_glue.o md5-sparc64-y := md5_asm.o md5_glue.o aes-sparc64-y := aes_asm.o aes_glue.o diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c deleted file mode 100644 index fb81c3290c8c..000000000000 --- a/arch/sparc/crypto/sha512_glue.c +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Glue code for SHA512 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon crypto/sha512_generic.c - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2003 Kyle McMartin - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, - unsigned int rounds); - -static void sha512_block(struct sha512_state *sctx, const u8 *src, int blocks) -{ - sha512_sparc64_transform(sctx->state, src, blocks); -} - -static int sha512_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, sha512_block); -} - -static int sha512_sparc64_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, src, len, sha512_block); - return sha512_base_finish(desc, out); -} - -static struct shash_alg sha512_alg = { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_base_init, - .update = sha512_sparc64_update, - .finup = sha512_sparc64_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name= "sha512-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static struct shash_alg sha384_alg = { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_base_init, - .update = sha512_sparc64_update, - .finup = sha512_sparc64_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name= "sha384-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool __init sparc64_has_sha512_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_SHA512)) - return false; - - return true; -} - -static int __init sha512_sparc64_mod_init(void) -{ - if (sparc64_has_sha512_opcode()) { - int ret = crypto_register_shash(&sha384_alg); - if (ret < 0) - return ret; - - ret = crypto_register_shash(&sha512_alg); - if (ret < 0) { - crypto_unregister_shash(&sha384_alg); - return ret; - } - - pr_info("Using sparc64 sha512 opcode optimized SHA-512/SHA-384 implementation\n"); - return 0; - } - pr_info("sparc64 sha512 opcode not available.\n"); - return -ENODEV; -} - -static void __exit sha512_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&sha384_alg); - crypto_unregister_shash(&sha512_alg); -} - -module_init(sha512_sparc64_mod_init); -module_exit(sha512_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-384 and SHA-512 Secure Hash Algorithm, sparc64 sha512 opcode accelerated"); - -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha512"); - -#include "crop_devid.c" diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 6f1386b8d79d..c3edb78484f0 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -182,6 +182,7 @@ config CRYPTO_LIB_SHA512_ARCH default y if MIPS && CPU_CAVIUM_OCTEON default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO default y if S390 + default y if SPARC64 config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index bc42464a279b..0fb93f2469b3 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -94,6 +94,7 @@ libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o endif libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o +libsha512-$(CONFIG_SPARC) += sparc/sha512_asm.o endif # CONFIG_CRYPTO_LIB_SHA512_ARCH obj-$(CONFIG_MPILIB) += mpi/ diff --git a/lib/crypto/sparc/sha512.h b/lib/crypto/sparc/sha512.h new file mode 100644 index 000000000000..55303ab6b15f --- /dev/null +++ b/lib/crypto/sparc/sha512.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * SHA-512 accelerated using the sparc64 sha512 opcodes + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2003 Kyle McMartin + */ + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_opcodes); + +asmlinkage void sha512_sparc64_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_sha512_opcodes)) + sha512_sparc64_transform(state, data, nblocks); + else + sha512_blocks_generic(state, data, nblocks); +} + +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_SHA512)) + return; + + static_branch_enable(&have_sha512_opcodes); + pr_info("Using sparc64 sha512 opcode optimized SHA-512/SHA-384 implementation\n"); +} diff --git a/arch/sparc/crypto/sha512_asm.S b/lib/crypto/sparc/sha512_asm.S similarity index 100% rename from arch/sparc/crypto/sha512_asm.S rename to lib/crypto/sparc/sha512_asm.S From 484c18119f4fbc6bca7c41e64b6fa84133e1057b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:18 -0700 Subject: [PATCH 15/67] lib/crypto: x86/sha512: Migrate optimized SHA-512 code to library Instead of exposing the x86-optimized SHA-512 code via x86-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be x86-optimized, and it fixes the longstanding issue where the x86-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-15-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/x86/crypto/Kconfig | 13 - arch/x86/crypto/Makefile | 3 - arch/x86/crypto/sha512_ssse3_glue.c | 322 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 3 + .../crypto/x86}/sha512-avx-asm.S | 9 +- .../crypto/x86}/sha512-avx2-asm.S | 9 +- .../crypto/x86}/sha512-ssse3-asm.S | 10 +- lib/crypto/x86/sha512.h | 54 +++ 9 files changed, 72 insertions(+), 352 deletions(-) delete mode 100644 arch/x86/crypto/sha512_ssse3_glue.c rename {arch/x86/crypto => lib/crypto/x86}/sha512-avx-asm.S (98%) rename {arch/x86/crypto => lib/crypto/x86}/sha512-avx2-asm.S (99%) rename {arch/x86/crypto => lib/crypto/x86}/sha512-ssse3-asm.S (97%) create mode 100644 lib/crypto/x86/sha512.h diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 56cfdc79e2c6..eb641a300154 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -390,19 +390,6 @@ config CRYPTO_SHA1_SSSE3 - AVX2 (Advanced Vector Extensions 2) - SHA-NI (SHA Extensions New Instructions) -config CRYPTO_SHA512_SSSE3 - tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)" - depends on 64BIT - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: x86_64 using: - - SSSE3 (Supplemental SSE3) - - AVX (Advanced Vector Extensions) - - AVX2 (Advanced Vector Extensions 2) - config CRYPTO_SM3_AVX_X86_64 tristate "Hash functions: SM3 (AVX)" depends on 64BIT diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index aa289a9e0153..d31348be8370 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -54,9 +54,6 @@ endif obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sha1_ssse3_glue.o -obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o -sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o - obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c deleted file mode 100644 index 97744b7d2381..000000000000 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Cryptographic API. - * - * Glue code for the SHA512 Secure Hash Algorithm assembler - * implementation using supplemental SSE3 / AVX / AVX2 instructions. - * - * This file is based on sha512_generic.c - * - * Copyright (C) 2013 Intel Corporation - * Author: Tim Chen - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha512_transform_ssse3(struct sha512_state *state, - const u8 *data, int blocks); - -static int sha512_update_x86(struct shash_desc *desc, const u8 *data, - unsigned int len, sha512_block_fn *sha512_xform) -{ - int remain; - - /* - * Make sure struct sha512_state begins directly with the SHA512 - * 512-bit internal state, as this is what the asm functions expect. - */ - BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); - - kernel_fpu_begin(); - remain = sha512_base_do_update_blocks(desc, data, len, sha512_xform); - kernel_fpu_end(); - - return remain; -} - -static int sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, sha512_block_fn *sha512_xform) -{ - kernel_fpu_begin(); - sha512_base_do_finup(desc, data, len, sha512_xform); - kernel_fpu_end(); - - return sha512_base_finish(desc, out); -} - -static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_update_x86(desc, data, len, sha512_transform_ssse3); -} - -static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_finup(desc, data, len, out, sha512_transform_ssse3); -} - -static struct shash_alg sha512_ssse3_algs[] = { { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_base_init, - .update = sha512_ssse3_update, - .finup = sha512_ssse3_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name = "sha512-ssse3", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_base_init, - .update = sha512_ssse3_update, - .finup = sha512_ssse3_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name = "sha384-ssse3", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int register_sha512_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - return crypto_register_shashes(sha512_ssse3_algs, - ARRAY_SIZE(sha512_ssse3_algs)); - return 0; -} - -static void unregister_sha512_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_shashes(sha512_ssse3_algs, - ARRAY_SIZE(sha512_ssse3_algs)); -} - -asmlinkage void sha512_transform_avx(struct sha512_state *state, - const u8 *data, int blocks); -static bool avx_usable(void) -{ - if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { - if (boot_cpu_has(X86_FEATURE_AVX)) - pr_info("AVX detected but unusable.\n"); - return false; - } - - return true; -} - -static int sha512_avx_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_update_x86(desc, data, len, sha512_transform_avx); -} - -static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_finup(desc, data, len, out, sha512_transform_avx); -} - -static struct shash_alg sha512_avx_algs[] = { { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_base_init, - .update = sha512_avx_update, - .finup = sha512_avx_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name = "sha512-avx", - .cra_priority = 160, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_base_init, - .update = sha512_avx_update, - .finup = sha512_avx_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name = "sha384-avx", - .cra_priority = 160, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int register_sha512_avx(void) -{ - if (avx_usable()) - return crypto_register_shashes(sha512_avx_algs, - ARRAY_SIZE(sha512_avx_algs)); - return 0; -} - -static void unregister_sha512_avx(void) -{ - if (avx_usable()) - crypto_unregister_shashes(sha512_avx_algs, - ARRAY_SIZE(sha512_avx_algs)); -} - -asmlinkage void sha512_transform_rorx(struct sha512_state *state, - const u8 *data, int blocks); - -static int sha512_avx2_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_update_x86(desc, data, len, sha512_transform_rorx); -} - -static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_finup(desc, data, len, out, sha512_transform_rorx); -} - -static struct shash_alg sha512_avx2_algs[] = { { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_base_init, - .update = sha512_avx2_update, - .finup = sha512_avx2_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha512", - .cra_driver_name = "sha512-avx2", - .cra_priority = 170, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_base_init, - .update = sha512_avx2_update, - .finup = sha512_avx2_finup, - .descsize = SHA512_STATE_SIZE, - .base = { - .cra_name = "sha384", - .cra_driver_name = "sha384-avx2", - .cra_priority = 170, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static bool avx2_usable(void) -{ - if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_BMI2)) - return true; - - return false; -} - -static int register_sha512_avx2(void) -{ - if (avx2_usable()) - return crypto_register_shashes(sha512_avx2_algs, - ARRAY_SIZE(sha512_avx2_algs)); - return 0; -} -static const struct x86_cpu_id module_cpu_ids[] = { - X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), - X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), - X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); - -static void unregister_sha512_avx2(void) -{ - if (avx2_usable()) - crypto_unregister_shashes(sha512_avx2_algs, - ARRAY_SIZE(sha512_avx2_algs)); -} - -static int __init sha512_ssse3_mod_init(void) -{ - if (!x86_match_cpu(module_cpu_ids)) - return -ENODEV; - - if (register_sha512_ssse3()) - goto fail; - - if (register_sha512_avx()) { - unregister_sha512_ssse3(); - goto fail; - } - - if (register_sha512_avx2()) { - unregister_sha512_avx(); - unregister_sha512_ssse3(); - goto fail; - } - - return 0; -fail: - return -ENODEV; -} - -static void __exit sha512_ssse3_mod_fini(void) -{ - unregister_sha512_avx2(); - unregister_sha512_avx(); - unregister_sha512_ssse3(); -} - -module_init(sha512_ssse3_mod_init); -module_exit(sha512_ssse3_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); - -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha512-ssse3"); -MODULE_ALIAS_CRYPTO("sha512-avx"); -MODULE_ALIAS_CRYPTO("sha512-avx2"); -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha384-ssse3"); -MODULE_ALIAS_CRYPTO("sha384-avx"); -MODULE_ALIAS_CRYPTO("sha384-avx2"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index c3edb78484f0..dce127a69f13 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -183,6 +183,7 @@ config CRYPTO_LIB_SHA512_ARCH default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO default y if S390 default y if SPARC64 + default y if X86_64 config CRYPTO_LIB_SM3 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 0fb93f2469b3..aaf445a85384 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -95,6 +95,9 @@ endif libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o libsha512-$(CONFIG_SPARC) += sparc/sha512_asm.o +libsha512-$(CONFIG_X86) += x86/sha512-ssse3-asm.o \ + x86/sha512-avx-asm.o \ + x86/sha512-avx2-asm.o endif # CONFIG_CRYPTO_LIB_SHA512_ARCH obj-$(CONFIG_MPILIB) += mpi/ diff --git a/arch/x86/crypto/sha512-avx-asm.S b/lib/crypto/x86/sha512-avx-asm.S similarity index 98% rename from arch/x86/crypto/sha512-avx-asm.S rename to lib/crypto/x86/sha512-avx-asm.S index 5bfce4b045fd..0b5f69179d62 100644 --- a/arch/x86/crypto/sha512-avx-asm.S +++ b/lib/crypto/x86/sha512-avx-asm.S @@ -48,7 +48,6 @@ ######################################################################## #include -#include .text @@ -267,14 +266,16 @@ frame_size = frame_WK + WK_SIZE .endm ######################################################################## -# void sha512_transform_avx(sha512_state *state, const u8 *data, int blocks) +# void sha512_transform_avx(struct sha512_block_state *state, +# const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "blocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks ######################################################################## -SYM_TYPED_FUNC_START(sha512_transform_avx) +SYM_FUNC_START(sha512_transform_avx) + test msglen, msglen je .Lnowork diff --git a/arch/x86/crypto/sha512-avx2-asm.S b/lib/crypto/x86/sha512-avx2-asm.S similarity index 99% rename from arch/x86/crypto/sha512-avx2-asm.S rename to lib/crypto/x86/sha512-avx2-asm.S index 24973f42c43f..2309c01e316b 100644 --- a/arch/x86/crypto/sha512-avx2-asm.S +++ b/lib/crypto/x86/sha512-avx2-asm.S @@ -50,7 +50,6 @@ ######################################################################## #include -#include .text @@ -559,14 +558,16 @@ frame_size = frame_CTX + CTX_SIZE .endm ######################################################################## -# void sha512_transform_rorx(sha512_state *state, const u8 *data, int blocks) +# void sha512_transform_rorx(struct sha512_block_state *state, +# const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "blocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks ######################################################################## -SYM_TYPED_FUNC_START(sha512_transform_rorx) +SYM_FUNC_START(sha512_transform_rorx) + # Save GPRs push %rbx push %r12 diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S similarity index 97% rename from arch/x86/crypto/sha512-ssse3-asm.S rename to lib/crypto/x86/sha512-ssse3-asm.S index 30a2c4777f9d..12e78142f2e3 100644 --- a/arch/x86/crypto/sha512-ssse3-asm.S +++ b/lib/crypto/x86/sha512-ssse3-asm.S @@ -48,7 +48,6 @@ ######################################################################## #include -#include .text @@ -266,16 +265,15 @@ frame_size = frame_WK + WK_SIZE .endm ######################################################################## -## void sha512_transform_ssse3(struct sha512_state *state, const u8 *data, -## int blocks); -# (struct sha512_state is assumed to begin with u64 state[8]) +# void sha512_transform_ssse3(struct sha512_block_state *state, +# const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "blocks" is the message length in SHA512 blocks. +# "nblocks" is the message length in SHA512 blocks ######################################################################## -SYM_TYPED_FUNC_START(sha512_transform_ssse3) +SYM_FUNC_START(sha512_transform_ssse3) test msglen, msglen je .Lnowork diff --git a/lib/crypto/x86/sha512.h b/lib/crypto/x86/sha512.h new file mode 100644 index 000000000000..c13503d9d57d --- /dev/null +++ b/lib/crypto/x86/sha512.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * x86-optimized SHA-512 block function + * + * Copyright 2025 Google LLC + */ + +#include +#include +#include + +DEFINE_STATIC_CALL(sha512_blocks_x86, sha512_blocks_generic); + +#define DEFINE_X86_SHA512_FN(c_fn, asm_fn) \ + asmlinkage void asm_fn(struct sha512_block_state *state, \ + const u8 *data, size_t nblocks); \ + static void c_fn(struct sha512_block_state *state, const u8 *data, \ + size_t nblocks) \ + { \ + if (likely(crypto_simd_usable())) { \ + kernel_fpu_begin(); \ + asm_fn(state, data, nblocks); \ + kernel_fpu_end(); \ + } else { \ + sha512_blocks_generic(state, data, nblocks); \ + } \ + } + +DEFINE_X86_SHA512_FN(sha512_blocks_ssse3, sha512_transform_ssse3); +DEFINE_X86_SHA512_FN(sha512_blocks_avx, sha512_transform_avx); +DEFINE_X86_SHA512_FN(sha512_blocks_avx2, sha512_transform_rorx); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + static_call(sha512_blocks_x86)(state, data, nblocks); +} + +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && + boot_cpu_has(X86_FEATURE_AVX)) { + if (boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_BMI2)) + static_call_update(sha512_blocks_x86, + sha512_blocks_avx2); + else + static_call_update(sha512_blocks_x86, + sha512_blocks_avx); + } else if (boot_cpu_has(X86_FEATURE_SSSE3)) { + static_call_update(sha512_blocks_x86, sha512_blocks_ssse3); + } +} From 6486f2b0368dca5b7ba003d3904bfc64ff2439d5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:19 -0700 Subject: [PATCH 16/67] lib/crypto: x86/sha512: Remove unnecessary checks for nblocks==0 Since sha512_blocks() is called only with nblocks >= 1, remove unnecessary checks for nblocks == 0 from the x86 SHA-512 assembly code. Link: https://lore.kernel.org/r/20250630160320.2888-16-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/x86/sha512-avx-asm.S | 6 +----- lib/crypto/x86/sha512-avx2-asm.S | 5 +---- lib/crypto/x86/sha512-ssse3-asm.S | 6 +----- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/lib/crypto/x86/sha512-avx-asm.S b/lib/crypto/x86/sha512-avx-asm.S index 0b5f69179d62..7732aa8fd850 100644 --- a/lib/crypto/x86/sha512-avx-asm.S +++ b/lib/crypto/x86/sha512-avx-asm.S @@ -272,13 +272,10 @@ frame_size = frame_WK + WK_SIZE # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "nblocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks. Must be >= 1. ######################################################################## SYM_FUNC_START(sha512_transform_avx) - test msglen, msglen - je .Lnowork - # Save GPRs push %rbx push %r12 @@ -362,7 +359,6 @@ SYM_FUNC_START(sha512_transform_avx) pop %r12 pop %rbx -.Lnowork: RET SYM_FUNC_END(sha512_transform_avx) diff --git a/lib/crypto/x86/sha512-avx2-asm.S b/lib/crypto/x86/sha512-avx2-asm.S index 2309c01e316b..22bdbfd899d0 100644 --- a/lib/crypto/x86/sha512-avx2-asm.S +++ b/lib/crypto/x86/sha512-avx2-asm.S @@ -564,7 +564,7 @@ frame_size = frame_CTX + CTX_SIZE # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "nblocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks. Must be >= 1. ######################################################################## SYM_FUNC_START(sha512_transform_rorx) @@ -582,7 +582,6 @@ SYM_FUNC_START(sha512_transform_rorx) and $~(0x20 - 1), %rsp shl $7, NUM_BLKS # convert to bytes - jz .Ldone_hash add INP, NUM_BLKS # pointer to end of data mov NUM_BLKS, frame_INPEND(%rsp) @@ -668,8 +667,6 @@ SYM_FUNC_START(sha512_transform_rorx) cmp frame_INPEND(%rsp), INP jne .Lloop0 -.Ldone_hash: - # Restore Stack Pointer mov %rbp, %rsp pop %rbp diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S index 12e78142f2e3..4cae7445b2a8 100644 --- a/lib/crypto/x86/sha512-ssse3-asm.S +++ b/lib/crypto/x86/sha512-ssse3-asm.S @@ -271,13 +271,10 @@ frame_size = frame_WK + WK_SIZE # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "nblocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks. Must be >= 1. ######################################################################## SYM_FUNC_START(sha512_transform_ssse3) - test msglen, msglen - je .Lnowork - # Save GPRs push %rbx push %r12 @@ -361,7 +358,6 @@ SYM_FUNC_START(sha512_transform_ssse3) pop %r12 pop %rbx -.Lnowork: RET SYM_FUNC_END(sha512_transform_ssse3) From 9b5c0d82b26d10733f67e10ea1889fc24aa6840a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:03:20 -0700 Subject: [PATCH 17/67] crypto: sha512 - Remove sha512_base.h sha512_base.h is no longer used, so remove it. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160320.2888-17-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha512_base.h | 117 ----------------------------------- 1 file changed, 117 deletions(-) delete mode 100644 include/crypto/sha512_base.h diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h deleted file mode 100644 index d1361b3eb70b..000000000000 --- a/include/crypto/sha512_base.h +++ /dev/null @@ -1,117 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sha512_base.h - core logic for SHA-512 implementations - * - * Copyright (C) 2015 Linaro Ltd - */ - -#ifndef _CRYPTO_SHA512_BASE_H -#define _CRYPTO_SHA512_BASE_H - -#include -#include -#include -#include -#include -#include -#include - -typedef void (sha512_block_fn)(struct sha512_state *sst, u8 const *src, - int blocks); - -static inline int sha384_base_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA384_H0; - sctx->state[1] = SHA384_H1; - sctx->state[2] = SHA384_H2; - sctx->state[3] = SHA384_H3; - sctx->state[4] = SHA384_H4; - sctx->state[5] = SHA384_H5; - sctx->state[6] = SHA384_H6; - sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static inline int sha512_base_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA512_H0; - sctx->state[1] = SHA512_H1; - sctx->state[2] = SHA512_H2; - sctx->state[3] = SHA512_H3; - sctx->state[4] = SHA512_H4; - sctx->state[5] = SHA512_H5; - sctx->state[6] = SHA512_H6; - sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static inline int sha512_base_do_update_blocks(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha512_block_fn *block_fn) -{ - unsigned int remain = len - round_down(len, SHA512_BLOCK_SIZE); - struct sha512_state *sctx = shash_desc_ctx(desc); - - len -= remain; - sctx->count[0] += len; - if (sctx->count[0] < len) - sctx->count[1]++; - block_fn(sctx, data, len / SHA512_BLOCK_SIZE); - return remain; -} - -static inline int sha512_base_do_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, - sha512_block_fn *block_fn) -{ - unsigned int bit_offset = SHA512_BLOCK_SIZE / 8 - 2; - struct sha512_state *sctx = shash_desc_ctx(desc); - union { - __be64 b64[SHA512_BLOCK_SIZE / 4]; - u8 u8[SHA512_BLOCK_SIZE * 2]; - } block = {}; - - if (len >= SHA512_BLOCK_SIZE) { - int remain; - - remain = sha512_base_do_update_blocks(desc, src, len, block_fn); - src += len - remain; - len = remain; - } - - if (len >= bit_offset * 8) - bit_offset += SHA512_BLOCK_SIZE / 8; - memcpy(&block, src, len); - block.u8[len] = 0x80; - sctx->count[0] += len; - block.b64[bit_offset] = cpu_to_be64(sctx->count[1] << 3 | - sctx->count[0] >> 61); - block.b64[bit_offset + 1] = cpu_to_be64(sctx->count[0] << 3); - block_fn(sctx, block.u8, (bit_offset + 2) * 8 / SHA512_BLOCK_SIZE); - memzero_explicit(&block, sizeof(block)); - - return 0; -} - -static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) -{ - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - struct sha512_state *sctx = shash_desc_ctx(desc); - __be64 *digest = (__be64 *)out; - int i; - - for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64)) - put_unaligned_be64(sctx->state[i], digest++); - return 0; -} - -#endif /* _CRYPTO_SHA512_BASE_H */ From 4a32e5dc1dcfa8f1ed6ca33328e71dcb6196c09e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:00 -0700 Subject: [PATCH 18/67] lib/crypto: arm: Move arch/arm/lib/crypto/ into lib/crypto/ Move the contents of arch/arm/lib/crypto/ into lib/crypto/arm/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Add a gitignore entry for the removed directory arch/arm/lib/crypto/ so that people don't accidentally commit leftover generated files. Acked-by: Ard Biesheuvel Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm/lib/.gitignore | 4 ++++ arch/arm/lib/Makefile | 2 -- arch/arm/lib/crypto/.gitignore | 3 --- lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 2 ++ lib/crypto/arm/.gitignore | 2 ++ {arch/arm/lib/crypto => lib/crypto/arm}/Kconfig | 0 {arch/arm/lib/crypto => lib/crypto/arm}/Makefile | 0 {arch/arm/lib/crypto => lib/crypto/arm}/blake2s-core.S | 0 {arch/arm/lib/crypto => lib/crypto/arm}/blake2s-glue.c | 0 {arch/arm/lib/crypto => lib/crypto/arm}/chacha-glue.c | 0 {arch/arm/lib/crypto => lib/crypto/arm}/chacha-neon-core.S | 0 {arch/arm/lib/crypto => lib/crypto/arm}/chacha-scalar-core.S | 0 {arch/arm/lib/crypto => lib/crypto/arm}/poly1305-armv4.pl | 0 {arch/arm/lib/crypto => lib/crypto/arm}/poly1305-glue.c | 0 {arch/arm/lib/crypto => lib/crypto/arm}/sha256-armv4.pl | 0 {arch/arm/lib/crypto => lib/crypto/arm}/sha256-ce.S | 0 {arch/arm/lib/crypto => lib/crypto/arm}/sha256.c | 0 18 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 arch/arm/lib/.gitignore delete mode 100644 arch/arm/lib/crypto/.gitignore rename {arch/arm/lib/crypto => lib/crypto/arm}/Kconfig (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/Makefile (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/blake2s-core.S (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/blake2s-glue.c (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/chacha-glue.c (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/chacha-neon-core.S (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/chacha-scalar-core.S (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/poly1305-armv4.pl (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/poly1305-glue.c (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/sha256-armv4.pl (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/sha256-ce.S (100%) rename {arch/arm/lib/crypto => lib/crypto/arm}/sha256.c (100%) diff --git a/arch/arm/lib/.gitignore b/arch/arm/lib/.gitignore new file mode 100644 index 000000000000..647d7a922e68 --- /dev/null +++ b/arch/arm/lib/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +# This now-removed directory used to contain generated files. +/crypto/ diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile index 91ea0e29107a..d05dd672bcd9 100644 --- a/arch/arm/lib/Makefile +++ b/arch/arm/lib/Makefile @@ -5,8 +5,6 @@ # Copyright (C) 1995-2000 Russell King # -obj-y += crypto/ - lib-y := changebit.o csumipv6.o csumpartial.o \ csumpartialcopy.o csumpartialcopyuser.o clearbit.o \ delay.o delay-loop.o findbit.o memchr.o memcpy.o \ diff --git a/arch/arm/lib/crypto/.gitignore b/arch/arm/lib/crypto/.gitignore deleted file mode 100644 index 12d74d8b03d0..000000000000 --- a/arch/arm/lib/crypto/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -poly1305-core.S -sha256-core.S diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index dce127a69f13..e14bef8e87af 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -190,7 +190,7 @@ config CRYPTO_LIB_SM3 if !KMSAN # avoid false positives from assembly if ARM -source "arch/arm/lib/crypto/Kconfig" +source "lib/crypto/arm/Kconfig" endif if ARM64 source "arch/arm64/lib/crypto/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index aaf445a85384..5f2b81f82a85 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -106,3 +106,5 @@ obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o libsm3-y := sm3.o + +obj-$(CONFIG_ARM) += arm/ diff --git a/lib/crypto/arm/.gitignore b/lib/crypto/arm/.gitignore index 670a4d97b568..f6c4e8ef80da 100644 --- a/lib/crypto/arm/.gitignore +++ b/lib/crypto/arm/.gitignore @@ -1,2 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only +poly1305-core.S +sha256-core.S sha512-core.S diff --git a/arch/arm/lib/crypto/Kconfig b/lib/crypto/arm/Kconfig similarity index 100% rename from arch/arm/lib/crypto/Kconfig rename to lib/crypto/arm/Kconfig diff --git a/arch/arm/lib/crypto/Makefile b/lib/crypto/arm/Makefile similarity index 100% rename from arch/arm/lib/crypto/Makefile rename to lib/crypto/arm/Makefile diff --git a/arch/arm/lib/crypto/blake2s-core.S b/lib/crypto/arm/blake2s-core.S similarity index 100% rename from arch/arm/lib/crypto/blake2s-core.S rename to lib/crypto/arm/blake2s-core.S diff --git a/arch/arm/lib/crypto/blake2s-glue.c b/lib/crypto/arm/blake2s-glue.c similarity index 100% rename from arch/arm/lib/crypto/blake2s-glue.c rename to lib/crypto/arm/blake2s-glue.c diff --git a/arch/arm/lib/crypto/chacha-glue.c b/lib/crypto/arm/chacha-glue.c similarity index 100% rename from arch/arm/lib/crypto/chacha-glue.c rename to lib/crypto/arm/chacha-glue.c diff --git a/arch/arm/lib/crypto/chacha-neon-core.S b/lib/crypto/arm/chacha-neon-core.S similarity index 100% rename from arch/arm/lib/crypto/chacha-neon-core.S rename to lib/crypto/arm/chacha-neon-core.S diff --git a/arch/arm/lib/crypto/chacha-scalar-core.S b/lib/crypto/arm/chacha-scalar-core.S similarity index 100% rename from arch/arm/lib/crypto/chacha-scalar-core.S rename to lib/crypto/arm/chacha-scalar-core.S diff --git a/arch/arm/lib/crypto/poly1305-armv4.pl b/lib/crypto/arm/poly1305-armv4.pl similarity index 100% rename from arch/arm/lib/crypto/poly1305-armv4.pl rename to lib/crypto/arm/poly1305-armv4.pl diff --git a/arch/arm/lib/crypto/poly1305-glue.c b/lib/crypto/arm/poly1305-glue.c similarity index 100% rename from arch/arm/lib/crypto/poly1305-glue.c rename to lib/crypto/arm/poly1305-glue.c diff --git a/arch/arm/lib/crypto/sha256-armv4.pl b/lib/crypto/arm/sha256-armv4.pl similarity index 100% rename from arch/arm/lib/crypto/sha256-armv4.pl rename to lib/crypto/arm/sha256-armv4.pl diff --git a/arch/arm/lib/crypto/sha256-ce.S b/lib/crypto/arm/sha256-ce.S similarity index 100% rename from arch/arm/lib/crypto/sha256-ce.S rename to lib/crypto/arm/sha256-ce.S diff --git a/arch/arm/lib/crypto/sha256.c b/lib/crypto/arm/sha256.c similarity index 100% rename from arch/arm/lib/crypto/sha256.c rename to lib/crypto/arm/sha256.c From 61f86c70cf419c826ee3b9c175d1e355844d8b45 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:01 -0700 Subject: [PATCH 19/67] lib/crypto: arm64: Move arch/arm64/lib/crypto/ into lib/crypto/ Move the contents of arch/arm64/lib/crypto/ into lib/crypto/arm64/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Add a gitignore entry for the removed directory arch/arm64/lib/crypto/ so that people don't accidentally commit leftover generated files. Acked-by: Ard Biesheuvel Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/lib/.gitignore | 4 ++++ arch/arm64/lib/Makefile | 3 --- arch/arm64/lib/crypto/.gitignore | 3 --- lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 3 ++- lib/crypto/arm64/.gitignore | 2 ++ {arch/arm64/lib/crypto => lib/crypto/arm64}/Kconfig | 0 {arch/arm64/lib/crypto => lib/crypto/arm64}/Makefile | 0 .../arm64/lib/crypto => lib/crypto/arm64}/chacha-neon-core.S | 0 .../arm64/lib/crypto => lib/crypto/arm64}/chacha-neon-glue.c | 0 {arch/arm64/lib/crypto => lib/crypto/arm64}/poly1305-armv8.pl | 0 {arch/arm64/lib/crypto => lib/crypto/arm64}/poly1305-glue.c | 0 {arch/arm64/lib/crypto => lib/crypto/arm64}/sha2-armv8.pl | 0 {arch/arm64/lib/crypto => lib/crypto/arm64}/sha256-ce.S | 0 {arch/arm64/lib/crypto => lib/crypto/arm64}/sha256.c | 0 15 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 arch/arm64/lib/.gitignore delete mode 100644 arch/arm64/lib/crypto/.gitignore rename {arch/arm64/lib/crypto => lib/crypto/arm64}/Kconfig (100%) rename {arch/arm64/lib/crypto => lib/crypto/arm64}/Makefile (100%) rename {arch/arm64/lib/crypto => lib/crypto/arm64}/chacha-neon-core.S (100%) rename {arch/arm64/lib/crypto => lib/crypto/arm64}/chacha-neon-glue.c (100%) rename {arch/arm64/lib/crypto => lib/crypto/arm64}/poly1305-armv8.pl (100%) rename {arch/arm64/lib/crypto => lib/crypto/arm64}/poly1305-glue.c (100%) rename {arch/arm64/lib/crypto => lib/crypto/arm64}/sha2-armv8.pl (100%) rename {arch/arm64/lib/crypto => lib/crypto/arm64}/sha256-ce.S (100%) rename {arch/arm64/lib/crypto => lib/crypto/arm64}/sha256.c (100%) diff --git a/arch/arm64/lib/.gitignore b/arch/arm64/lib/.gitignore new file mode 100644 index 000000000000..647d7a922e68 --- /dev/null +++ b/arch/arm64/lib/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +# This now-removed directory used to contain generated files. +/crypto/ diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 027bfa9689c6..d97e290619bc 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,7 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 - -obj-y += crypto/ - lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_page.o \ clear_page.o csum.o insn.o memchr.o memcpy.o \ diff --git a/arch/arm64/lib/crypto/.gitignore b/arch/arm64/lib/crypto/.gitignore deleted file mode 100644 index 12d74d8b03d0..000000000000 --- a/arch/arm64/lib/crypto/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -poly1305-core.S -sha256-core.S diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index e14bef8e87af..fdeb91bf0032 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -193,7 +193,7 @@ if ARM source "lib/crypto/arm/Kconfig" endif if ARM64 -source "arch/arm64/lib/crypto/Kconfig" +source "lib/crypto/arm64/Kconfig" endif if MIPS source "arch/mips/lib/crypto/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 5f2b81f82a85..19e9880c5d5f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -87,7 +87,7 @@ endif ifeq ($(CONFIG_ARM64),y) libsha512-y += arm64/sha512-core.o -$(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.pl +$(obj)/arm64/sha512-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) clean-files += arm64/sha512-core.S libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o @@ -108,3 +108,4 @@ obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o libsm3-y := sm3.o obj-$(CONFIG_ARM) += arm/ +obj-$(CONFIG_ARM64) += arm64/ diff --git a/lib/crypto/arm64/.gitignore b/lib/crypto/arm64/.gitignore index 670a4d97b568..f6c4e8ef80da 100644 --- a/lib/crypto/arm64/.gitignore +++ b/lib/crypto/arm64/.gitignore @@ -1,2 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only +poly1305-core.S +sha256-core.S sha512-core.S diff --git a/arch/arm64/lib/crypto/Kconfig b/lib/crypto/arm64/Kconfig similarity index 100% rename from arch/arm64/lib/crypto/Kconfig rename to lib/crypto/arm64/Kconfig diff --git a/arch/arm64/lib/crypto/Makefile b/lib/crypto/arm64/Makefile similarity index 100% rename from arch/arm64/lib/crypto/Makefile rename to lib/crypto/arm64/Makefile diff --git a/arch/arm64/lib/crypto/chacha-neon-core.S b/lib/crypto/arm64/chacha-neon-core.S similarity index 100% rename from arch/arm64/lib/crypto/chacha-neon-core.S rename to lib/crypto/arm64/chacha-neon-core.S diff --git a/arch/arm64/lib/crypto/chacha-neon-glue.c b/lib/crypto/arm64/chacha-neon-glue.c similarity index 100% rename from arch/arm64/lib/crypto/chacha-neon-glue.c rename to lib/crypto/arm64/chacha-neon-glue.c diff --git a/arch/arm64/lib/crypto/poly1305-armv8.pl b/lib/crypto/arm64/poly1305-armv8.pl similarity index 100% rename from arch/arm64/lib/crypto/poly1305-armv8.pl rename to lib/crypto/arm64/poly1305-armv8.pl diff --git a/arch/arm64/lib/crypto/poly1305-glue.c b/lib/crypto/arm64/poly1305-glue.c similarity index 100% rename from arch/arm64/lib/crypto/poly1305-glue.c rename to lib/crypto/arm64/poly1305-glue.c diff --git a/arch/arm64/lib/crypto/sha2-armv8.pl b/lib/crypto/arm64/sha2-armv8.pl similarity index 100% rename from arch/arm64/lib/crypto/sha2-armv8.pl rename to lib/crypto/arm64/sha2-armv8.pl diff --git a/arch/arm64/lib/crypto/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S similarity index 100% rename from arch/arm64/lib/crypto/sha256-ce.S rename to lib/crypto/arm64/sha256-ce.S diff --git a/arch/arm64/lib/crypto/sha256.c b/lib/crypto/arm64/sha256.c similarity index 100% rename from arch/arm64/lib/crypto/sha256.c rename to lib/crypto/arm64/sha256.c From 7e54e993ab8c98d912f54ad6f46bfcc9dcd65368 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:02 -0700 Subject: [PATCH 20/67] lib/crypto: mips: Move arch/mips/lib/crypto/ into lib/crypto/ Move the contents of arch/mips/lib/crypto/ into lib/crypto/mips/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Add a gitignore entry for the removed directory arch/mips/lib/crypto/ so that people don't accidentally commit leftover generated files. Acked-by: Ard Biesheuvel Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/mips/lib/.gitignore | 4 ++++ arch/mips/lib/Makefile | 2 -- lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 1 + {arch/mips/lib/crypto => lib/crypto/mips}/.gitignore | 0 {arch/mips/lib/crypto => lib/crypto/mips}/Kconfig | 0 {arch/mips/lib/crypto => lib/crypto/mips}/Makefile | 0 {arch/mips/lib/crypto => lib/crypto/mips}/chacha-core.S | 0 {arch/mips/lib/crypto => lib/crypto/mips}/chacha-glue.c | 0 {arch/mips/lib/crypto => lib/crypto/mips}/poly1305-glue.c | 0 {arch/mips/lib/crypto => lib/crypto/mips}/poly1305-mips.pl | 0 11 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 arch/mips/lib/.gitignore rename {arch/mips/lib/crypto => lib/crypto/mips}/.gitignore (100%) rename {arch/mips/lib/crypto => lib/crypto/mips}/Kconfig (100%) rename {arch/mips/lib/crypto => lib/crypto/mips}/Makefile (100%) rename {arch/mips/lib/crypto => lib/crypto/mips}/chacha-core.S (100%) rename {arch/mips/lib/crypto => lib/crypto/mips}/chacha-glue.c (100%) rename {arch/mips/lib/crypto => lib/crypto/mips}/poly1305-glue.c (100%) rename {arch/mips/lib/crypto => lib/crypto/mips}/poly1305-mips.pl (100%) diff --git a/arch/mips/lib/.gitignore b/arch/mips/lib/.gitignore new file mode 100644 index 000000000000..647d7a922e68 --- /dev/null +++ b/arch/mips/lib/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +# This now-removed directory used to contain generated files. +/crypto/ diff --git a/arch/mips/lib/Makefile b/arch/mips/lib/Makefile index 9d75845ef78e..9c024e6d5e54 100644 --- a/arch/mips/lib/Makefile +++ b/arch/mips/lib/Makefile @@ -3,8 +3,6 @@ # Makefile for MIPS-specific library files.. # -obj-y += crypto/ - lib-y += bitops.o csum_partial.o delay.o memcpy.o memset.o \ mips-atomic.o strncpy_user.o \ strnlen_user.o uncached.o diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index fdeb91bf0032..43c44316fbbd 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -196,7 +196,7 @@ if ARM64 source "lib/crypto/arm64/Kconfig" endif if MIPS -source "arch/mips/lib/crypto/Kconfig" +source "lib/crypto/mips/Kconfig" endif if PPC source "arch/powerpc/lib/crypto/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 19e9880c5d5f..f54d2f3edc40 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -109,3 +109,4 @@ libsm3-y := sm3.o obj-$(CONFIG_ARM) += arm/ obj-$(CONFIG_ARM64) += arm64/ +obj-$(CONFIG_MIPS) += mips/ diff --git a/arch/mips/lib/crypto/.gitignore b/lib/crypto/mips/.gitignore similarity index 100% rename from arch/mips/lib/crypto/.gitignore rename to lib/crypto/mips/.gitignore diff --git a/arch/mips/lib/crypto/Kconfig b/lib/crypto/mips/Kconfig similarity index 100% rename from arch/mips/lib/crypto/Kconfig rename to lib/crypto/mips/Kconfig diff --git a/arch/mips/lib/crypto/Makefile b/lib/crypto/mips/Makefile similarity index 100% rename from arch/mips/lib/crypto/Makefile rename to lib/crypto/mips/Makefile diff --git a/arch/mips/lib/crypto/chacha-core.S b/lib/crypto/mips/chacha-core.S similarity index 100% rename from arch/mips/lib/crypto/chacha-core.S rename to lib/crypto/mips/chacha-core.S diff --git a/arch/mips/lib/crypto/chacha-glue.c b/lib/crypto/mips/chacha-glue.c similarity index 100% rename from arch/mips/lib/crypto/chacha-glue.c rename to lib/crypto/mips/chacha-glue.c diff --git a/arch/mips/lib/crypto/poly1305-glue.c b/lib/crypto/mips/poly1305-glue.c similarity index 100% rename from arch/mips/lib/crypto/poly1305-glue.c rename to lib/crypto/mips/poly1305-glue.c diff --git a/arch/mips/lib/crypto/poly1305-mips.pl b/lib/crypto/mips/poly1305-mips.pl similarity index 100% rename from arch/mips/lib/crypto/poly1305-mips.pl rename to lib/crypto/mips/poly1305-mips.pl From 676d45aba8c498dc14ffd27338dba8d4395787e2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:03 -0700 Subject: [PATCH 21/67] lib/crypto: powerpc: Move arch/powerpc/lib/crypto/ into lib/crypto/ Move the contents of arch/powerpc/lib/crypto/ into lib/crypto/powerpc/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Acked-by: Ard Biesheuvel Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/powerpc/lib/Makefile | 2 -- lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 1 + {arch/powerpc/lib/crypto => lib/crypto/powerpc}/Kconfig | 0 {arch/powerpc/lib/crypto => lib/crypto/powerpc}/Makefile | 0 .../powerpc/lib/crypto => lib/crypto/powerpc}/chacha-p10-glue.c | 0 .../powerpc/lib/crypto => lib/crypto/powerpc}/chacha-p10le-8x.S | 0 .../lib/crypto => lib/crypto/powerpc}/poly1305-p10-glue.c | 0 .../lib/crypto => lib/crypto/powerpc}/poly1305-p10le_64.S | 0 .../powerpc/lib/crypto => lib/crypto/powerpc}/sha256-spe-asm.S | 0 {arch/powerpc/lib/crypto => lib/crypto/powerpc}/sha256.c | 0 11 files changed, 2 insertions(+), 3 deletions(-) rename {arch/powerpc/lib/crypto => lib/crypto/powerpc}/Kconfig (100%) rename {arch/powerpc/lib/crypto => lib/crypto/powerpc}/Makefile (100%) rename {arch/powerpc/lib/crypto => lib/crypto/powerpc}/chacha-p10-glue.c (100%) rename {arch/powerpc/lib/crypto => lib/crypto/powerpc}/chacha-p10le-8x.S (100%) rename {arch/powerpc/lib/crypto => lib/crypto/powerpc}/poly1305-p10-glue.c (100%) rename {arch/powerpc/lib/crypto => lib/crypto/powerpc}/poly1305-p10le_64.S (100%) rename {arch/powerpc/lib/crypto => lib/crypto/powerpc}/sha256-spe-asm.S (100%) rename {arch/powerpc/lib/crypto => lib/crypto/powerpc}/sha256.c (100%) diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index 481f968e42c7..27f8a0143860 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -3,8 +3,6 @@ # Makefile for ppc-specific library files.. # -obj-y += crypto/ - CFLAGS_code-patching.o += -fno-stack-protector CFLAGS_feature-fixups.o += -fno-stack-protector diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 43c44316fbbd..f4f9a70dd089 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -199,7 +199,7 @@ if MIPS source "lib/crypto/mips/Kconfig" endif if PPC -source "arch/powerpc/lib/crypto/Kconfig" +source "lib/crypto/powerpc/Kconfig" endif if RISCV source "arch/riscv/lib/crypto/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index f54d2f3edc40..f5f1dcec2f89 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -110,3 +110,4 @@ libsm3-y := sm3.o obj-$(CONFIG_ARM) += arm/ obj-$(CONFIG_ARM64) += arm64/ obj-$(CONFIG_MIPS) += mips/ +obj-$(CONFIG_PPC) += powerpc/ diff --git a/arch/powerpc/lib/crypto/Kconfig b/lib/crypto/powerpc/Kconfig similarity index 100% rename from arch/powerpc/lib/crypto/Kconfig rename to lib/crypto/powerpc/Kconfig diff --git a/arch/powerpc/lib/crypto/Makefile b/lib/crypto/powerpc/Makefile similarity index 100% rename from arch/powerpc/lib/crypto/Makefile rename to lib/crypto/powerpc/Makefile diff --git a/arch/powerpc/lib/crypto/chacha-p10-glue.c b/lib/crypto/powerpc/chacha-p10-glue.c similarity index 100% rename from arch/powerpc/lib/crypto/chacha-p10-glue.c rename to lib/crypto/powerpc/chacha-p10-glue.c diff --git a/arch/powerpc/lib/crypto/chacha-p10le-8x.S b/lib/crypto/powerpc/chacha-p10le-8x.S similarity index 100% rename from arch/powerpc/lib/crypto/chacha-p10le-8x.S rename to lib/crypto/powerpc/chacha-p10le-8x.S diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/lib/crypto/powerpc/poly1305-p10-glue.c similarity index 100% rename from arch/powerpc/lib/crypto/poly1305-p10-glue.c rename to lib/crypto/powerpc/poly1305-p10-glue.c diff --git a/arch/powerpc/lib/crypto/poly1305-p10le_64.S b/lib/crypto/powerpc/poly1305-p10le_64.S similarity index 100% rename from arch/powerpc/lib/crypto/poly1305-p10le_64.S rename to lib/crypto/powerpc/poly1305-p10le_64.S diff --git a/arch/powerpc/lib/crypto/sha256-spe-asm.S b/lib/crypto/powerpc/sha256-spe-asm.S similarity index 100% rename from arch/powerpc/lib/crypto/sha256-spe-asm.S rename to lib/crypto/powerpc/sha256-spe-asm.S diff --git a/arch/powerpc/lib/crypto/sha256.c b/lib/crypto/powerpc/sha256.c similarity index 100% rename from arch/powerpc/lib/crypto/sha256.c rename to lib/crypto/powerpc/sha256.c From daed4fcf04db425e44d377a425424752881a2a68 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:04 -0700 Subject: [PATCH 22/67] lib/crypto: riscv: Move arch/riscv/lib/crypto/ into lib/crypto/ Move the contents of arch/riscv/lib/crypto/ into lib/crypto/riscv/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Acked-by: Ard Biesheuvel Acked-by: Palmer Dabbelt Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/riscv/lib/Makefile | 1 - lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 1 + {arch/riscv/lib/crypto => lib/crypto/riscv}/Kconfig | 0 {arch/riscv/lib/crypto => lib/crypto/riscv}/Makefile | 0 .../riscv/lib/crypto => lib/crypto/riscv}/chacha-riscv64-glue.c | 0 .../riscv/lib/crypto => lib/crypto/riscv}/chacha-riscv64-zvkb.S | 0 .../crypto/riscv}/sha256-riscv64-zvknha_or_zvknhb-zvkb.S | 0 {arch/riscv/lib/crypto => lib/crypto/riscv}/sha256.c | 0 9 files changed, 2 insertions(+), 2 deletions(-) rename {arch/riscv/lib/crypto => lib/crypto/riscv}/Kconfig (100%) rename {arch/riscv/lib/crypto => lib/crypto/riscv}/Makefile (100%) rename {arch/riscv/lib/crypto => lib/crypto/riscv}/chacha-riscv64-glue.c (100%) rename {arch/riscv/lib/crypto => lib/crypto/riscv}/chacha-riscv64-zvkb.S (100%) rename {arch/riscv/lib/crypto => lib/crypto/riscv}/sha256-riscv64-zvknha_or_zvknhb-zvkb.S (100%) rename {arch/riscv/lib/crypto => lib/crypto/riscv}/sha256.c (100%) diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 0baec92d2f55..b1c46153606a 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y += crypto/ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index f4f9a70dd089..a2b58ca2df0c 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -202,7 +202,7 @@ if PPC source "lib/crypto/powerpc/Kconfig" endif if RISCV -source "arch/riscv/lib/crypto/Kconfig" +source "lib/crypto/riscv/Kconfig" endif if S390 source "arch/s390/lib/crypto/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index f5f1dcec2f89..05b7e29ea0e8 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -111,3 +111,4 @@ obj-$(CONFIG_ARM) += arm/ obj-$(CONFIG_ARM64) += arm64/ obj-$(CONFIG_MIPS) += mips/ obj-$(CONFIG_PPC) += powerpc/ +obj-$(CONFIG_RISCV) += riscv/ diff --git a/arch/riscv/lib/crypto/Kconfig b/lib/crypto/riscv/Kconfig similarity index 100% rename from arch/riscv/lib/crypto/Kconfig rename to lib/crypto/riscv/Kconfig diff --git a/arch/riscv/lib/crypto/Makefile b/lib/crypto/riscv/Makefile similarity index 100% rename from arch/riscv/lib/crypto/Makefile rename to lib/crypto/riscv/Makefile diff --git a/arch/riscv/lib/crypto/chacha-riscv64-glue.c b/lib/crypto/riscv/chacha-riscv64-glue.c similarity index 100% rename from arch/riscv/lib/crypto/chacha-riscv64-glue.c rename to lib/crypto/riscv/chacha-riscv64-glue.c diff --git a/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S b/lib/crypto/riscv/chacha-riscv64-zvkb.S similarity index 100% rename from arch/riscv/lib/crypto/chacha-riscv64-zvkb.S rename to lib/crypto/riscv/chacha-riscv64-zvkb.S diff --git a/arch/riscv/lib/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S b/lib/crypto/riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.S similarity index 100% rename from arch/riscv/lib/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S rename to lib/crypto/riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.S diff --git a/arch/riscv/lib/crypto/sha256.c b/lib/crypto/riscv/sha256.c similarity index 100% rename from arch/riscv/lib/crypto/sha256.c rename to lib/crypto/riscv/sha256.c From b8456f7aaf35bc5af247bc8f58412c2cffc331c9 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:05 -0700 Subject: [PATCH 23/67] lib/crypto: s390: Move arch/s390/lib/crypto/ into lib/crypto/ Move the contents of arch/s390/lib/crypto/ into lib/crypto/s390/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Acked-by: Ard Biesheuvel Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-7-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/s390/lib/Makefile | 1 - lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 1 + {arch/s390/lib/crypto => lib/crypto/s390}/Kconfig | 0 {arch/s390/lib/crypto => lib/crypto/s390}/Makefile | 0 {arch/s390/lib/crypto => lib/crypto/s390}/chacha-glue.c | 0 {arch/s390/lib/crypto => lib/crypto/s390}/chacha-s390.S | 0 {arch/s390/lib/crypto => lib/crypto/s390}/chacha-s390.h | 0 {arch/s390/lib/crypto => lib/crypto/s390}/sha256.c | 0 9 files changed, 2 insertions(+), 2 deletions(-) rename {arch/s390/lib/crypto => lib/crypto/s390}/Kconfig (100%) rename {arch/s390/lib/crypto => lib/crypto/s390}/Makefile (100%) rename {arch/s390/lib/crypto => lib/crypto/s390}/chacha-glue.c (100%) rename {arch/s390/lib/crypto => lib/crypto/s390}/chacha-s390.S (100%) rename {arch/s390/lib/crypto => lib/crypto/s390}/chacha-s390.h (100%) rename {arch/s390/lib/crypto => lib/crypto/s390}/sha256.c (100%) diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index cd35cdbfa871..271a1c407121 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,7 +3,6 @@ # Makefile for s390-specific library files.. # -obj-y += crypto/ lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o lib-y += csum-partial.o obj-y += mem.o xor.o diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index a2b58ca2df0c..278b7ef5ec4f 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -205,7 +205,7 @@ if RISCV source "lib/crypto/riscv/Kconfig" endif if S390 -source "arch/s390/lib/crypto/Kconfig" +source "lib/crypto/s390/Kconfig" endif if SPARC source "arch/sparc/lib/crypto/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 05b7e29ea0e8..26f65bb4c8d8 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -112,3 +112,4 @@ obj-$(CONFIG_ARM64) += arm64/ obj-$(CONFIG_MIPS) += mips/ obj-$(CONFIG_PPC) += powerpc/ obj-$(CONFIG_RISCV) += riscv/ +obj-$(CONFIG_S390) += s390/ diff --git a/arch/s390/lib/crypto/Kconfig b/lib/crypto/s390/Kconfig similarity index 100% rename from arch/s390/lib/crypto/Kconfig rename to lib/crypto/s390/Kconfig diff --git a/arch/s390/lib/crypto/Makefile b/lib/crypto/s390/Makefile similarity index 100% rename from arch/s390/lib/crypto/Makefile rename to lib/crypto/s390/Makefile diff --git a/arch/s390/lib/crypto/chacha-glue.c b/lib/crypto/s390/chacha-glue.c similarity index 100% rename from arch/s390/lib/crypto/chacha-glue.c rename to lib/crypto/s390/chacha-glue.c diff --git a/arch/s390/lib/crypto/chacha-s390.S b/lib/crypto/s390/chacha-s390.S similarity index 100% rename from arch/s390/lib/crypto/chacha-s390.S rename to lib/crypto/s390/chacha-s390.S diff --git a/arch/s390/lib/crypto/chacha-s390.h b/lib/crypto/s390/chacha-s390.h similarity index 100% rename from arch/s390/lib/crypto/chacha-s390.h rename to lib/crypto/s390/chacha-s390.h diff --git a/arch/s390/lib/crypto/sha256.c b/lib/crypto/s390/sha256.c similarity index 100% rename from arch/s390/lib/crypto/sha256.c rename to lib/crypto/s390/sha256.c From a32e93e10067d19dec302220a9124a21573e7e7e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:06 -0700 Subject: [PATCH 24/67] lib/crypto: sparc: Move arch/sparc/lib/crypto/ into lib/crypto/ Move the contents of arch/sparc/lib/crypto/ into lib/crypto/sparc/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Acked-by: Ard Biesheuvel Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-8-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/sparc/lib/Makefile | 1 - lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 1 + {arch/sparc/lib/crypto => lib/crypto/sparc}/Kconfig | 0 {arch/sparc/lib/crypto => lib/crypto/sparc}/Makefile | 0 {arch/sparc/lib/crypto => lib/crypto/sparc}/sha256.c | 0 {arch/sparc/lib/crypto => lib/crypto/sparc}/sha256_asm.S | 0 7 files changed, 2 insertions(+), 2 deletions(-) rename {arch/sparc/lib/crypto => lib/crypto/sparc}/Kconfig (100%) rename {arch/sparc/lib/crypto => lib/crypto/sparc}/Makefile (100%) rename {arch/sparc/lib/crypto => lib/crypto/sparc}/sha256.c (100%) rename {arch/sparc/lib/crypto => lib/crypto/sparc}/sha256_asm.S (100%) diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 5cf9781d68b4..ef8860eb3f3d 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -4,7 +4,6 @@ asflags-y := -ansi -DST_DIV0=0x02 -obj-y += crypto/ lib-$(CONFIG_SPARC32) += ashrdi3.o lib-$(CONFIG_SPARC32) += memcpy.o memset.o lib-y += strlen.o diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 278b7ef5ec4f..b98543c7ef23 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -208,7 +208,7 @@ if S390 source "lib/crypto/s390/Kconfig" endif if SPARC -source "arch/sparc/lib/crypto/Kconfig" +source "lib/crypto/sparc/Kconfig" endif if X86 source "arch/x86/lib/crypto/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 26f65bb4c8d8..7c1e7d06cac9 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -113,3 +113,4 @@ obj-$(CONFIG_MIPS) += mips/ obj-$(CONFIG_PPC) += powerpc/ obj-$(CONFIG_RISCV) += riscv/ obj-$(CONFIG_S390) += s390/ +obj-$(CONFIG_SPARC) += sparc/ diff --git a/arch/sparc/lib/crypto/Kconfig b/lib/crypto/sparc/Kconfig similarity index 100% rename from arch/sparc/lib/crypto/Kconfig rename to lib/crypto/sparc/Kconfig diff --git a/arch/sparc/lib/crypto/Makefile b/lib/crypto/sparc/Makefile similarity index 100% rename from arch/sparc/lib/crypto/Makefile rename to lib/crypto/sparc/Makefile diff --git a/arch/sparc/lib/crypto/sha256.c b/lib/crypto/sparc/sha256.c similarity index 100% rename from arch/sparc/lib/crypto/sha256.c rename to lib/crypto/sparc/sha256.c diff --git a/arch/sparc/lib/crypto/sha256_asm.S b/lib/crypto/sparc/sha256_asm.S similarity index 100% rename from arch/sparc/lib/crypto/sha256_asm.S rename to lib/crypto/sparc/sha256_asm.S From 74750aa78de33794aa9ab55de15da04bd41d1ac8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:07 -0700 Subject: [PATCH 25/67] lib/crypto: x86: Move arch/x86/lib/crypto/ into lib/crypto/ Move the contents of arch/x86/lib/crypto/ into lib/crypto/x86/. The new code organization makes a lot more sense for how this code actually works and is developed. In particular, it makes it possible to build each algorithm as a single module, with better inlining and dead code elimination. For a more detailed explanation, see the patchset which did this for the CRC library code: https://lore.kernel.org/r/20250607200454.73587-1-ebiggers@kernel.org/. Also see the patchset which did this for SHA-512: https://lore.kernel.org/linux-crypto/20250616014019.415791-1-ebiggers@kernel.org/ This is just a preparatory commit, which does the move to get the files into their new location but keeps them building the same way as before. Later commits will make the actual improvements to the way the arch-optimized code is integrated for each algorithm. Add a gitignore entry for the removed directory arch/x86/lib/crypto/ so that people don't accidentally commit leftover generated files. Acked-by: Ard Biesheuvel Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-9-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/x86/lib/.gitignore | 4 ++++ arch/x86/lib/Makefile | 2 -- lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 1 + {arch/x86/lib/crypto => lib/crypto/x86}/.gitignore | 0 {arch/x86/lib/crypto => lib/crypto/x86}/Kconfig | 0 {arch/x86/lib/crypto => lib/crypto/x86}/Makefile | 0 {arch/x86/lib/crypto => lib/crypto/x86}/blake2s-core.S | 0 {arch/x86/lib/crypto => lib/crypto/x86}/blake2s-glue.c | 0 {arch/x86/lib/crypto => lib/crypto/x86}/chacha-avx2-x86_64.S | 0 .../lib/crypto => lib/crypto/x86}/chacha-avx512vl-x86_64.S | 0 {arch/x86/lib/crypto => lib/crypto/x86}/chacha-ssse3-x86_64.S | 0 {arch/x86/lib/crypto => lib/crypto/x86}/chacha_glue.c | 0 .../crypto => lib/crypto/x86}/poly1305-x86_64-cryptogams.pl | 0 {arch/x86/lib/crypto => lib/crypto/x86}/poly1305_glue.c | 0 {arch/x86/lib/crypto => lib/crypto/x86}/sha256-avx-asm.S | 0 {arch/x86/lib/crypto => lib/crypto/x86}/sha256-avx2-asm.S | 0 {arch/x86/lib/crypto => lib/crypto/x86}/sha256-ni-asm.S | 0 {arch/x86/lib/crypto => lib/crypto/x86}/sha256-ssse3-asm.S | 0 {arch/x86/lib/crypto => lib/crypto/x86}/sha256.c | 0 20 files changed, 6 insertions(+), 3 deletions(-) rename {arch/x86/lib/crypto => lib/crypto/x86}/.gitignore (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/Kconfig (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/Makefile (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/blake2s-core.S (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/blake2s-glue.c (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/chacha-avx2-x86_64.S (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/chacha-avx512vl-x86_64.S (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/chacha-ssse3-x86_64.S (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/chacha_glue.c (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/poly1305-x86_64-cryptogams.pl (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/poly1305_glue.c (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/sha256-avx-asm.S (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/sha256-avx2-asm.S (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/sha256-ni-asm.S (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/sha256-ssse3-asm.S (100%) rename {arch/x86/lib/crypto => lib/crypto/x86}/sha256.c (100%) diff --git a/arch/x86/lib/.gitignore b/arch/x86/lib/.gitignore index 8ae0f93ecbfd..ec2131c9fd20 100644 --- a/arch/x86/lib/.gitignore +++ b/arch/x86/lib/.gitignore @@ -1,2 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only + +# This now-removed directory used to contain generated files. +/crypto/ + inat-tables.c diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 4fa5c4e1ba8a..7cf8681cba0f 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -3,8 +3,6 @@ # Makefile for x86 specific library files. # -obj-y += crypto/ - # Produces uninteresting flaky coverage. KCOV_INSTRUMENT_delay.o := n diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index b98543c7ef23..2460ddff967f 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -211,7 +211,7 @@ if SPARC source "lib/crypto/sparc/Kconfig" endif if X86 -source "arch/x86/lib/crypto/Kconfig" +source "lib/crypto/x86/Kconfig" endif endif diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 7c1e7d06cac9..5823137fa5a8 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -114,3 +114,4 @@ obj-$(CONFIG_PPC) += powerpc/ obj-$(CONFIG_RISCV) += riscv/ obj-$(CONFIG_S390) += s390/ obj-$(CONFIG_SPARC) += sparc/ +obj-$(CONFIG_X86) += x86/ diff --git a/arch/x86/lib/crypto/.gitignore b/lib/crypto/x86/.gitignore similarity index 100% rename from arch/x86/lib/crypto/.gitignore rename to lib/crypto/x86/.gitignore diff --git a/arch/x86/lib/crypto/Kconfig b/lib/crypto/x86/Kconfig similarity index 100% rename from arch/x86/lib/crypto/Kconfig rename to lib/crypto/x86/Kconfig diff --git a/arch/x86/lib/crypto/Makefile b/lib/crypto/x86/Makefile similarity index 100% rename from arch/x86/lib/crypto/Makefile rename to lib/crypto/x86/Makefile diff --git a/arch/x86/lib/crypto/blake2s-core.S b/lib/crypto/x86/blake2s-core.S similarity index 100% rename from arch/x86/lib/crypto/blake2s-core.S rename to lib/crypto/x86/blake2s-core.S diff --git a/arch/x86/lib/crypto/blake2s-glue.c b/lib/crypto/x86/blake2s-glue.c similarity index 100% rename from arch/x86/lib/crypto/blake2s-glue.c rename to lib/crypto/x86/blake2s-glue.c diff --git a/arch/x86/lib/crypto/chacha-avx2-x86_64.S b/lib/crypto/x86/chacha-avx2-x86_64.S similarity index 100% rename from arch/x86/lib/crypto/chacha-avx2-x86_64.S rename to lib/crypto/x86/chacha-avx2-x86_64.S diff --git a/arch/x86/lib/crypto/chacha-avx512vl-x86_64.S b/lib/crypto/x86/chacha-avx512vl-x86_64.S similarity index 100% rename from arch/x86/lib/crypto/chacha-avx512vl-x86_64.S rename to lib/crypto/x86/chacha-avx512vl-x86_64.S diff --git a/arch/x86/lib/crypto/chacha-ssse3-x86_64.S b/lib/crypto/x86/chacha-ssse3-x86_64.S similarity index 100% rename from arch/x86/lib/crypto/chacha-ssse3-x86_64.S rename to lib/crypto/x86/chacha-ssse3-x86_64.S diff --git a/arch/x86/lib/crypto/chacha_glue.c b/lib/crypto/x86/chacha_glue.c similarity index 100% rename from arch/x86/lib/crypto/chacha_glue.c rename to lib/crypto/x86/chacha_glue.c diff --git a/arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl b/lib/crypto/x86/poly1305-x86_64-cryptogams.pl similarity index 100% rename from arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl rename to lib/crypto/x86/poly1305-x86_64-cryptogams.pl diff --git a/arch/x86/lib/crypto/poly1305_glue.c b/lib/crypto/x86/poly1305_glue.c similarity index 100% rename from arch/x86/lib/crypto/poly1305_glue.c rename to lib/crypto/x86/poly1305_glue.c diff --git a/arch/x86/lib/crypto/sha256-avx-asm.S b/lib/crypto/x86/sha256-avx-asm.S similarity index 100% rename from arch/x86/lib/crypto/sha256-avx-asm.S rename to lib/crypto/x86/sha256-avx-asm.S diff --git a/arch/x86/lib/crypto/sha256-avx2-asm.S b/lib/crypto/x86/sha256-avx2-asm.S similarity index 100% rename from arch/x86/lib/crypto/sha256-avx2-asm.S rename to lib/crypto/x86/sha256-avx2-asm.S diff --git a/arch/x86/lib/crypto/sha256-ni-asm.S b/lib/crypto/x86/sha256-ni-asm.S similarity index 100% rename from arch/x86/lib/crypto/sha256-ni-asm.S rename to lib/crypto/x86/sha256-ni-asm.S diff --git a/arch/x86/lib/crypto/sha256-ssse3-asm.S b/lib/crypto/x86/sha256-ssse3-asm.S similarity index 100% rename from arch/x86/lib/crypto/sha256-ssse3-asm.S rename to lib/crypto/x86/sha256-ssse3-asm.S diff --git a/arch/x86/lib/crypto/sha256.c b/lib/crypto/x86/sha256.c similarity index 100% rename from arch/x86/lib/crypto/sha256.c rename to lib/crypto/x86/sha256.c From a6d2f48b00f71590e1e4f174a88bd149516e6440 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 12:19:08 -0700 Subject: [PATCH 26/67] MAINTAINERS: Drop arch/*/lib/crypto/ pattern Since all files arch/*/lib/crypto/* have been moved into lib/crypto/, remove the arch/*/lib/crypto/ file pattern from MAINTAINERS. Acked-by: Ard Biesheuvel Reviewed-by: Martin K. Petersen Reviewed-by: Sohil Mehta Link: https://lore.kernel.org/r/20250619191908.134235-10-ebiggers@kernel.org Signed-off-by: Eric Biggers --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 4bac4ea21b64..dabd8f3fae9b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6411,7 +6411,6 @@ L: linux-crypto@vger.kernel.org S: Maintained T: git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-next T: git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-fixes -F: arch/*/lib/crypto/ F: lib/crypto/ CRYPTO SPEED TEST COMPARE From 22375adaa0d9fbba9646c8e2b099c6e87c97bfae Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 19 Jun 2025 15:55:35 -0700 Subject: [PATCH 27/67] lib/crypto: mips/chacha: Fix clang build and remove unneeded byteswap The MIPS32r2 ChaCha code has never been buildable with the clang assembler. First, clang doesn't support the 'rotl' pseudo-instruction: error: unknown instruction, did you mean: rol, rotr? Second, clang requires that both operands of the 'wsbh' instruction be explicitly given: error: too few operands for instruction To fix this, align the code with the real instruction set by (1) using the real instruction 'rotr' instead of the nonstandard pseudo- instruction 'rotl', and (2) explicitly giving both operands to 'wsbh'. To make removing the use of 'rotl' a bit easier, also remove the unnecessary special-casing for big endian CPUs at .Lchacha_mips_xor_bytes. The tail handling is actually endian-independent since it processes one byte at a time. On big endian CPUs the old code byte-swapped SAVED_X, then iterated through it in reverse order. But the byteswap and reverse iteration canceled out. Tested with chacha20poly1305-selftest in QEMU using "-M malta" with both little endian and big endian mips32r2 kernels. Fixes: 49aa7c00eddf ("crypto: mips/chacha - import 32r2 ChaCha code from Zinc") Cc: stable@vger.kernel.org Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202505080409.EujEBwA0-lkp@intel.com/ Link: https://lore.kernel.org/r/20250619225535.679301-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/mips/chacha-core.S | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/crypto/mips/chacha-core.S b/lib/crypto/mips/chacha-core.S index 5755f69cfe00..706aeb850fb0 100644 --- a/lib/crypto/mips/chacha-core.S +++ b/lib/crypto/mips/chacha-core.S @@ -55,17 +55,13 @@ #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define MSB 0 #define LSB 3 -#define ROTx rotl -#define ROTR(n) rotr n, 24 #define CPU_TO_LE32(n) \ - wsbh n; \ + wsbh n, n; \ rotr n, 16; #else #define MSB 3 #define LSB 0 -#define ROTx rotr #define CPU_TO_LE32(n) -#define ROTR(n) #endif #define FOR_EACH_WORD(x) \ @@ -192,10 +188,10 @@ CONCAT3(.Lchacha_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \ xor X(W), X(B); \ xor X(Y), X(C); \ xor X(Z), X(D); \ - rotl X(V), S; \ - rotl X(W), S; \ - rotl X(Y), S; \ - rotl X(Z), S; + rotr X(V), 32 - S; \ + rotr X(W), 32 - S; \ + rotr X(Y), 32 - S; \ + rotr X(Z), 32 - S; .text .set reorder @@ -372,21 +368,19 @@ chacha_crypt_arch: /* First byte */ lbu T1, 0(IN) addiu $at, BYTES, 1 - CPU_TO_LE32(SAVED_X) - ROTR(SAVED_X) xor T1, SAVED_X sb T1, 0(OUT) beqz $at, .Lchacha_mips_xor_done /* Second byte */ lbu T1, 1(IN) addiu $at, BYTES, 2 - ROTx SAVED_X, 8 + rotr SAVED_X, 8 xor T1, SAVED_X sb T1, 1(OUT) beqz $at, .Lchacha_mips_xor_done /* Third byte */ lbu T1, 2(IN) - ROTx SAVED_X, 8 + rotr SAVED_X, 8 xor T1, SAVED_X sb T1, 2(OUT) b .Lchacha_mips_xor_done From 1cf5cdf8d2ae3a55cc4890c02feb5da18e423e5d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:32 -0700 Subject: [PATCH 28/67] libceph: Rename hmac_sha256() to ceph_hmac_sha256() Rename hmac_sha256() to ceph_hmac_sha256(), to avoid a naming conflict with the upcoming hmac_sha256() library function. This code will be able to use the HMAC-SHA256 library, but that's left for a later commit. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- net/ceph/messenger_v2.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index bd608ffa0627..5483b4eed94e 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -793,8 +793,8 @@ static int setup_crypto(struct ceph_connection *con, return 0; /* auth_x, secure mode */ } -static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs, - int kvec_cnt, u8 *hmac) +static int ceph_hmac_sha256(struct ceph_connection *con, + const struct kvec *kvecs, int kvec_cnt, u8 *hmac) { SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */ int ret; @@ -1462,8 +1462,8 @@ static int prepare_auth_signature(struct ceph_connection *con) if (!buf) return -ENOMEM; - ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt, - CTRL_BODY(buf)); + ret = ceph_hmac_sha256(con, con->v2.in_sign_kvecs, + con->v2.in_sign_kvec_cnt, CTRL_BODY(buf)); if (ret) return ret; @@ -2460,8 +2460,8 @@ static int process_auth_signature(struct ceph_connection *con, return -EINVAL; } - ret = hmac_sha256(con, con->v2.out_sign_kvecs, - con->v2.out_sign_kvec_cnt, hmac); + ret = ceph_hmac_sha256(con, con->v2.out_sign_kvecs, + con->v2.out_sign_kvec_cnt, hmac); if (ret) return ret; From 85de1929e17e71fafc51473ba1d14101e80180f5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:33 -0700 Subject: [PATCH 29/67] cxl/test: Simplify fw_buf_checksum_show() First, just use sha256() instead of a sequence of sha256_init(), sha256_update(), and sha256_final(). The result is the same. Second, use *phN instead of open-coding the conversion of bytes to hex. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- tools/testing/cxl/test/mem.c | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 0f1d91f57ba3..d533481672b7 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -1828,27 +1828,10 @@ static ssize_t fw_buf_checksum_show(struct device *dev, { struct cxl_mockmem_data *mdata = dev_get_drvdata(dev); u8 hash[SHA256_DIGEST_SIZE]; - unsigned char *hstr, *hptr; - struct sha256_state sctx; - ssize_t written = 0; - int i; - sha256_init(&sctx); - sha256_update(&sctx, mdata->fw, mdata->fw_size); - sha256_final(&sctx, hash); + sha256(mdata->fw, mdata->fw_size, hash); - hstr = kzalloc((SHA256_DIGEST_SIZE * 2) + 1, GFP_KERNEL); - if (!hstr) - return -ENOMEM; - - hptr = hstr; - for (i = 0; i < SHA256_DIGEST_SIZE; i++) - hptr += sprintf(hptr, "%02x", hash[i]); - - written = sysfs_emit(buf, "%s\n", hstr); - - kfree(hstr); - return written; + return sysfs_emit(buf, "%*phN\n", SHA256_DIGEST_SIZE, hash); } static DEVICE_ATTR_RO(fw_buf_checksum); From 3135d5be7c27841526d98150c245304ab312e9f4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:34 -0700 Subject: [PATCH 30/67] lib/crypto: sha256: Reorder some code First, move the declarations of sha224_init/update/final to be just above the corresponding SHA-256 code, matching the order that I used for SHA-384 and SHA-512. In sha2.h, the end result is that SHA-224, SHA-256, SHA-384, and SHA-512 are all in the logical order. Second, move sha224_block_init() and sha256_block_init() to be just below crypto_sha256_state. In later changes, these functions as well as struct crypto_sha256_state will no longer be used by the library functions. They'll remain just for some legacy offload drivers. This gets them into a logical place in the file for that. No code changes other than reordering. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 60 +++++++++++++++++++++---------------------- lib/crypto/sha256.c | 12 ++++----- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 296ce9d468bf..bb181b7996cd 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -71,6 +71,32 @@ struct crypto_sha256_state { u64 count; }; +static inline void sha224_block_init(struct crypto_sha256_state *sctx) +{ + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; +} + +static inline void sha256_block_init(struct crypto_sha256_state *sctx) +{ + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; +} + struct sha256_state { union { struct crypto_sha256_state ctx; @@ -88,18 +114,12 @@ struct sha512_state { u8 buf[SHA512_BLOCK_SIZE]; }; -static inline void sha256_block_init(struct crypto_sha256_state *sctx) +static inline void sha224_init(struct sha256_state *sctx) { - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; + sha224_block_init(&sctx->ctx); } +/* Simply use sha256_update as it is equivalent to sha224_update. */ +void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); static inline void sha256_init(struct sha256_state *sctx) { @@ -109,26 +129,6 @@ void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len); void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]); void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); -static inline void sha224_block_init(struct crypto_sha256_state *sctx) -{ - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; -} - -static inline void sha224_init(struct sha256_state *sctx) -{ - sha224_block_init(&sctx->ctx); -} -/* Simply use sha256_update as it is equivalent to sha224_update. */ -void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); - /* State for the SHA-512 (and SHA-384) compression function */ struct sha512_block_state { u64 h[8]; diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 6bfa4ae8dfb5..573ccecbf48b 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -58,18 +58,18 @@ static inline void __sha256_final(struct sha256_state *sctx, u8 *out, memzero_explicit(sctx, sizeof(*sctx)); } -void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) -{ - __sha256_final(sctx, out, SHA256_DIGEST_SIZE); -} -EXPORT_SYMBOL(sha256_final); - void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]) { __sha256_final(sctx, out, SHA224_DIGEST_SIZE); } EXPORT_SYMBOL(sha224_final); +void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) +{ + __sha256_final(sctx, out, SHA256_DIGEST_SIZE); +} +EXPORT_SYMBOL(sha256_final); + void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) { struct sha256_state sctx; From 9f97707bdb1e479ea15e14e5525164f5f1128e97 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:35 -0700 Subject: [PATCH 31/67] lib/crypto: sha256: Remove sha256_blocks_simd() Instead of having both sha256_blocks_arch() and sha256_blocks_simd(), instead have just sha256_blocks_arch() which uses the most efficient implementation that is available in the calling context. This is simpler, as it reduces the API surface. It's also safer, since sha256_blocks_arch() just works in all contexts, including contexts where the FPU/SIMD/vector registers cannot be used. This doesn't mean that SHA-256 computations *should* be done in such contexts, but rather we should just do the right thing instead of corrupting a random task's registers. Eliminating this footgun and simplifying the code is well worth the very small performance cost of doing the check. Note: in the case of arm and arm64, what used to be sha256_blocks_arch() is renamed back to its original name of sha256_block_data_order(). sha256_blocks_arch() is now used for the higher-level dispatch function. This renaming also required an update to lib/crypto/arm64/sha512.h, since sha2-armv8.pl is shared by both SHA-256 and SHA-512. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/internal/sha2.h | 6 ------ lib/crypto/Kconfig | 8 -------- lib/crypto/arm/Kconfig | 1 - lib/crypto/arm/sha256-armv4.pl | 20 ++++++++++---------- lib/crypto/arm/sha256.c | 14 +++++++------- lib/crypto/arm64/Kconfig | 1 - lib/crypto/arm64/sha2-armv8.pl | 2 +- lib/crypto/arm64/sha256.c | 14 +++++++------- lib/crypto/arm64/sha512.h | 6 +++--- lib/crypto/riscv/Kconfig | 1 - lib/crypto/riscv/sha256.c | 12 +++--------- lib/crypto/x86/Kconfig | 1 - lib/crypto/x86/sha256.c | 12 +++--------- 13 files changed, 34 insertions(+), 64 deletions(-) diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h index 21a27fd5e198..5a25ccc49388 100644 --- a/include/crypto/internal/sha2.h +++ b/include/crypto/internal/sha2.h @@ -3,7 +3,6 @@ #ifndef _CRYPTO_INTERNAL_SHA2_H #define _CRYPTO_INTERNAL_SHA2_H -#include #include #include #include @@ -22,8 +21,6 @@ void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks); static __always_inline void sha256_choose_blocks( u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks, @@ -31,9 +28,6 @@ static __always_inline void sha256_choose_blocks( { if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic) sha256_blocks_generic(state, data, nblocks); - else if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD) && - (force_simd || crypto_simd_usable())) - sha256_blocks_simd(state, data, nblocks); else sha256_blocks_arch(state, data, nblocks); } diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 2460ddff967f..9bd740475a89 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -150,14 +150,6 @@ config CRYPTO_ARCH_HAVE_LIB_SHA256 Declares whether the architecture provides an arch-specific accelerated implementation of the SHA-256 library interface. -config CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD - bool - help - Declares whether the architecture provides an arch-specific - accelerated implementation of the SHA-256 library interface - that is SIMD-based and therefore not usable in hardirq - context. - config CRYPTO_LIB_SHA256_GENERIC tristate default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256 diff --git a/lib/crypto/arm/Kconfig b/lib/crypto/arm/Kconfig index d1ad664f0c67..9f3ff30f4032 100644 --- a/lib/crypto/arm/Kconfig +++ b/lib/crypto/arm/Kconfig @@ -28,4 +28,3 @@ config CRYPTO_SHA256_ARM depends on !CPU_V7M default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD diff --git a/lib/crypto/arm/sha256-armv4.pl b/lib/crypto/arm/sha256-armv4.pl index 8122db7fd599..f3a2b54efd4e 100644 --- a/lib/crypto/arm/sha256-armv4.pl +++ b/lib/crypto/arm/sha256-armv4.pl @@ -204,18 +204,18 @@ K256: .word 0 @ terminator #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) .LOPENSSL_armcap: -.word OPENSSL_armcap_P-sha256_blocks_arch +.word OPENSSL_armcap_P-sha256_block_data_order #endif .align 5 -.global sha256_blocks_arch -.type sha256_blocks_arch,%function -sha256_blocks_arch: -.Lsha256_blocks_arch: +.global sha256_block_data_order +.type sha256_block_data_order,%function +sha256_block_data_order: +.Lsha256_block_data_order: #if __ARM_ARCH__<7 - sub r3,pc,#8 @ sha256_blocks_arch + sub r3,pc,#8 @ sha256_block_data_order #else - adr r3,.Lsha256_blocks_arch + adr r3,.Lsha256_block_data_order #endif #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) ldr r12,.LOPENSSL_armcap @@ -282,7 +282,7 @@ $code.=<<___; moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) #endif -.size sha256_blocks_arch,.-sha256_blocks_arch +.size sha256_block_data_order,.-sha256_block_data_order ___ ###################################################################### # NEON stuff @@ -470,8 +470,8 @@ sha256_block_data_order_neon: stmdb sp!,{r4-r12,lr} sub $H,sp,#16*4+16 - adr $Ktbl,.Lsha256_blocks_arch - sub $Ktbl,$Ktbl,#.Lsha256_blocks_arch-K256 + adr $Ktbl,.Lsha256_block_data_order + sub $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256 bic $H,$H,#15 @ align for 128-bit stores mov $t2,sp mov sp,$H @ alloca diff --git a/lib/crypto/arm/sha256.c b/lib/crypto/arm/sha256.c index 109192e54b0f..2c9cfdaaa069 100644 --- a/lib/crypto/arm/sha256.c +++ b/lib/crypto/arm/sha256.c @@ -6,12 +6,12 @@ */ #include #include +#include #include #include -asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks); -EXPORT_SYMBOL_GPL(sha256_blocks_arch); +asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); asmlinkage void sha256_block_data_order_neon(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], @@ -20,11 +20,11 @@ asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon)) { + static_branch_likely(&have_neon) && crypto_simd_usable()) { kernel_neon_begin(); if (static_branch_likely(&have_ce)) sha256_ce_transform(state, data, nblocks); @@ -32,10 +32,10 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], sha256_block_data_order_neon(state, data, nblocks); kernel_neon_end(); } else { - sha256_blocks_arch(state, data, nblocks); + sha256_block_data_order(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_simd); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { diff --git a/lib/crypto/arm64/Kconfig b/lib/crypto/arm64/Kconfig index 129a7685cb4c..49e57bfdb5b5 100644 --- a/lib/crypto/arm64/Kconfig +++ b/lib/crypto/arm64/Kconfig @@ -17,4 +17,3 @@ config CRYPTO_SHA256_ARM64 tristate default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD diff --git a/lib/crypto/arm64/sha2-armv8.pl b/lib/crypto/arm64/sha2-armv8.pl index 4aebd20c498b..35ec9ae99fe1 100644 --- a/lib/crypto/arm64/sha2-armv8.pl +++ b/lib/crypto/arm64/sha2-armv8.pl @@ -95,7 +95,7 @@ if ($output =~ /512/) { $reg_t="w"; } -$func="sha${BITS}_blocks_arch"; +$func="sha${BITS}_block_data_order"; ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30)); diff --git a/lib/crypto/arm64/sha256.c b/lib/crypto/arm64/sha256.c index bcf7a3adc0c4..fb9bff40357b 100644 --- a/lib/crypto/arm64/sha256.c +++ b/lib/crypto/arm64/sha256.c @@ -6,12 +6,12 @@ */ #include #include +#include #include #include -asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks); -EXPORT_SYMBOL_GPL(sha256_blocks_arch); +asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); asmlinkage void sha256_block_neon(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], @@ -20,11 +20,11 @@ asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon)) { + static_branch_likely(&have_neon) && crypto_simd_usable()) { if (static_branch_likely(&have_ce)) { do { size_t rem; @@ -42,10 +42,10 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], kernel_neon_end(); } } else { - sha256_blocks_arch(state, data, nblocks); + sha256_block_data_order(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_simd); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) { diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h index eae14f9752e0..6abb40b467f2 100644 --- a/lib/crypto/arm64/sha512.h +++ b/lib/crypto/arm64/sha512.h @@ -11,8 +11,8 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); -asmlinkage void sha512_blocks_arch(struct sha512_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha512_block_data_order(struct sha512_block_state *state, + const u8 *data, size_t nblocks); asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, const u8 *data, size_t nblocks); @@ -32,7 +32,7 @@ static void sha512_blocks(struct sha512_block_state *state, nblocks = rem; } while (nblocks); } else { - sha512_blocks_arch(state, data, nblocks); + sha512_block_data_order(state, data, nblocks); } } diff --git a/lib/crypto/riscv/Kconfig b/lib/crypto/riscv/Kconfig index 47c99ea97ce2..c100571feb7e 100644 --- a/lib/crypto/riscv/Kconfig +++ b/lib/crypto/riscv/Kconfig @@ -12,5 +12,4 @@ config CRYPTO_SHA256_RISCV64 depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD select CRYPTO_LIB_SHA256_GENERIC diff --git a/lib/crypto/riscv/sha256.c b/lib/crypto/riscv/sha256.c index 71808397dff4..aa77349d08f3 100644 --- a/lib/crypto/riscv/sha256.c +++ b/lib/crypto/riscv/sha256.c @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -19,10 +20,10 @@ asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { - if (static_branch_likely(&have_extensions)) { + if (static_branch_likely(&have_extensions) && crypto_simd_usable()) { kernel_vector_begin(); sha256_transform_zvknha_or_zvknhb_zvkb(state, data, nblocks); kernel_vector_end(); @@ -30,13 +31,6 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], sha256_blocks_generic(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_simd); - -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks) -{ - sha256_blocks_generic(state, data, nblocks); -} EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) diff --git a/lib/crypto/x86/Kconfig b/lib/crypto/x86/Kconfig index 5e94cdee492c..e344579db3d8 100644 --- a/lib/crypto/x86/Kconfig +++ b/lib/crypto/x86/Kconfig @@ -30,5 +30,4 @@ config CRYPTO_SHA256_X86_64 depends on 64BIT default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD select CRYPTO_LIB_SHA256_GENERIC diff --git a/lib/crypto/x86/sha256.c b/lib/crypto/x86/sha256.c index 80380f8fdcee..baba74d7d26f 100644 --- a/lib/crypto/x86/sha256.c +++ b/lib/crypto/x86/sha256.c @@ -6,6 +6,7 @@ */ #include #include +#include #include #include #include @@ -23,10 +24,10 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { - if (static_branch_likely(&have_sha256_x86)) { + if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { kernel_fpu_begin(); static_call(sha256_blocks_x86)(state, data, nblocks); kernel_fpu_end(); @@ -34,13 +35,6 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], sha256_blocks_generic(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_simd); - -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks) -{ - sha256_blocks_generic(state, data, nblocks); -} EXPORT_SYMBOL_GPL(sha256_blocks_arch); bool sha256_is_arch_optimized(void) From 6fa4b292204b15e0e269a9fd33bc99b5e36b6883 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:36 -0700 Subject: [PATCH 32/67] lib/crypto: sha256: Add sha224() and sha224_update() Add a one-shot SHA-224 computation function sha224(), for consistency with sha256(), sha384(), and sha512() which all already exist. Similarly, add sha224_update(). While for now it's identical to sha256_update(), omitting it makes the API harder to use since users have to "know" which functions are the same between SHA-224 and SHA-256. Also, this is a prerequisite for using different context types for each. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 10 ++++++++-- lib/crypto/sha256.c | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index bb181b7996cd..e31da0743a52 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -114,18 +114,24 @@ struct sha512_state { u8 buf[SHA512_BLOCK_SIZE]; }; +void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len); + static inline void sha224_init(struct sha256_state *sctx) { sha224_block_init(&sctx->ctx); } -/* Simply use sha256_update as it is equivalent to sha224_update. */ +static inline void sha224_update(struct sha256_state *sctx, + const u8 *data, size_t len) +{ + sha256_update(sctx, data, len); +} void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); +void sha224(const u8 *data, size_t len, u8 out[SHA224_DIGEST_SIZE]); static inline void sha256_init(struct sha256_state *sctx) { sha256_block_init(&sctx->ctx); } -void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len); void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]); void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 573ccecbf48b..ccaae7088016 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -70,6 +70,16 @@ void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) } EXPORT_SYMBOL(sha256_final); +void sha224(const u8 *data, size_t len, u8 out[SHA224_DIGEST_SIZE]) +{ + struct sha256_state sctx; + + sha224_init(&sctx); + sha224_update(&sctx, data, len); + sha224_final(&sctx, out); +} +EXPORT_SYMBOL(sha224); + void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) { struct sha256_state sctx; From b86ced882b8e667758afddffd8d6354197842110 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:37 -0700 Subject: [PATCH 33/67] lib/crypto: sha256: Make library API use strongly-typed contexts Currently the SHA-224 and SHA-256 library functions can be mixed arbitrarily, even in ways that are incorrect, for example using sha224_init() and sha256_final(). This is because they operate on the same structure, sha256_state. Introduce stronger typing, as I did for SHA-384 and SHA-512. Also as I did for SHA-384 and SHA-512, use the names *_ctx instead of *_state. The *_ctx names have the following small benefits: - They're shorter. - They avoid an ambiguity with the compression function state. - They're consistent with the well-known OpenSSL API. - Users usually name the variable 'sctx' anyway, which suggests that *_ctx would be the more natural name for the actual struct. Therefore: update the SHA-224 and SHA-256 APIs, implementation, and calling code accordingly. In the new structs, also strongly-type the compression function state. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-7-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/riscv/purgatory/purgatory.c | 8 +-- arch/s390/purgatory/purgatory.c | 2 +- arch/x86/purgatory/purgatory.c | 2 +- crypto/sha256.c | 16 ++--- drivers/char/tpm/tpm2-sessions.c | 12 ++-- include/crypto/sha2.h | 52 +++++++++++---- kernel/kexec_file.c | 10 +-- lib/crypto/sha256.c | 110 ++++++++++++++++++++++--------- 8 files changed, 144 insertions(+), 68 deletions(-) diff --git a/arch/riscv/purgatory/purgatory.c b/arch/riscv/purgatory/purgatory.c index 80596ab5fb62..bbd5cfa4d741 100644 --- a/arch/riscv/purgatory/purgatory.c +++ b/arch/riscv/purgatory/purgatory.c @@ -20,14 +20,14 @@ struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX] __section(".kex static int verify_sha256_digest(void) { struct kexec_sha_region *ptr, *end; - struct sha256_state ss; + struct sha256_ctx sctx; u8 digest[SHA256_DIGEST_SIZE]; - sha256_init(&ss); + sha256_init(&sctx); end = purgatory_sha_regions + ARRAY_SIZE(purgatory_sha_regions); for (ptr = purgatory_sha_regions; ptr < end; ptr++) - sha256_update(&ss, (uint8_t *)(ptr->start), ptr->len); - sha256_final(&ss, digest); + sha256_update(&sctx, (uint8_t *)(ptr->start), ptr->len); + sha256_final(&sctx, digest); if (memcmp(digest, purgatory_sha256_digest, sizeof(digest)) != 0) return 1; return 0; diff --git a/arch/s390/purgatory/purgatory.c b/arch/s390/purgatory/purgatory.c index 030efda05dbe..ecb38102187c 100644 --- a/arch/s390/purgatory/purgatory.c +++ b/arch/s390/purgatory/purgatory.c @@ -16,7 +16,7 @@ int verify_sha256_digest(void) { struct kexec_sha_region *ptr, *end; u8 digest[SHA256_DIGEST_SIZE]; - struct sha256_state sctx; + struct sha256_ctx sctx; sha256_init(&sctx); end = purgatory_sha_regions + ARRAY_SIZE(purgatory_sha_regions); diff --git a/arch/x86/purgatory/purgatory.c b/arch/x86/purgatory/purgatory.c index aea47e793963..655139dd0532 100644 --- a/arch/x86/purgatory/purgatory.c +++ b/arch/x86/purgatory/purgatory.c @@ -25,7 +25,7 @@ static int verify_sha256_digest(void) { struct kexec_sha_region *ptr, *end; u8 digest[SHA256_DIGEST_SIZE]; - struct sha256_state sctx; + struct sha256_ctx sctx; sha256_init(&sctx); end = purgatory_sha_regions + ARRAY_SIZE(purgatory_sha_regions); diff --git a/crypto/sha256.c b/crypto/sha256.c index 4aeb213bab11..15c57fba256b 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -137,24 +137,24 @@ static int crypto_sha224_final_lib(struct shash_desc *desc, u8 *out) static int crypto_sha256_import_lib(struct shash_desc *desc, const void *in) { - struct sha256_state *sctx = shash_desc_ctx(desc); + struct __sha256_ctx *sctx = shash_desc_ctx(desc); const u8 *p = in; memcpy(sctx, p, sizeof(*sctx)); p += sizeof(*sctx); - sctx->count += *p; + sctx->bytecount += *p; return 0; } static int crypto_sha256_export_lib(struct shash_desc *desc, void *out) { - struct sha256_state *sctx0 = shash_desc_ctx(desc); - struct sha256_state sctx = *sctx0; + struct __sha256_ctx *sctx0 = shash_desc_ctx(desc); + struct __sha256_ctx sctx = *sctx0; unsigned int partial; u8 *p = out; - partial = sctx.count % SHA256_BLOCK_SIZE; - sctx.count -= partial; + partial = sctx.bytecount % SHA256_BLOCK_SIZE; + sctx.bytecount -= partial; memcpy(p, &sctx, sizeof(sctx)); p += sizeof(sctx); *p = partial; @@ -201,7 +201,7 @@ static struct shash_alg algs[] = { .update = crypto_sha256_update_lib, .final = crypto_sha256_final_lib, .digest = crypto_sha256_digest_lib, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct sha256_ctx), .statesize = sizeof(struct crypto_sha256_state) + SHA256_BLOCK_SIZE + 1, .import = crypto_sha256_import_lib, @@ -216,7 +216,7 @@ static struct shash_alg algs[] = { .init = crypto_sha224_init, .update = crypto_sha256_update_lib, .final = crypto_sha224_final_lib, - .descsize = sizeof(struct sha256_state), + .descsize = sizeof(struct sha224_ctx), .statesize = sizeof(struct crypto_sha256_state) + SHA256_BLOCK_SIZE + 1, .import = crypto_sha256_import_lib, diff --git a/drivers/char/tpm/tpm2-sessions.c b/drivers/char/tpm/tpm2-sessions.c index 7b5049b3d476..bdb119453dfb 100644 --- a/drivers/char/tpm/tpm2-sessions.c +++ b/drivers/char/tpm/tpm2-sessions.c @@ -390,7 +390,7 @@ static int tpm2_create_primary(struct tpm_chip *chip, u32 hierarchy, * on every operation, so we weld the hmac init and final functions in * here to give it the same usage characteristics as a regular hash */ -static void tpm2_hmac_init(struct sha256_state *sctx, u8 *key, u32 key_len) +static void tpm2_hmac_init(struct sha256_ctx *sctx, u8 *key, u32 key_len) { u8 pad[SHA256_BLOCK_SIZE]; int i; @@ -406,7 +406,7 @@ static void tpm2_hmac_init(struct sha256_state *sctx, u8 *key, u32 key_len) sha256_update(sctx, pad, sizeof(pad)); } -static void tpm2_hmac_final(struct sha256_state *sctx, u8 *key, u32 key_len, +static void tpm2_hmac_final(struct sha256_ctx *sctx, u8 *key, u32 key_len, u8 *out) { u8 pad[SHA256_BLOCK_SIZE]; @@ -440,7 +440,7 @@ static void tpm2_KDFa(u8 *key, u32 key_len, const char *label, u8 *u, const __be32 bits = cpu_to_be32(bytes * 8); while (bytes > 0) { - struct sha256_state sctx; + struct sha256_ctx sctx; __be32 c = cpu_to_be32(counter); tpm2_hmac_init(&sctx, key, key_len); @@ -467,7 +467,7 @@ static void tpm2_KDFa(u8 *key, u32 key_len, const char *label, u8 *u, static void tpm2_KDFe(u8 z[EC_PT_SZ], const char *str, u8 *pt_u, u8 *pt_v, u8 *out) { - struct sha256_state sctx; + struct sha256_ctx sctx; /* * this should be an iterative counter, but because we know * we're only taking 32 bytes for the point using a sha256 @@ -592,7 +592,7 @@ void tpm_buf_fill_hmac_session(struct tpm_chip *chip, struct tpm_buf *buf) u8 *hmac = NULL; u32 attrs; u8 cphash[SHA256_DIGEST_SIZE]; - struct sha256_state sctx; + struct sha256_ctx sctx; if (!auth) return; @@ -750,7 +750,7 @@ int tpm_buf_check_hmac_response(struct tpm_chip *chip, struct tpm_buf *buf, off_t offset_s, offset_p; u8 rphash[SHA256_DIGEST_SIZE]; u32 attrs, cc; - struct sha256_state sctx; + struct sha256_ctx sctx; u16 tag = be16_to_cpu(head->tag); int parm_len, len, i, handles; diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index e31da0743a52..18e1eec841b7 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -114,25 +114,55 @@ struct sha512_state { u8 buf[SHA512_BLOCK_SIZE]; }; -void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len); +/* State for the SHA-256 (and SHA-224) compression function */ +struct sha256_block_state { + u32 h[SHA256_STATE_WORDS]; +}; -static inline void sha224_init(struct sha256_state *sctx) -{ - sha224_block_init(&sctx->ctx); -} -static inline void sha224_update(struct sha256_state *sctx, +/* + * Context structure, shared by SHA-224 and SHA-256. The sha224_ctx and + * sha256_ctx structs wrap this one so that the API has proper typing and + * doesn't allow mixing the SHA-224 and SHA-256 functions arbitrarily. + */ +struct __sha256_ctx { + struct sha256_block_state state; + u64 bytecount; + u8 buf[SHA256_BLOCK_SIZE] __aligned(__alignof__(__be64)); +}; +void __sha256_update(struct __sha256_ctx *ctx, const u8 *data, size_t len); + +/** + * struct sha224_ctx - Context for hashing a message with SHA-224 + * @ctx: private + */ +struct sha224_ctx { + struct __sha256_ctx ctx; +}; + +void sha224_init(struct sha224_ctx *ctx); +static inline void sha224_update(struct sha224_ctx *ctx, const u8 *data, size_t len) { - sha256_update(sctx, data, len); + __sha256_update(&ctx->ctx, data, len); } -void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); +void sha224_final(struct sha224_ctx *ctx, u8 out[SHA224_DIGEST_SIZE]); void sha224(const u8 *data, size_t len, u8 out[SHA224_DIGEST_SIZE]); -static inline void sha256_init(struct sha256_state *sctx) +/** + * struct sha256_ctx - Context for hashing a message with SHA-256 + * @ctx: private + */ +struct sha256_ctx { + struct __sha256_ctx ctx; +}; + +void sha256_init(struct sha256_ctx *ctx); +static inline void sha256_update(struct sha256_ctx *ctx, + const u8 *data, size_t len) { - sha256_block_init(&sctx->ctx); + __sha256_update(&ctx->ctx, data, len); } -void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]); +void sha256_final(struct sha256_ctx *ctx, u8 out[SHA256_DIGEST_SIZE]); void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); /* State for the SHA-512 (and SHA-384) compression function */ diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 69fe76fd9233..b835033c65eb 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -751,7 +751,7 @@ int kexec_add_buffer(struct kexec_buf *kbuf) /* Calculate and store the digest of segments */ static int kexec_calculate_store_digests(struct kimage *image) { - struct sha256_state state; + struct sha256_ctx sctx; int ret = 0, i, j, zero_buf_sz, sha_region_sz; size_t nullsz; u8 digest[SHA256_DIGEST_SIZE]; @@ -770,7 +770,7 @@ static int kexec_calculate_store_digests(struct kimage *image) if (!sha_regions) return -ENOMEM; - sha256_init(&state); + sha256_init(&sctx); for (j = i = 0; i < image->nr_segments; i++) { struct kexec_segment *ksegment; @@ -796,7 +796,7 @@ static int kexec_calculate_store_digests(struct kimage *image) if (check_ima_segment_index(image, i)) continue; - sha256_update(&state, ksegment->kbuf, ksegment->bufsz); + sha256_update(&sctx, ksegment->kbuf, ksegment->bufsz); /* * Assume rest of the buffer is filled with zero and @@ -808,7 +808,7 @@ static int kexec_calculate_store_digests(struct kimage *image) if (bytes > zero_buf_sz) bytes = zero_buf_sz; - sha256_update(&state, zero_buf, bytes); + sha256_update(&sctx, zero_buf, bytes); nullsz -= bytes; } @@ -817,7 +817,7 @@ static int kexec_calculate_store_digests(struct kimage *image) j++; } - sha256_final(&state, digest); + sha256_final(&sctx, digest); ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha_regions", sha_regions, sha_region_sz, 0); diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index ccaae7088016..3e7797a4489d 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -18,6 +18,20 @@ #include #include +static const struct sha256_block_state sha224_iv = { + .h = { + SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, + SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, + }, +}; + +static const struct sha256_block_state sha256_iv = { + .h = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, + }, +}; + /* * If __DISABLE_EXPORTS is defined, then this file is being compiled for a * pre-boot environment. In that case, ignore the kconfig options, pull the @@ -32,61 +46,93 @@ static inline bool sha256_purgatory(void) return __is_defined(__DISABLE_EXPORTS); } -static inline void sha256_blocks(u32 state[SHA256_STATE_WORDS], const u8 *data, - size_t nblocks) +static inline void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { - sha256_choose_blocks(state, data, nblocks, sha256_purgatory(), false); + sha256_choose_blocks(state->h, data, nblocks, sha256_purgatory(), false); } -void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len) +static void __sha256_init(struct __sha256_ctx *ctx, + const struct sha256_block_state *iv, + u64 initial_bytecount) { - size_t partial = sctx->count % SHA256_BLOCK_SIZE; - - sctx->count += len; - BLOCK_HASH_UPDATE_BLOCKS(sha256_blocks, sctx->ctx.state, data, len, - SHA256_BLOCK_SIZE, sctx->buf, partial); -} -EXPORT_SYMBOL(sha256_update); - -static inline void __sha256_final(struct sha256_state *sctx, u8 *out, - size_t digest_size) -{ - size_t partial = sctx->count % SHA256_BLOCK_SIZE; - - sha256_finup(&sctx->ctx, sctx->buf, partial, out, digest_size, - sha256_purgatory(), false); - memzero_explicit(sctx, sizeof(*sctx)); + ctx->state = *iv; + ctx->bytecount = initial_bytecount; } -void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]) +void sha224_init(struct sha224_ctx *ctx) { - __sha256_final(sctx, out, SHA224_DIGEST_SIZE); + __sha256_init(&ctx->ctx, &sha224_iv, 0); +} +EXPORT_SYMBOL_GPL(sha224_init); + +void sha256_init(struct sha256_ctx *ctx) +{ + __sha256_init(&ctx->ctx, &sha256_iv, 0); +} +EXPORT_SYMBOL_GPL(sha256_init); + +void __sha256_update(struct __sha256_ctx *ctx, const u8 *data, size_t len) +{ + size_t partial = ctx->bytecount % SHA256_BLOCK_SIZE; + + ctx->bytecount += len; + BLOCK_HASH_UPDATE_BLOCKS(sha256_blocks, &ctx->state, data, len, + SHA256_BLOCK_SIZE, ctx->buf, partial); +} +EXPORT_SYMBOL(__sha256_update); + +static void __sha256_final(struct __sha256_ctx *ctx, + u8 *out, size_t digest_size) +{ + u64 bitcount = ctx->bytecount << 3; + size_t partial = ctx->bytecount % SHA256_BLOCK_SIZE; + + ctx->buf[partial++] = 0x80; + if (partial > SHA256_BLOCK_SIZE - 8) { + memset(&ctx->buf[partial], 0, SHA256_BLOCK_SIZE - partial); + sha256_blocks(&ctx->state, ctx->buf, 1); + partial = 0; + } + memset(&ctx->buf[partial], 0, SHA256_BLOCK_SIZE - 8 - partial); + *(__be64 *)&ctx->buf[SHA256_BLOCK_SIZE - 8] = cpu_to_be64(bitcount); + sha256_blocks(&ctx->state, ctx->buf, 1); + + for (size_t i = 0; i < digest_size; i += 4) + put_unaligned_be32(ctx->state.h[i / 4], out + i); +} + +void sha224_final(struct sha224_ctx *ctx, u8 out[SHA224_DIGEST_SIZE]) +{ + __sha256_final(&ctx->ctx, out, SHA224_DIGEST_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); } EXPORT_SYMBOL(sha224_final); -void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) +void sha256_final(struct sha256_ctx *ctx, u8 out[SHA256_DIGEST_SIZE]) { - __sha256_final(sctx, out, SHA256_DIGEST_SIZE); + __sha256_final(&ctx->ctx, out, SHA256_DIGEST_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); } EXPORT_SYMBOL(sha256_final); void sha224(const u8 *data, size_t len, u8 out[SHA224_DIGEST_SIZE]) { - struct sha256_state sctx; + struct sha224_ctx ctx; - sha224_init(&sctx); - sha224_update(&sctx, data, len); - sha224_final(&sctx, out); + sha224_init(&ctx); + sha224_update(&ctx, data, len); + sha224_final(&ctx, out); } EXPORT_SYMBOL(sha224); void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) { - struct sha256_state sctx; + struct sha256_ctx ctx; - sha256_init(&sctx); - sha256_update(&sctx, data, len); - sha256_final(&sctx, out); + sha256_init(&ctx); + sha256_update(&ctx, data, len); + sha256_final(&ctx, out); } EXPORT_SYMBOL(sha256); From 4c855d5069ee2edbcf62fafc7f1a5d4cfea1bce1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:38 -0700 Subject: [PATCH 34/67] lib/crypto: sha256: Propagate sha256_block_state type to implementations The previous commit made the SHA-256 compression function state be strongly typed, but it wasn't propagated all the way down to the implementations of it. Do that now. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-8-ebiggers@kernel.org Signed-off-by: Eric Biggers --- .../mips/cavium-octeon/crypto/octeon-sha256.c | 2 +- include/crypto/internal/sha2.h | 8 +++---- lib/crypto/arm/sha256-ce.S | 2 +- lib/crypto/arm/sha256.c | 8 +++---- lib/crypto/arm64/sha256-ce.S | 2 +- lib/crypto/arm64/sha256.c | 8 +++---- lib/crypto/powerpc/sha256.c | 5 ++-- .../sha256-riscv64-zvknha_or_zvknhb-zvkb.S | 2 +- lib/crypto/riscv/sha256.c | 7 +++--- lib/crypto/s390/sha256.c | 2 +- lib/crypto/sha256-generic.c | 24 ++++++++++++++----- lib/crypto/sparc/sha256.c | 4 ++-- lib/crypto/x86/sha256-avx-asm.S | 2 +- lib/crypto/x86/sha256-avx2-asm.S | 2 +- lib/crypto/x86/sha256-ni-asm.S | 2 +- lib/crypto/x86/sha256-ssse3-asm.S | 2 +- lib/crypto/x86/sha256.c | 10 ++++---- 17 files changed, 53 insertions(+), 39 deletions(-) diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index c20038239cb6..f8664818d04e 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -22,7 +22,7 @@ * We pass everything as 64-bit. OCTEON can handle misaligned data. */ -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks) { struct octeon_cop2_state cop2_state; diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h index 5a25ccc49388..f0f455477bbd 100644 --- a/include/crypto/internal/sha2.h +++ b/include/crypto/internal/sha2.h @@ -17,9 +17,9 @@ static inline bool sha256_is_arch_optimized(void) return false; } #endif -void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_generic(struct sha256_block_state *state, const u8 *data, size_t nblocks); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks); static __always_inline void sha256_choose_blocks( @@ -27,9 +27,9 @@ static __always_inline void sha256_choose_blocks( bool force_generic, bool force_simd) { if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic) - sha256_blocks_generic(state, data, nblocks); + sha256_blocks_generic((struct sha256_block_state *)state, data, nblocks); else - sha256_blocks_arch(state, data, nblocks); + sha256_blocks_arch((struct sha256_block_state *)state, data, nblocks); } static __always_inline void sha256_finup( diff --git a/lib/crypto/arm/sha256-ce.S b/lib/crypto/arm/sha256-ce.S index ac2c9b01b22d..7481ac8e6c0d 100644 --- a/lib/crypto/arm/sha256-ce.S +++ b/lib/crypto/arm/sha256-ce.S @@ -67,7 +67,7 @@ .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 /* - * void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], + * void sha256_ce_transform(struct sha256_block_state *state, * const u8 *data, size_t nblocks); */ ENTRY(sha256_ce_transform) diff --git a/lib/crypto/arm/sha256.c b/lib/crypto/arm/sha256.c index 2c9cfdaaa069..7d9082358695 100644 --- a/lib/crypto/arm/sha256.c +++ b/lib/crypto/arm/sha256.c @@ -10,17 +10,17 @@ #include #include -asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage void sha256_block_data_order_neon(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_block_data_order_neon(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_ce_transform(struct sha256_block_state *state, const u8 *data, size_t nblocks); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S index f3e21c6d87d2..b99d9589c421 100644 --- a/lib/crypto/arm64/sha256-ce.S +++ b/lib/crypto/arm64/sha256-ce.S @@ -71,7 +71,7 @@ .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 /* - * size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], + * size_t __sha256_ce_transform(struct sha256_block_state *state, * const u8 *data, size_t nblocks); */ .text diff --git a/lib/crypto/arm64/sha256.c b/lib/crypto/arm64/sha256.c index fb9bff40357b..609ffb815198 100644 --- a/lib/crypto/arm64/sha256.c +++ b/lib/crypto/arm64/sha256.c @@ -10,17 +10,17 @@ #include #include -asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage void sha256_block_neon(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_block_neon(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], +asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state, const u8 *data, size_t nblocks); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && diff --git a/lib/crypto/powerpc/sha256.c b/lib/crypto/powerpc/sha256.c index 6b0f079587eb..55f42403d572 100644 --- a/lib/crypto/powerpc/sha256.c +++ b/lib/crypto/powerpc/sha256.c @@ -26,7 +26,8 @@ */ #define MAX_BYTES 1024 -extern void ppc_spe_sha256_transform(u32 *state, const u8 *src, u32 blocks); +extern void ppc_spe_sha256_transform(struct sha256_block_state *state, + const u8 *src, u32 blocks); static void spe_begin(void) { @@ -42,7 +43,7 @@ static void spe_end(void) preempt_enable(); } -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks) { do { diff --git a/lib/crypto/riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.S b/lib/crypto/riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.S index fad501ad0617..1618d1220a6e 100644 --- a/lib/crypto/riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.S +++ b/lib/crypto/riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.S @@ -106,7 +106,7 @@ sha256_4rounds \last, \k3, W3, W0, W1, W2 .endm -// void sha256_transform_zvknha_or_zvknhb_zvkb(u32 state[SHA256_STATE_WORDS], +// void sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state, // const u8 *data, size_t nblocks); SYM_FUNC_START(sha256_transform_zvknha_or_zvknhb_zvkb) diff --git a/lib/crypto/riscv/sha256.c b/lib/crypto/riscv/sha256.c index aa77349d08f3..a2079aa3ae92 100644 --- a/lib/crypto/riscv/sha256.c +++ b/lib/crypto/riscv/sha256.c @@ -15,12 +15,13 @@ #include #include -asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( - u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); +asmlinkage void +sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state, + const u8 *data, size_t nblocks); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_extensions) && crypto_simd_usable()) { diff --git a/lib/crypto/s390/sha256.c b/lib/crypto/s390/sha256.c index 7dfe120fafab..fb565718f753 100644 --- a/lib/crypto/s390/sha256.c +++ b/lib/crypto/s390/sha256.c @@ -12,7 +12,7 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha256); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_cpacf_sha256)) diff --git a/lib/crypto/sha256-generic.c b/lib/crypto/sha256-generic.c index 2968d95d0403..99f904033c26 100644 --- a/lib/crypto/sha256-generic.c +++ b/lib/crypto/sha256-generic.c @@ -70,7 +70,7 @@ static inline void BLEND_OP(int I, u32 *W) h = t1 + t2; \ } while (0) -static void sha256_block_generic(u32 state[SHA256_STATE_WORDS], +static void sha256_block_generic(struct sha256_block_state *state, const u8 *input, u32 W[64]) { u32 a, b, c, d, e, f, g, h; @@ -101,8 +101,14 @@ static void sha256_block_generic(u32 state[SHA256_STATE_WORDS], } /* load the state into our registers */ - a = state[0]; b = state[1]; c = state[2]; d = state[3]; - e = state[4]; f = state[5]; g = state[6]; h = state[7]; + a = state->h[0]; + b = state->h[1]; + c = state->h[2]; + d = state->h[3]; + e = state->h[4]; + f = state->h[5]; + g = state->h[6]; + h = state->h[7]; /* now iterate */ for (i = 0; i < 64; i += 8) { @@ -116,11 +122,17 @@ static void sha256_block_generic(u32 state[SHA256_STATE_WORDS], SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a); } - state[0] += a; state[1] += b; state[2] += c; state[3] += d; - state[4] += e; state[5] += f; state[6] += g; state[7] += h; + state->h[0] += a; + state->h[1] += b; + state->h[2] += c; + state->h[3] += d; + state->h[4] += e; + state->h[5] += f; + state->h[6] += g; + state->h[7] += h; } -void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_generic(struct sha256_block_state *state, const u8 *data, size_t nblocks) { u32 W[64]; diff --git a/lib/crypto/sparc/sha256.c b/lib/crypto/sparc/sha256.c index 8bdec2db08b3..060664b88a6d 100644 --- a/lib/crypto/sparc/sha256.c +++ b/lib/crypto/sparc/sha256.c @@ -19,10 +19,10 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_opcodes); -asmlinkage void sha256_sparc64_transform(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_sparc64_transform(struct sha256_block_state *state, const u8 *data, size_t nblocks); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_sha256_opcodes)) diff --git a/lib/crypto/x86/sha256-avx-asm.S b/lib/crypto/x86/sha256-avx-asm.S index 0d7b2c3e45d9..73bcff2b548f 100644 --- a/lib/crypto/x86/sha256-avx-asm.S +++ b/lib/crypto/x86/sha256-avx-asm.S @@ -341,7 +341,7 @@ a = TMP_ .endm ######################################################################## -## void sha256_transform_avx(u32 state[SHA256_STATE_WORDS], +## void sha256_transform_avx(struct sha256_block_state *state, ## const u8 *data, size_t nblocks); ######################################################################## .text diff --git a/lib/crypto/x86/sha256-avx2-asm.S b/lib/crypto/x86/sha256-avx2-asm.S index 25d3380321ec..45787570387f 100644 --- a/lib/crypto/x86/sha256-avx2-asm.S +++ b/lib/crypto/x86/sha256-avx2-asm.S @@ -518,7 +518,7 @@ STACK_SIZE = _CTX + _CTX_SIZE .endm ######################################################################## -## void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS], +## void sha256_transform_rorx(struct sha256_block_state *state, ## const u8 *data, size_t nblocks); ######################################################################## .text diff --git a/lib/crypto/x86/sha256-ni-asm.S b/lib/crypto/x86/sha256-ni-asm.S index d3548206cf3d..4af7d22e29e4 100644 --- a/lib/crypto/x86/sha256-ni-asm.S +++ b/lib/crypto/x86/sha256-ni-asm.S @@ -106,7 +106,7 @@ * only processes complete blocks. State initialization, buffering of partial * blocks, and digest finalization is expected to be handled elsewhere. * - * void sha256_ni_transform(u32 state[SHA256_STATE_WORDS], + * void sha256_ni_transform(struct sha256_block_state *state, * const u8 *data, size_t nblocks); */ .text diff --git a/lib/crypto/x86/sha256-ssse3-asm.S b/lib/crypto/x86/sha256-ssse3-asm.S index 7f24a4cdcb25..407b30adcd37 100644 --- a/lib/crypto/x86/sha256-ssse3-asm.S +++ b/lib/crypto/x86/sha256-ssse3-asm.S @@ -348,7 +348,7 @@ a = TMP_ .endm ######################################################################## -## void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS], +## void sha256_transform_ssse3(struct sha256_block_state *state, ## const u8 *data, size_t nblocks); ######################################################################## .text diff --git a/lib/crypto/x86/sha256.c b/lib/crypto/x86/sha256.c index baba74d7d26f..cbb45defbefa 100644 --- a/lib/crypto/x86/sha256.c +++ b/lib/crypto/x86/sha256.c @@ -11,20 +11,20 @@ #include #include -asmlinkage void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_transform_ssse3(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage void sha256_transform_avx(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_transform_avx(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_transform_rorx(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage void sha256_ni_transform(u32 state[SHA256_STATE_WORDS], +asmlinkage void sha256_ni_transform(struct sha256_block_state *state, const u8 *data, size_t nblocks); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(struct sha256_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { From 077833cd600908359391bd22d5350c9106ea238c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:39 -0700 Subject: [PATCH 35/67] lib/crypto: sha256: Add HMAC-SHA224 and HMAC-SHA256 support Since HMAC support is commonly needed and is fairly simple, include it as a first-class citizen of the SHA-256 library. The API supports both incremental and one-shot computation, and either preparing the key ahead of time or just using a raw key. The implementation is much more streamlined than crypto/hmac.c. I've kept it consistent with the HMAC-SHA384 and HMAC-SHA512 code as much as possible. Testing of these functions will be via sha224_kunit and sha256_kunit, added by a later commit. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-9-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 222 ++++++++++++++++++++++++++++++++++++++++++ lib/crypto/sha256.c | 147 +++++++++++++++++++++++++++- 2 files changed, 364 insertions(+), 5 deletions(-) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 18e1eec841b7..2e3fc2cf4aa0 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -131,6 +131,22 @@ struct __sha256_ctx { }; void __sha256_update(struct __sha256_ctx *ctx, const u8 *data, size_t len); +/* + * HMAC key and message context structs, shared by HMAC-SHA224 and HMAC-SHA256. + * The hmac_sha224_* and hmac_sha256_* structs wrap this one so that the API has + * proper typing and doesn't allow mixing the functions arbitrarily. + */ +struct __hmac_sha256_key { + struct sha256_block_state istate; + struct sha256_block_state ostate; +}; +struct __hmac_sha256_ctx { + struct __sha256_ctx sha_ctx; + struct sha256_block_state ostate; +}; +void __hmac_sha256_init(struct __hmac_sha256_ctx *ctx, + const struct __hmac_sha256_key *key); + /** * struct sha224_ctx - Context for hashing a message with SHA-224 * @ctx: private @@ -148,6 +164,109 @@ static inline void sha224_update(struct sha224_ctx *ctx, void sha224_final(struct sha224_ctx *ctx, u8 out[SHA224_DIGEST_SIZE]); void sha224(const u8 *data, size_t len, u8 out[SHA224_DIGEST_SIZE]); +/** + * struct hmac_sha224_key - Prepared key for HMAC-SHA224 + * @key: private + */ +struct hmac_sha224_key { + struct __hmac_sha256_key key; +}; + +/** + * struct hmac_sha224_ctx - Context for computing HMAC-SHA224 of a message + * @ctx: private + */ +struct hmac_sha224_ctx { + struct __hmac_sha256_ctx ctx; +}; + +/** + * hmac_sha224_preparekey() - Prepare a key for HMAC-SHA224 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA224 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha224_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha224_preparekey(struct hmac_sha224_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha224_init() - Initialize an HMAC-SHA224 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha224() instead. + * + * Context: Any context. + */ +static inline void hmac_sha224_init(struct hmac_sha224_ctx *ctx, + const struct hmac_sha224_key *key) +{ + __hmac_sha256_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha224_update() - Update an HMAC-SHA224 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha224_update(struct hmac_sha224_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha256_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha224_final() - Finish computing an HMAC-SHA224 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA224 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha224_final(struct hmac_sha224_ctx *ctx, u8 out[SHA224_DIGEST_SIZE]); + +/** + * hmac_sha224() - Compute HMAC-SHA224 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA224 value + * + * If you're using the key only once, consider using hmac_sha224_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha224(const struct hmac_sha224_key *key, + const u8 *data, size_t data_len, u8 out[SHA224_DIGEST_SIZE]); + +/** + * hmac_sha224_usingrawkey() - Compute HMAC-SHA224 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA224 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA224 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha224_preparekey() followed by multiple calls to hmac_sha224() instead. + * + * Context: Any context. + */ +void hmac_sha224_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA224_DIGEST_SIZE]); + /** * struct sha256_ctx - Context for hashing a message with SHA-256 * @ctx: private @@ -165,6 +284,109 @@ static inline void sha256_update(struct sha256_ctx *ctx, void sha256_final(struct sha256_ctx *ctx, u8 out[SHA256_DIGEST_SIZE]); void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); +/** + * struct hmac_sha256_key - Prepared key for HMAC-SHA256 + * @key: private + */ +struct hmac_sha256_key { + struct __hmac_sha256_key key; +}; + +/** + * struct hmac_sha256_ctx - Context for computing HMAC-SHA256 of a message + * @ctx: private + */ +struct hmac_sha256_ctx { + struct __hmac_sha256_ctx ctx; +}; + +/** + * hmac_sha256_preparekey() - Prepare a key for HMAC-SHA256 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA256 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha256_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha256_preparekey(struct hmac_sha256_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha256_init() - Initialize an HMAC-SHA256 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha256() instead. + * + * Context: Any context. + */ +static inline void hmac_sha256_init(struct hmac_sha256_ctx *ctx, + const struct hmac_sha256_key *key) +{ + __hmac_sha256_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha256_update() - Update an HMAC-SHA256 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha256_update(struct hmac_sha256_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha256_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha256_final() - Finish computing an HMAC-SHA256 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA256 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha256_final(struct hmac_sha256_ctx *ctx, u8 out[SHA256_DIGEST_SIZE]); + +/** + * hmac_sha256() - Compute HMAC-SHA256 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA256 value + * + * If you're using the key only once, consider using hmac_sha256_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha256(const struct hmac_sha256_key *key, + const u8 *data, size_t data_len, u8 out[SHA256_DIGEST_SIZE]); + +/** + * hmac_sha256_usingrawkey() - Compute HMAC-SHA256 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA256 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA256 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha256_preparekey() followed by multiple calls to hmac_sha256() instead. + * + * Context: Any context. + */ +void hmac_sha256_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA256_DIGEST_SIZE]); + /* State for the SHA-512 (and SHA-384) compression function */ struct sha512_block_state { u64 h[8]; diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 3e7797a4489d..12b4b59052c4 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -1,9 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * SHA-256, as specified in - * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf - * - * SHA-256 code by Jean-Luc Cooke . + * SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 library functions * * Copyright (c) Jean-Luc Cooke * Copyright (c) Andrew McDonald @@ -11,12 +8,14 @@ * Copyright (c) 2014 Red Hat Inc. */ +#include #include #include #include #include #include #include +#include static const struct sha256_block_state sha224_iv = { .h = { @@ -136,5 +135,143 @@ void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) } EXPORT_SYMBOL(sha256); -MODULE_DESCRIPTION("SHA-256 Algorithm"); +/* pre-boot environment (as indicated by __DISABLE_EXPORTS) doesn't need HMAC */ +#ifndef __DISABLE_EXPORTS +static void __hmac_sha256_preparekey(struct __hmac_sha256_key *key, + const u8 *raw_key, size_t raw_key_len, + const struct sha256_block_state *iv) +{ + union { + u8 b[SHA256_BLOCK_SIZE]; + unsigned long w[SHA256_BLOCK_SIZE / sizeof(unsigned long)]; + } derived_key = { 0 }; + + if (unlikely(raw_key_len > SHA256_BLOCK_SIZE)) { + if (iv == &sha224_iv) + sha224(raw_key, raw_key_len, derived_key.b); + else + sha256(raw_key, raw_key_len, derived_key.b); + } else { + memcpy(derived_key.b, raw_key, raw_key_len); + } + + for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) + derived_key.w[i] ^= REPEAT_BYTE(HMAC_IPAD_VALUE); + key->istate = *iv; + sha256_blocks(&key->istate, derived_key.b, 1); + + for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) + derived_key.w[i] ^= REPEAT_BYTE(HMAC_OPAD_VALUE ^ + HMAC_IPAD_VALUE); + key->ostate = *iv; + sha256_blocks(&key->ostate, derived_key.b, 1); + + memzero_explicit(&derived_key, sizeof(derived_key)); +} + +void hmac_sha224_preparekey(struct hmac_sha224_key *key, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha256_preparekey(&key->key, raw_key, raw_key_len, &sha224_iv); +} +EXPORT_SYMBOL_GPL(hmac_sha224_preparekey); + +void hmac_sha256_preparekey(struct hmac_sha256_key *key, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha256_preparekey(&key->key, raw_key, raw_key_len, &sha256_iv); +} +EXPORT_SYMBOL_GPL(hmac_sha256_preparekey); + +void __hmac_sha256_init(struct __hmac_sha256_ctx *ctx, + const struct __hmac_sha256_key *key) +{ + __sha256_init(&ctx->sha_ctx, &key->istate, SHA256_BLOCK_SIZE); + ctx->ostate = key->ostate; +} +EXPORT_SYMBOL_GPL(__hmac_sha256_init); + +static void __hmac_sha256_final(struct __hmac_sha256_ctx *ctx, + u8 *out, size_t digest_size) +{ + /* Generate the padded input for the outer hash in ctx->sha_ctx.buf. */ + __sha256_final(&ctx->sha_ctx, ctx->sha_ctx.buf, digest_size); + memset(&ctx->sha_ctx.buf[digest_size], 0, + SHA256_BLOCK_SIZE - digest_size); + ctx->sha_ctx.buf[digest_size] = 0x80; + *(__be32 *)&ctx->sha_ctx.buf[SHA256_BLOCK_SIZE - 4] = + cpu_to_be32(8 * (SHA256_BLOCK_SIZE + digest_size)); + + /* Compute the outer hash, which gives the HMAC value. */ + sha256_blocks(&ctx->ostate, ctx->sha_ctx.buf, 1); + for (size_t i = 0; i < digest_size; i += 4) + put_unaligned_be32(ctx->ostate.h[i / 4], out + i); + + memzero_explicit(ctx, sizeof(*ctx)); +} + +void hmac_sha224_final(struct hmac_sha224_ctx *ctx, + u8 out[SHA224_DIGEST_SIZE]) +{ + __hmac_sha256_final(&ctx->ctx, out, SHA224_DIGEST_SIZE); +} +EXPORT_SYMBOL_GPL(hmac_sha224_final); + +void hmac_sha256_final(struct hmac_sha256_ctx *ctx, + u8 out[SHA256_DIGEST_SIZE]) +{ + __hmac_sha256_final(&ctx->ctx, out, SHA256_DIGEST_SIZE); +} +EXPORT_SYMBOL_GPL(hmac_sha256_final); + +void hmac_sha224(const struct hmac_sha224_key *key, + const u8 *data, size_t data_len, u8 out[SHA224_DIGEST_SIZE]) +{ + struct hmac_sha224_ctx ctx; + + hmac_sha224_init(&ctx, key); + hmac_sha224_update(&ctx, data, data_len); + hmac_sha224_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(hmac_sha224); + +void hmac_sha256(const struct hmac_sha256_key *key, + const u8 *data, size_t data_len, u8 out[SHA256_DIGEST_SIZE]) +{ + struct hmac_sha256_ctx ctx; + + hmac_sha256_init(&ctx, key); + hmac_sha256_update(&ctx, data, data_len); + hmac_sha256_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(hmac_sha256); + +void hmac_sha224_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA224_DIGEST_SIZE]) +{ + struct hmac_sha224_key key; + + hmac_sha224_preparekey(&key, raw_key, raw_key_len); + hmac_sha224(&key, data, data_len, out); + + memzero_explicit(&key, sizeof(key)); +} +EXPORT_SYMBOL_GPL(hmac_sha224_usingrawkey); + +void hmac_sha256_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA256_DIGEST_SIZE]) +{ + struct hmac_sha256_key key; + + hmac_sha256_preparekey(&key, raw_key, raw_key_len); + hmac_sha256(&key, data, data_len, out); + + memzero_explicit(&key, sizeof(key)); +} +EXPORT_SYMBOL_GPL(hmac_sha256_usingrawkey); +#endif /* !__DISABLE_EXPORTS */ + +MODULE_DESCRIPTION("SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 library functions"); MODULE_LICENSE("GPL"); From e0cd3716910385ba1ccbd433c860516cf806fc71 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:40 -0700 Subject: [PATCH 36/67] crypto: sha256 - Wrap library and add HMAC support Like I did for crypto/sha512.c, rework crypto/sha256.c to simply wrap the normal library functions instead of accessing the low-level arch- optimized and generic block functions directly. Also add support for HMAC-SHA224 and HMAC-SHA256, again just wrapping the library functions. Since the replacement crypto_shash algorithms are implemented using the (potentially arch-optimized) library functions, give them driver names ending with "-lib" rather than "-generic". Update crypto/testmgr.c and a couple odd drivers to take this change in driver name into account. Besides the above cases which are accounted for, there are no known cases where the driver names were being depended on. There is potential for confusion for people manually checking /proc/crypto (e.g. https://lore.kernel.org/r/9e33c893-2466-4d4e-afb1-966334e451a2@linux.ibm.com/), but really people just need to get used to the driver name not being meaningful for the software algorithms. Historically, the optimized code was disabled by default, so there was some purpose to checking whether it was enabled or not. However, this is now fixed for all SHA-2 algorithms, and the library code just always does the right thing. E.g. if the CPU supports SHA-256 instructions, they are used. This change does also mean that the generic partial block handling code in crypto/shash.c, which got added in 6.16, no longer gets used. But that's fine; the library has to implement the partial block handling anyway, and it's better to do it in the library since the block size and other properties of the algorithm are all fixed at compile time there, resulting in more streamlined code. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-10-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 4 +- crypto/Makefile | 1 - crypto/sha256.c | 322 ++++++++++++-------------- crypto/testmgr.c | 12 + drivers/crypto/img-hash.c | 4 +- drivers/crypto/starfive/jh7110-hash.c | 8 +- 6 files changed, 166 insertions(+), 185 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index cb40a9b46972..3ea1397214e0 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -992,9 +992,9 @@ config CRYPTO_SHA256 tristate "SHA-224 and SHA-256" select CRYPTO_HASH select CRYPTO_LIB_SHA256 - select CRYPTO_LIB_SHA256_GENERIC help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC 10118-3) + SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC + 10118-3), including HMAC support. This is required for IPsec AH (XFRM_AH) and IPsec ESP (XFRM_ESP). Used by the btrfs filesystem, Ceph, NFS, and SMB. diff --git a/crypto/Makefile b/crypto/Makefile index 271c77462cec..5098fa6d5f39 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -77,7 +77,6 @@ obj-$(CONFIG_CRYPTO_MD5) += md5.o obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) += sha256.o -CFLAGS_sha256.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_SHA512) += sha512.o obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o diff --git a/crypto/sha256.c b/crypto/sha256.c index 15c57fba256b..d81166cbba95 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -1,17 +1,20 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * Crypto API wrapper for the SHA-256 and SHA-224 library functions + * Crypto API support for SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 * * Copyright (c) Jean-Luc Cooke * Copyright (c) Andrew McDonald * Copyright (c) 2002 James Morris * SHA224 Support Copyright 2007 Intel Corporation + * Copyright 2025 Google LLC */ #include -#include +#include #include #include +/* SHA-224 */ + const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = { 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2, @@ -20,6 +23,36 @@ const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = { }; EXPORT_SYMBOL_GPL(sha224_zero_message_hash); +#define SHA224_CTX(desc) ((struct sha224_ctx *)shash_desc_ctx(desc)) + +static int crypto_sha224_init(struct shash_desc *desc) +{ + sha224_init(SHA224_CTX(desc)); + return 0; +} + +static int crypto_sha224_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sha224_update(SHA224_CTX(desc), data, len); + return 0; +} + +static int crypto_sha224_final(struct shash_desc *desc, u8 *out) +{ + sha224_final(SHA224_CTX(desc), out); + return 0; +} + +static int crypto_sha224_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sha224(data, len, out); + return 0; +} + +/* SHA-256 */ + const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, @@ -28,256 +61,193 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { }; EXPORT_SYMBOL_GPL(sha256_zero_message_hash); +#define SHA256_CTX(desc) ((struct sha256_ctx *)shash_desc_ctx(desc)) + static int crypto_sha256_init(struct shash_desc *desc) { - sha256_block_init(shash_desc_ctx(desc)); + sha256_init(SHA256_CTX(desc)); return 0; } -static inline int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len, bool force_generic) +static int crypto_sha256_update(struct shash_desc *desc, + const u8 *data, unsigned int len) { - struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - int remain = len % SHA256_BLOCK_SIZE; - - sctx->count += len - remain; - sha256_choose_blocks(sctx->state, data, len / SHA256_BLOCK_SIZE, - force_generic, !force_generic); - return remain; -} - -static int crypto_sha256_update_generic(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return crypto_sha256_update(desc, data, len, true); -} - -static int crypto_sha256_update_lib(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - sha256_update(shash_desc_ctx(desc), data, len); + sha256_update(SHA256_CTX(desc), data, len); return 0; } -static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int crypto_sha256_final(struct shash_desc *desc, u8 *out) { - return crypto_sha256_update(desc, data, len, false); -} - -static int crypto_sha256_final_lib(struct shash_desc *desc, u8 *out) -{ - sha256_final(shash_desc_ctx(desc), out); + sha256_final(SHA256_CTX(desc), out); return 0; } -static __always_inline int crypto_sha256_finup(struct shash_desc *desc, - const u8 *data, - unsigned int len, u8 *out, - bool force_generic) -{ - struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - unsigned int remain = len; - u8 *buf; - - if (len >= SHA256_BLOCK_SIZE) - remain = crypto_sha256_update(desc, data, len, force_generic); - sctx->count += remain; - buf = memcpy(sctx + 1, data + len - remain, remain); - sha256_finup(sctx, buf, remain, out, - crypto_shash_digestsize(desc->tfm), force_generic, - !force_generic); - return 0; -} - -static int crypto_sha256_finup_generic(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return crypto_sha256_finup(desc, data, len, out, true); -} - -static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return crypto_sha256_finup(desc, data, len, out, false); -} - -static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - crypto_sha256_init(desc); - return crypto_sha256_finup_generic(desc, data, len, out); -} - -static int crypto_sha256_digest_lib(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int crypto_sha256_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) { sha256(data, len, out); return 0; } -static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - crypto_sha256_init(desc); - return crypto_sha256_finup_arch(desc, data, len, out); -} +/* HMAC-SHA224 */ -static int crypto_sha224_init(struct shash_desc *desc) +#define HMAC_SHA224_KEY(tfm) ((struct hmac_sha224_key *)crypto_shash_ctx(tfm)) +#define HMAC_SHA224_CTX(desc) ((struct hmac_sha224_ctx *)shash_desc_ctx(desc)) + +static int crypto_hmac_sha224_setkey(struct crypto_shash *tfm, + const u8 *raw_key, unsigned int keylen) { - sha224_block_init(shash_desc_ctx(desc)); + hmac_sha224_preparekey(HMAC_SHA224_KEY(tfm), raw_key, keylen); return 0; } -static int crypto_sha224_final_lib(struct shash_desc *desc, u8 *out) +static int crypto_hmac_sha224_init(struct shash_desc *desc) { - sha224_final(shash_desc_ctx(desc), out); + hmac_sha224_init(HMAC_SHA224_CTX(desc), HMAC_SHA224_KEY(desc->tfm)); return 0; } -static int crypto_sha256_import_lib(struct shash_desc *desc, const void *in) +static int crypto_hmac_sha224_update(struct shash_desc *desc, + const u8 *data, unsigned int len) { - struct __sha256_ctx *sctx = shash_desc_ctx(desc); - const u8 *p = in; - - memcpy(sctx, p, sizeof(*sctx)); - p += sizeof(*sctx); - sctx->bytecount += *p; + hmac_sha224_update(HMAC_SHA224_CTX(desc), data, len); return 0; } -static int crypto_sha256_export_lib(struct shash_desc *desc, void *out) +static int crypto_hmac_sha224_final(struct shash_desc *desc, u8 *out) { - struct __sha256_ctx *sctx0 = shash_desc_ctx(desc); - struct __sha256_ctx sctx = *sctx0; - unsigned int partial; - u8 *p = out; - - partial = sctx.bytecount % SHA256_BLOCK_SIZE; - sctx.bytecount -= partial; - memcpy(p, &sctx, sizeof(sctx)); - p += sizeof(sctx); - *p = partial; + hmac_sha224_final(HMAC_SHA224_CTX(desc), out); return 0; } +static int crypto_hmac_sha224_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, + u8 *out) +{ + hmac_sha224(HMAC_SHA224_KEY(desc->tfm), data, len, out); + return 0; +} + +/* HMAC-SHA256 */ + +#define HMAC_SHA256_KEY(tfm) ((struct hmac_sha256_key *)crypto_shash_ctx(tfm)) +#define HMAC_SHA256_CTX(desc) ((struct hmac_sha256_ctx *)shash_desc_ctx(desc)) + +static int crypto_hmac_sha256_setkey(struct crypto_shash *tfm, + const u8 *raw_key, unsigned int keylen) +{ + hmac_sha256_preparekey(HMAC_SHA256_KEY(tfm), raw_key, keylen); + return 0; +} + +static int crypto_hmac_sha256_init(struct shash_desc *desc) +{ + hmac_sha256_init(HMAC_SHA256_CTX(desc), HMAC_SHA256_KEY(desc->tfm)); + return 0; +} + +static int crypto_hmac_sha256_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + hmac_sha256_update(HMAC_SHA256_CTX(desc), data, len); + return 0; +} + +static int crypto_hmac_sha256_final(struct shash_desc *desc, u8 *out) +{ + hmac_sha256_final(HMAC_SHA256_CTX(desc), out); + return 0; +} + +static int crypto_hmac_sha256_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, + u8 *out) +{ + hmac_sha256(HMAC_SHA256_KEY(desc->tfm), data, len, out); + return 0; +} + +/* Algorithm definitions */ + static struct shash_alg algs[] = { - { - .base.cra_name = "sha256", - .base.cra_driver_name = "sha256-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA256_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .digestsize = SHA256_DIGEST_SIZE, - .init = crypto_sha256_init, - .update = crypto_sha256_update_generic, - .finup = crypto_sha256_finup_generic, - .digest = crypto_sha256_digest_generic, - .descsize = sizeof(struct crypto_sha256_state), - }, { .base.cra_name = "sha224", - .base.cra_driver_name = "sha224-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, + .base.cra_driver_name = "sha224-lib", + .base.cra_priority = 300, .base.cra_blocksize = SHA224_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .digestsize = SHA224_DIGEST_SIZE, .init = crypto_sha224_init, - .update = crypto_sha256_update_generic, - .finup = crypto_sha256_finup_generic, - .descsize = sizeof(struct crypto_sha256_state), + .update = crypto_sha224_update, + .final = crypto_sha224_final, + .digest = crypto_sha224_digest, + .descsize = sizeof(struct sha224_ctx), }, { .base.cra_name = "sha256", .base.cra_driver_name = "sha256-lib", + .base.cra_priority = 300, .base.cra_blocksize = SHA256_BLOCK_SIZE, .base.cra_module = THIS_MODULE, .digestsize = SHA256_DIGEST_SIZE, .init = crypto_sha256_init, - .update = crypto_sha256_update_lib, - .final = crypto_sha256_final_lib, - .digest = crypto_sha256_digest_lib, + .update = crypto_sha256_update, + .final = crypto_sha256_final, + .digest = crypto_sha256_digest, .descsize = sizeof(struct sha256_ctx), - .statesize = sizeof(struct crypto_sha256_state) + - SHA256_BLOCK_SIZE + 1, - .import = crypto_sha256_import_lib, - .export = crypto_sha256_export_lib, }, { - .base.cra_name = "sha224", - .base.cra_driver_name = "sha224-lib", + .base.cra_name = "hmac(sha224)", + .base.cra_driver_name = "hmac-sha224-lib", + .base.cra_priority = 300, .base.cra_blocksize = SHA224_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct hmac_sha224_key), .base.cra_module = THIS_MODULE, .digestsize = SHA224_DIGEST_SIZE, - .init = crypto_sha224_init, - .update = crypto_sha256_update_lib, - .final = crypto_sha224_final_lib, - .descsize = sizeof(struct sha224_ctx), - .statesize = sizeof(struct crypto_sha256_state) + - SHA256_BLOCK_SIZE + 1, - .import = crypto_sha256_import_lib, - .export = crypto_sha256_export_lib, + .setkey = crypto_hmac_sha224_setkey, + .init = crypto_hmac_sha224_init, + .update = crypto_hmac_sha224_update, + .final = crypto_hmac_sha224_final, + .digest = crypto_hmac_sha224_digest, + .descsize = sizeof(struct hmac_sha224_ctx), }, { - .base.cra_name = "sha256", - .base.cra_driver_name = "sha256-" __stringify(ARCH), + .base.cra_name = "hmac(sha256)", + .base.cra_driver_name = "hmac-sha256-lib", .base.cra_priority = 300, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize = SHA256_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct hmac_sha256_key), .base.cra_module = THIS_MODULE, .digestsize = SHA256_DIGEST_SIZE, - .init = crypto_sha256_init, - .update = crypto_sha256_update_arch, - .finup = crypto_sha256_finup_arch, - .digest = crypto_sha256_digest_arch, - .descsize = sizeof(struct crypto_sha256_state), - }, - { - .base.cra_name = "sha224", - .base.cra_driver_name = "sha224-" __stringify(ARCH), - .base.cra_priority = 300, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SHA224_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .digestsize = SHA224_DIGEST_SIZE, - .init = crypto_sha224_init, - .update = crypto_sha256_update_arch, - .finup = crypto_sha256_finup_arch, - .descsize = sizeof(struct crypto_sha256_state), + .setkey = crypto_hmac_sha256_setkey, + .init = crypto_hmac_sha256_init, + .update = crypto_hmac_sha256_update, + .final = crypto_hmac_sha256_final, + .digest = crypto_hmac_sha256_digest, + .descsize = sizeof(struct hmac_sha256_ctx), }, }; -static unsigned int num_algs; - static int __init crypto_sha256_mod_init(void) { - /* register the arch flavours only if they differ from generic */ - num_algs = ARRAY_SIZE(algs); - BUILD_BUG_ON(ARRAY_SIZE(algs) <= 2); - if (!sha256_is_arch_optimized()) - num_algs -= 2; return crypto_register_shashes(algs, ARRAY_SIZE(algs)); } module_init(crypto_sha256_mod_init); static void __exit crypto_sha256_mod_exit(void) { - crypto_unregister_shashes(algs, num_algs); + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } module_exit(crypto_sha256_mod_exit); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Crypto API wrapper for the SHA-256 and SHA-224 library functions"); +MODULE_DESCRIPTION("Crypto API support for SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256"); -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha256-generic"); -MODULE_ALIAS_CRYPTO("sha256-" __stringify(ARCH)); MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha224-generic"); -MODULE_ALIAS_CRYPTO("sha224-" __stringify(ARCH)); +MODULE_ALIAS_CRYPTO("sha224-lib"); +MODULE_ALIAS_CRYPTO("sha256"); +MODULE_ALIAS_CRYPTO("sha256-lib"); +MODULE_ALIAS_CRYPTO("hmac(sha224)"); +MODULE_ALIAS_CRYPTO("hmac-sha224-lib"); +MODULE_ALIAS_CRYPTO("hmac(sha256)"); +MODULE_ALIAS_CRYPTO("hmac-sha256-lib"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 9d8b11ea4af7..4e95567f7ed1 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4270,18 +4270,21 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "authenc(hmac(sha224),cbc(des))", + .generic_driver = "authenc(hmac-sha224-lib,cbc(des-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha224_des_cbc_tv_temp) } }, { .alg = "authenc(hmac(sha224),cbc(des3_ede))", + .generic_driver = "authenc(hmac-sha224-lib,cbc(des3_ede-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp) } }, { .alg = "authenc(hmac(sha256),cbc(aes))", + .generic_driver = "authenc(hmac-sha256-lib,cbc(aes-generic))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -4289,12 +4292,14 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "authenc(hmac(sha256),cbc(des))", + .generic_driver = "authenc(hmac-sha256-lib,cbc(des-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha256_des_cbc_tv_temp) } }, { .alg = "authenc(hmac(sha256),cbc(des3_ede))", + .generic_driver = "authenc(hmac-sha256-lib,cbc(des3_ede-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp) @@ -4305,6 +4310,7 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "authenc(hmac(sha256),cts(cbc(aes)))", + .generic_driver = "authenc(hmac-sha256-lib,cts(cbc(aes-generic)))", .test = alg_test_aead, .suite = { .aead = __VECS(krb5_test_aes128_cts_hmac_sha256_128) @@ -5015,6 +5021,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)", + .generic_driver = "essiv(authenc(hmac-sha256-lib,cbc(aes-generic)),sha256-lib)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -5022,6 +5029,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "essiv(cbc(aes),sha256)", + .generic_driver = "essiv(cbc(aes-generic),sha256-lib)", .test = alg_test_skcipher, .fips_allowed = 1, .suite = { @@ -5121,6 +5129,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "hmac(sha224)", + .generic_driver = "hmac-sha224-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { @@ -5128,6 +5137,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "hmac(sha256)", + .generic_driver = "hmac-sha256-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { @@ -5459,6 +5469,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "sha224", + .generic_driver = "sha224-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { @@ -5466,6 +5477,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "sha256", + .generic_driver = "sha256-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index e050f5ff5efb..f312eb075fec 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -710,12 +710,12 @@ static int img_hash_cra_sha1_init(struct crypto_tfm *tfm) static int img_hash_cra_sha224_init(struct crypto_tfm *tfm) { - return img_hash_cra_init(tfm, "sha224-generic"); + return img_hash_cra_init(tfm, "sha224-lib"); } static int img_hash_cra_sha256_init(struct crypto_tfm *tfm) { - return img_hash_cra_init(tfm, "sha256-generic"); + return img_hash_cra_init(tfm, "sha256-lib"); } static void img_hash_cra_exit(struct crypto_tfm *tfm) diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index 4abbff07412f..6cfe0238f615 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -493,13 +493,13 @@ static int starfive_hash_setkey(struct crypto_ahash *hash, static int starfive_sha224_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha224-generic", + return starfive_hash_init_tfm(hash, "sha224-lib", STARFIVE_HASH_SHA224, 0); } static int starfive_sha256_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha256-generic", + return starfive_hash_init_tfm(hash, "sha256-lib", STARFIVE_HASH_SHA256, 0); } @@ -523,13 +523,13 @@ static int starfive_sm3_init_tfm(struct crypto_ahash *hash) static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha224-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha224-lib", STARFIVE_HASH_SHA224, 1); } static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha256-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha256-lib", STARFIVE_HASH_SHA256, 1); } From 07f090959bba437dd531bd2e08440ec6bf588878 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:41 -0700 Subject: [PATCH 37/67] crypto: sha256 - Use same state format as legacy drivers Make the export and import functions for the sha224, sha256, hmac(sha224), and hmac(sha256) shash algorithms use the same format as the padlock-sha and nx-sha256 drivers, as required by Herbert. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-11-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/sha256.c | 95 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/crypto/sha256.c b/crypto/sha256.c index d81166cbba95..052806559f06 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -13,6 +13,43 @@ #include #include +/* + * Export and import functions. crypto_shash wants a particular format that + * matches that used by some legacy drivers. It currently is the same as the + * library SHA context, except the value in bytecount must be block-aligned and + * the remainder must be stored in an extra u8 appended to the struct. + */ + +#define SHA256_SHASH_STATE_SIZE 105 +static_assert(offsetof(struct __sha256_ctx, state) == 0); +static_assert(offsetof(struct __sha256_ctx, bytecount) == 32); +static_assert(offsetof(struct __sha256_ctx, buf) == 40); +static_assert(sizeof(struct __sha256_ctx) + 1 == SHA256_SHASH_STATE_SIZE); + +static int __crypto_sha256_export(const struct __sha256_ctx *ctx0, void *out) +{ + struct __sha256_ctx ctx = *ctx0; + unsigned int partial; + u8 *p = out; + + partial = ctx.bytecount % SHA256_BLOCK_SIZE; + ctx.bytecount -= partial; + memcpy(p, &ctx, sizeof(ctx)); + p += sizeof(ctx); + *p = partial; + return 0; +} + +static int __crypto_sha256_import(struct __sha256_ctx *ctx, const void *in) +{ + const u8 *p = in; + + memcpy(ctx, p, sizeof(*ctx)); + p += sizeof(*ctx); + ctx->bytecount += *p; + return 0; +} + /* SHA-224 */ const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE] = { @@ -51,6 +88,16 @@ static int crypto_sha224_digest(struct shash_desc *desc, return 0; } +static int crypto_sha224_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha256_export(&SHA224_CTX(desc)->ctx, out); +} + +static int crypto_sha224_import(struct shash_desc *desc, const void *in) +{ + return __crypto_sha256_import(&SHA224_CTX(desc)->ctx, in); +} + /* SHA-256 */ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = { @@ -89,6 +136,16 @@ static int crypto_sha256_digest(struct shash_desc *desc, return 0; } +static int crypto_sha256_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha256_export(&SHA256_CTX(desc)->ctx, out); +} + +static int crypto_sha256_import(struct shash_desc *desc, const void *in) +{ + return __crypto_sha256_import(&SHA256_CTX(desc)->ctx, in); +} + /* HMAC-SHA224 */ #define HMAC_SHA224_KEY(tfm) ((struct hmac_sha224_key *)crypto_shash_ctx(tfm)) @@ -128,6 +185,19 @@ static int crypto_hmac_sha224_digest(struct shash_desc *desc, return 0; } +static int crypto_hmac_sha224_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha256_export(&HMAC_SHA224_CTX(desc)->ctx.sha_ctx, out); +} + +static int crypto_hmac_sha224_import(struct shash_desc *desc, const void *in) +{ + struct hmac_sha224_ctx *ctx = HMAC_SHA224_CTX(desc); + + ctx->ctx.ostate = HMAC_SHA224_KEY(desc->tfm)->key.ostate; + return __crypto_sha256_import(&ctx->ctx.sha_ctx, in); +} + /* HMAC-SHA256 */ #define HMAC_SHA256_KEY(tfm) ((struct hmac_sha256_key *)crypto_shash_ctx(tfm)) @@ -167,6 +237,19 @@ static int crypto_hmac_sha256_digest(struct shash_desc *desc, return 0; } +static int crypto_hmac_sha256_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha256_export(&HMAC_SHA256_CTX(desc)->ctx.sha_ctx, out); +} + +static int crypto_hmac_sha256_import(struct shash_desc *desc, const void *in) +{ + struct hmac_sha256_ctx *ctx = HMAC_SHA256_CTX(desc); + + ctx->ctx.ostate = HMAC_SHA256_KEY(desc->tfm)->key.ostate; + return __crypto_sha256_import(&ctx->ctx.sha_ctx, in); +} + /* Algorithm definitions */ static struct shash_alg algs[] = { @@ -181,7 +264,10 @@ static struct shash_alg algs[] = { .update = crypto_sha224_update, .final = crypto_sha224_final, .digest = crypto_sha224_digest, + .export = crypto_sha224_export, + .import = crypto_sha224_import, .descsize = sizeof(struct sha224_ctx), + .statesize = SHA256_SHASH_STATE_SIZE, }, { .base.cra_name = "sha256", @@ -194,7 +280,10 @@ static struct shash_alg algs[] = { .update = crypto_sha256_update, .final = crypto_sha256_final, .digest = crypto_sha256_digest, + .export = crypto_sha256_export, + .import = crypto_sha256_import, .descsize = sizeof(struct sha256_ctx), + .statesize = SHA256_SHASH_STATE_SIZE, }, { .base.cra_name = "hmac(sha224)", @@ -209,7 +298,10 @@ static struct shash_alg algs[] = { .update = crypto_hmac_sha224_update, .final = crypto_hmac_sha224_final, .digest = crypto_hmac_sha224_digest, + .export = crypto_hmac_sha224_export, + .import = crypto_hmac_sha224_import, .descsize = sizeof(struct hmac_sha224_ctx), + .statesize = SHA256_SHASH_STATE_SIZE, }, { .base.cra_name = "hmac(sha256)", @@ -224,7 +316,10 @@ static struct shash_alg algs[] = { .update = crypto_hmac_sha256_update, .final = crypto_hmac_sha256_final, .digest = crypto_hmac_sha256_digest, + .export = crypto_hmac_sha256_export, + .import = crypto_hmac_sha256_import, .descsize = sizeof(struct hmac_sha256_ctx), + .statesize = SHA256_SHASH_STATE_SIZE, }, }; From 9f9846a72eec406db9e1eadcad1dd5e90aa0f355 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:42 -0700 Subject: [PATCH 38/67] lib/crypto: sha256: Remove sha256_is_arch_optimized() Remove sha256_is_arch_optimized(), since it is no longer used. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-12-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/mips/cavium-octeon/crypto/octeon-sha256.c | 6 ------ include/crypto/internal/sha2.h | 8 -------- lib/crypto/arm/sha256.c | 7 ------- lib/crypto/arm64/sha256.c | 7 ------- lib/crypto/powerpc/sha256.c | 6 ------ lib/crypto/riscv/sha256.c | 6 ------ lib/crypto/s390/sha256.c | 6 ------ lib/crypto/sparc/sha256.c | 6 ------ lib/crypto/x86/sha256.c | 6 ------ 9 files changed, 58 deletions(-) diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index f8664818d04e..c7c67bdc2bd0 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -61,12 +61,6 @@ void sha256_blocks_arch(struct sha256_block_state *state, } EXPORT_SYMBOL_GPL(sha256_blocks_arch); -bool sha256_is_arch_optimized(void) -{ - return octeon_has_crypto(); -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm (OCTEON)"); MODULE_AUTHOR("Aaro Koskinen "); diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h index f0f455477bbd..7915a3a46bc8 100644 --- a/include/crypto/internal/sha2.h +++ b/include/crypto/internal/sha2.h @@ -9,14 +9,6 @@ #include #include -#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) -bool sha256_is_arch_optimized(void); -#else -static inline bool sha256_is_arch_optimized(void) -{ - return false; -} -#endif void sha256_blocks_generic(struct sha256_block_state *state, const u8 *data, size_t nblocks); void sha256_blocks_arch(struct sha256_block_state *state, diff --git a/lib/crypto/arm/sha256.c b/lib/crypto/arm/sha256.c index 7d9082358695..27181be0aa92 100644 --- a/lib/crypto/arm/sha256.c +++ b/lib/crypto/arm/sha256.c @@ -37,13 +37,6 @@ void sha256_blocks_arch(struct sha256_block_state *state, } EXPORT_SYMBOL_GPL(sha256_blocks_arch); -bool sha256_is_arch_optimized(void) -{ - /* We always can use at least the ARM scalar implementation. */ - return true; -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - static int __init sha256_arm_mod_init(void) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) { diff --git a/lib/crypto/arm64/sha256.c b/lib/crypto/arm64/sha256.c index 609ffb815198..a5a498276708 100644 --- a/lib/crypto/arm64/sha256.c +++ b/lib/crypto/arm64/sha256.c @@ -47,13 +47,6 @@ void sha256_blocks_arch(struct sha256_block_state *state, } EXPORT_SYMBOL_GPL(sha256_blocks_arch); -bool sha256_is_arch_optimized(void) -{ - /* We always can use at least the ARM64 scalar implementation. */ - return true; -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - static int __init sha256_arm64_mod_init(void) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && diff --git a/lib/crypto/powerpc/sha256.c b/lib/crypto/powerpc/sha256.c index 55f42403d572..14b8adcdcfc2 100644 --- a/lib/crypto/powerpc/sha256.c +++ b/lib/crypto/powerpc/sha256.c @@ -61,11 +61,5 @@ void sha256_blocks_arch(struct sha256_block_state *state, } EXPORT_SYMBOL_GPL(sha256_blocks_arch); -bool sha256_is_arch_optimized(void) -{ - return true; -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm, SPE optimized"); diff --git a/lib/crypto/riscv/sha256.c b/lib/crypto/riscv/sha256.c index a2079aa3ae92..01004cb9c6e9 100644 --- a/lib/crypto/riscv/sha256.c +++ b/lib/crypto/riscv/sha256.c @@ -34,12 +34,6 @@ void sha256_blocks_arch(struct sha256_block_state *state, } EXPORT_SYMBOL_GPL(sha256_blocks_arch); -bool sha256_is_arch_optimized(void) -{ - return static_key_enabled(&have_extensions); -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - static int __init riscv64_sha256_mod_init(void) { /* Both zvknha and zvknhb provide the SHA-256 instructions. */ diff --git a/lib/crypto/s390/sha256.c b/lib/crypto/s390/sha256.c index fb565718f753..6ebfd35a5d44 100644 --- a/lib/crypto/s390/sha256.c +++ b/lib/crypto/s390/sha256.c @@ -23,12 +23,6 @@ void sha256_blocks_arch(struct sha256_block_state *state, } EXPORT_SYMBOL_GPL(sha256_blocks_arch); -bool sha256_is_arch_optimized(void) -{ - return static_key_enabled(&have_cpacf_sha256); -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - static int __init sha256_s390_mod_init(void) { if (cpu_have_feature(S390_CPU_FEATURE_MSA) && diff --git a/lib/crypto/sparc/sha256.c b/lib/crypto/sparc/sha256.c index 060664b88a6d..f41c109c1c18 100644 --- a/lib/crypto/sparc/sha256.c +++ b/lib/crypto/sparc/sha256.c @@ -32,12 +32,6 @@ void sha256_blocks_arch(struct sha256_block_state *state, } EXPORT_SYMBOL_GPL(sha256_blocks_arch); -bool sha256_is_arch_optimized(void) -{ - return static_key_enabled(&have_sha256_opcodes); -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - static int __init sha256_sparc64_mod_init(void) { unsigned long cfr; diff --git a/lib/crypto/x86/sha256.c b/lib/crypto/x86/sha256.c index cbb45defbefa..9ee38d2b3d57 100644 --- a/lib/crypto/x86/sha256.c +++ b/lib/crypto/x86/sha256.c @@ -37,12 +37,6 @@ void sha256_blocks_arch(struct sha256_block_state *state, } EXPORT_SYMBOL_GPL(sha256_blocks_arch); -bool sha256_is_arch_optimized(void) -{ - return static_key_enabled(&have_sha256_x86); -} -EXPORT_SYMBOL_GPL(sha256_is_arch_optimized); - static int __init sha256_x86_mod_init(void) { if (boot_cpu_has(X86_FEATURE_SHA_NI)) { From e96cb9507f2d8ba150d417dcd283204564945831 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:43 -0700 Subject: [PATCH 39/67] lib/crypto: sha256: Consolidate into single module Consolidate the CPU-based SHA-256 code into a single module, following what I did with SHA-512: - Each arch now provides a header file lib/crypto/$(SRCARCH)/sha256.h, replacing lib/crypto/$(SRCARCH)/sha256.c. The header defines sha256_blocks() and optionally sha256_mod_init_arch(). It is included by lib/crypto/sha256.c, and thus the code gets built into the single libsha256 module, with proper inlining and dead code elimination. - sha256_blocks_generic() is moved from lib/crypto/sha256-generic.c into lib/crypto/sha256.c. It's now a static function marked with __maybe_unused, so the compiler automatically eliminates it in any cases where it's not used. - Whether arch-optimized SHA-256 is buildable is now controlled centrally by lib/crypto/Kconfig instead of by lib/crypto/$(SRCARCH)/Kconfig. The conditions for enabling it remain the same as before, and it remains enabled by default. - Any additional arch-specific translation units for the optimized SHA-256 code (such as assembly files) are now compiled by lib/crypto/Makefile instead of lib/crypto/$(SRCARCH)/Makefile. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/mips/cavium-octeon/Kconfig | 6 - arch/mips/cavium-octeon/crypto/Makefile | 1 - include/crypto/internal/sha2.h | 52 ------ lib/crypto/Kconfig | 26 ++- lib/crypto/Makefile | 39 ++++- lib/crypto/arm/Kconfig | 6 - lib/crypto/arm/Makefile | 8 +- lib/crypto/arm/{sha256.c => sha256.h} | 27 +-- lib/crypto/arm64/Kconfig | 5 - lib/crypto/arm64/Makefile | 9 +- lib/crypto/arm64/{sha256.c => sha256.h} | 29 +--- .../crypto/mips/sha256.h | 14 +- lib/crypto/powerpc/Kconfig | 6 - lib/crypto/powerpc/Makefile | 3 - lib/crypto/powerpc/{sha256.c => sha256.h} | 13 +- lib/crypto/riscv/Kconfig | 7 - lib/crypto/riscv/Makefile | 3 - lib/crypto/riscv/{sha256.c => sha256.h} | 24 +-- lib/crypto/s390/Kconfig | 6 - lib/crypto/s390/Makefile | 3 - lib/crypto/s390/{sha256.c => sha256.h} | 23 +-- lib/crypto/sha256-generic.c | 150 ----------------- lib/crypto/sha256.c | 156 +++++++++++++++--- lib/crypto/sparc/Kconfig | 8 - lib/crypto/sparc/Makefile | 4 - lib/crypto/sparc/{sha256.c => sha256.h} | 29 +--- lib/crypto/x86/Kconfig | 7 - lib/crypto/x86/Makefile | 3 - lib/crypto/x86/{sha256.c => sha256.h} | 29 +--- 29 files changed, 231 insertions(+), 465 deletions(-) delete mode 100644 include/crypto/internal/sha2.h rename lib/crypto/arm/{sha256.c => sha256.h} (64%) rename lib/crypto/arm64/{sha256.c => sha256.h} (67%) rename arch/mips/cavium-octeon/crypto/octeon-sha256.c => lib/crypto/mips/sha256.h (80%) rename lib/crypto/powerpc/{sha256.c => sha256.h} (80%) rename lib/crypto/riscv/{sha256.c => sha256.h} (63%) rename lib/crypto/s390/{sha256.c => sha256.h} (50%) delete mode 100644 lib/crypto/sha256-generic.c delete mode 100644 lib/crypto/sparc/Kconfig delete mode 100644 lib/crypto/sparc/Makefile rename lib/crypto/sparc/{sha256.c => sha256.h} (62%) rename lib/crypto/x86/{sha256.c => sha256.h} (70%) diff --git a/arch/mips/cavium-octeon/Kconfig b/arch/mips/cavium-octeon/Kconfig index 11f4aa6e80e9..450e979ef5d9 100644 --- a/arch/mips/cavium-octeon/Kconfig +++ b/arch/mips/cavium-octeon/Kconfig @@ -23,12 +23,6 @@ config CAVIUM_OCTEON_CVMSEG_SIZE legally range is from zero to 54 cache blocks (i.e. CVMSEG LM is between zero and 6192 bytes). -config CRYPTO_SHA256_OCTEON - tristate - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_LIB_SHA256_GENERIC - endif # CPU_CAVIUM_OCTEON if CAVIUM_OCTEON_SOC diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-octeon/crypto/Makefile index 168b19ef7ce8..db428e4b30bc 100644 --- a/arch/mips/cavium-octeon/crypto/Makefile +++ b/arch/mips/cavium-octeon/crypto/Makefile @@ -7,4 +7,3 @@ obj-y += octeon-crypto.o obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o obj-$(CONFIG_CRYPTO_SHA1_OCTEON) += octeon-sha1.o -obj-$(CONFIG_CRYPTO_SHA256_OCTEON) += octeon-sha256.o diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h deleted file mode 100644 index 7915a3a46bc8..000000000000 --- a/include/crypto/internal/sha2.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ - -#ifndef _CRYPTO_INTERNAL_SHA2_H -#define _CRYPTO_INTERNAL_SHA2_H - -#include -#include -#include -#include -#include - -void sha256_blocks_generic(struct sha256_block_state *state, - const u8 *data, size_t nblocks); -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks); - -static __always_inline void sha256_choose_blocks( - u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks, - bool force_generic, bool force_simd) -{ - if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic) - sha256_blocks_generic((struct sha256_block_state *)state, data, nblocks); - else - sha256_blocks_arch((struct sha256_block_state *)state, data, nblocks); -} - -static __always_inline void sha256_finup( - struct crypto_sha256_state *sctx, u8 buf[SHA256_BLOCK_SIZE], - size_t len, u8 out[SHA256_DIGEST_SIZE], size_t digest_size, - bool force_generic, bool force_simd) -{ - const size_t bit_offset = SHA256_BLOCK_SIZE - 8; - __be64 *bits = (__be64 *)&buf[bit_offset]; - int i; - - buf[len++] = 0x80; - if (len > bit_offset) { - memset(&buf[len], 0, SHA256_BLOCK_SIZE - len); - sha256_choose_blocks(sctx->state, buf, 1, force_generic, - force_simd); - len = 0; - } - - memset(&buf[len], 0, bit_offset - len); - *bits = cpu_to_be64(sctx->count << 3); - sha256_choose_blocks(sctx->state, buf, 1, force_generic, force_simd); - - for (i = 0; i < digest_size; i += 4) - put_unaligned_be32(sctx->state[i / 4], out + i); -} - -#endif /* _CRYPTO_INTERNAL_SHA2_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 9bd740475a89..3305c6908581 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -144,20 +144,17 @@ config CRYPTO_LIB_SHA256 by either the generic implementation or an arch-specific one, if one is available and enabled. -config CRYPTO_ARCH_HAVE_LIB_SHA256 +config CRYPTO_LIB_SHA256_ARCH bool - help - Declares whether the architecture provides an arch-specific - accelerated implementation of the SHA-256 library interface. - -config CRYPTO_LIB_SHA256_GENERIC - tristate - default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256 - help - This symbol can be selected by arch implementations of the SHA-256 - library interface that require the generic code as a fallback, e.g., - for SIMD implementations. If no arch specific implementation is - enabled, this implementation serves the users of CRYPTO_LIB_SHA256. + depends on CRYPTO_LIB_SHA256 && !UML + default y if ARM && !CPU_V7M + default y if ARM64 + default y if MIPS && CPU_CAVIUM_OCTEON + default y if PPC && SPE + default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + default y if S390 + default y if SPARC64 + default y if X86_64 config CRYPTO_LIB_SHA512 tristate @@ -199,9 +196,6 @@ endif if S390 source "lib/crypto/s390/Kconfig" endif -if SPARC -source "lib/crypto/sparc/Kconfig" -endif if X86 source "lib/crypto/x86/Kconfig" endif diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 5823137fa5a8..a887bf103bf0 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -66,11 +66,39 @@ libpoly1305-generic-y += poly1305-generic.o obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o libsha1-y := sha1.o -obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o -libsha256-y := sha256.o +################################################################################ -obj-$(CONFIG_CRYPTO_LIB_SHA256_GENERIC) += libsha256-generic.o -libsha256-generic-y := sha256-generic.o +obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o +libsha256-y := sha256.o +ifeq ($(CONFIG_CRYPTO_LIB_SHA256_ARCH),y) +CFLAGS_sha256.o += -I$(src)/$(SRCARCH) + +ifeq ($(CONFIG_ARM),y) +libsha256-y += arm/sha256-ce.o arm/sha256-core.o +$(obj)/arm/sha256-core.S: $(src)/arm/sha256-armv4.pl + $(call cmd,perlasm) +clean-files += arm/sha256-core.S +AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y) +endif + +ifeq ($(CONFIG_ARM64),y) +libsha256-y += arm64/sha256-core.o +$(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl + $(call cmd,perlasm_with_args) +clean-files += arm64/sha256-core.S +libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o +endif + +libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o +libsha256-$(CONFIG_RISCV) += riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.o +libsha256-$(CONFIG_SPARC) += sparc/sha256_asm.o +libsha256-$(CONFIG_X86) += x86/sha256-ssse3-asm.o \ + x86/sha256-avx-asm.o \ + x86/sha256-avx2-asm.o \ + x86/sha256-ni-asm.o +endif # CONFIG_CRYPTO_LIB_SHA256_ARCH + +################################################################################ obj-$(CONFIG_CRYPTO_LIB_SHA512) += libsha512.o libsha512-y := sha512.o @@ -100,6 +128,8 @@ libsha512-$(CONFIG_X86) += x86/sha512-ssse3-asm.o \ x86/sha512-avx2-asm.o endif # CONFIG_CRYPTO_LIB_SHA512_ARCH +################################################################################ + obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o @@ -113,5 +143,4 @@ obj-$(CONFIG_MIPS) += mips/ obj-$(CONFIG_PPC) += powerpc/ obj-$(CONFIG_RISCV) += riscv/ obj-$(CONFIG_S390) += s390/ -obj-$(CONFIG_SPARC) += sparc/ obj-$(CONFIG_X86) += x86/ diff --git a/lib/crypto/arm/Kconfig b/lib/crypto/arm/Kconfig index 9f3ff30f4032..e8444fd0aae3 100644 --- a/lib/crypto/arm/Kconfig +++ b/lib/crypto/arm/Kconfig @@ -22,9 +22,3 @@ config CRYPTO_POLY1305_ARM tristate default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 - -config CRYPTO_SHA256_ARM - tristate - depends on !CPU_V7M - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 diff --git a/lib/crypto/arm/Makefile b/lib/crypto/arm/Makefile index 431f77c3ff6f..4c042a4c77ed 100644 --- a/lib/crypto/arm/Makefile +++ b/lib/crypto/arm/Makefile @@ -10,17 +10,13 @@ chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o poly1305-arm-y := poly1305-core.o poly1305-glue.o -obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o -sha256-arm-y := sha256.o sha256-core.o -sha256-arm-$(CONFIG_KERNEL_MODE_NEON) += sha256-ce.o - quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $(<) > $(@) $(obj)/%-core.S: $(src)/%-armv4.pl $(call cmd,perl) -clean-files += poly1305-core.S sha256-core.S +clean-files += poly1305-core.S aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 @@ -28,5 +24,3 @@ aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1 poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5 poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7 AFLAGS_poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y) - -AFLAGS_sha256-core.o += $(aflags-thumb2-y) diff --git a/lib/crypto/arm/sha256.c b/lib/crypto/arm/sha256.h similarity index 64% rename from lib/crypto/arm/sha256.c rename to lib/crypto/arm/sha256.h index 27181be0aa92..da75cbdc51d4 100644 --- a/lib/crypto/arm/sha256.c +++ b/lib/crypto/arm/sha256.h @@ -1,14 +1,11 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SHA-256 optimized for ARM * * Copyright 2025 Google LLC */ #include -#include #include -#include -#include asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); @@ -20,8 +17,8 @@ asmlinkage void sha256_ce_transform(struct sha256_block_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks) +static void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && static_branch_likely(&have_neon) && crypto_simd_usable()) { @@ -35,23 +32,15 @@ void sha256_blocks_arch(struct sha256_block_state *state, sha256_block_data_order(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); -static int __init sha256_arm_mod_init(void) +#ifdef CONFIG_KERNEL_MODE_NEON +#define sha256_mod_init_arch sha256_mod_init_arch +static inline void sha256_mod_init_arch(void) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) { + if (elf_hwcap & HWCAP_NEON) { static_branch_enable(&have_neon); if (elf_hwcap2 & HWCAP2_SHA2) static_branch_enable(&have_ce); } - return 0; } -subsys_initcall(sha256_arm_mod_init); - -static void __exit sha256_arm_mod_exit(void) -{ -} -module_exit(sha256_arm_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-256 optimized for ARM"); +#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/Kconfig b/lib/crypto/arm64/Kconfig index 49e57bfdb5b5..0b903ef524d8 100644 --- a/lib/crypto/arm64/Kconfig +++ b/lib/crypto/arm64/Kconfig @@ -12,8 +12,3 @@ config CRYPTO_POLY1305_NEON depends on KERNEL_MODE_NEON default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 - -config CRYPTO_SHA256_ARM64 - tristate - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 diff --git a/lib/crypto/arm64/Makefile b/lib/crypto/arm64/Makefile index 946c09903711..6207088397a7 100644 --- a/lib/crypto/arm64/Makefile +++ b/lib/crypto/arm64/Makefile @@ -8,17 +8,10 @@ poly1305-neon-y := poly1305-core.o poly1305-glue.o AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch -obj-$(CONFIG_CRYPTO_SHA256_ARM64) += sha256-arm64.o -sha256-arm64-y := sha256.o sha256-core.o -sha256-arm64-$(CONFIG_KERNEL_MODE_NEON) += sha256-ce.o - quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $(<) void $(@) $(obj)/%-core.S: $(src)/%-armv8.pl $(call cmd,perlasm) -$(obj)/sha256-core.S: $(src)/sha2-armv8.pl - $(call cmd,perlasm) - -clean-files += poly1305-core.S sha256-core.S +clean-files += poly1305-core.S diff --git a/lib/crypto/arm64/sha256.c b/lib/crypto/arm64/sha256.h similarity index 67% rename from lib/crypto/arm64/sha256.c rename to lib/crypto/arm64/sha256.h index a5a498276708..a211966c124a 100644 --- a/lib/crypto/arm64/sha256.c +++ b/lib/crypto/arm64/sha256.h @@ -1,14 +1,12 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SHA-256 optimized for ARM64 * * Copyright 2025 Google LLC */ #include -#include #include -#include -#include +#include asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); @@ -20,8 +18,8 @@ asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks) +static void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && static_branch_likely(&have_neon) && crypto_simd_usable()) { @@ -45,24 +43,15 @@ void sha256_blocks_arch(struct sha256_block_state *state, sha256_block_data_order(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); -static int __init sha256_arm64_mod_init(void) +#ifdef CONFIG_KERNEL_MODE_NEON +#define sha256_mod_init_arch sha256_mod_init_arch +static inline void sha256_mod_init_arch(void) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - cpu_have_named_feature(ASIMD)) { + if (cpu_have_named_feature(ASIMD)) { static_branch_enable(&have_neon); if (cpu_have_named_feature(SHA2)) static_branch_enable(&have_ce); } - return 0; } -subsys_initcall(sha256_arm64_mod_init); - -static void __exit sha256_arm64_mod_exit(void) -{ -} -module_exit(sha256_arm64_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-256 optimized for ARM64"); +#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/lib/crypto/mips/sha256.h similarity index 80% rename from arch/mips/cavium-octeon/crypto/octeon-sha256.c rename to lib/crypto/mips/sha256.h index c7c67bdc2bd0..ccccfd131634 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/lib/crypto/mips/sha256.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SHA-256 Secure Hash Algorithm. * @@ -14,16 +14,13 @@ #include #include -#include -#include -#include /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks) +static void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { struct octeon_cop2_state cop2_state; u64 *state64 = (u64 *)state; @@ -59,8 +56,3 @@ void sha256_blocks_arch(struct sha256_block_state *state, state64[3] = read_octeon_64bit_hash_dword(3); octeon_crypto_disable(&cop2_state, flags); } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm (OCTEON)"); -MODULE_AUTHOR("Aaro Koskinen "); diff --git a/lib/crypto/powerpc/Kconfig b/lib/crypto/powerpc/Kconfig index 3f9e1bbd9905..2eaeb7665a6a 100644 --- a/lib/crypto/powerpc/Kconfig +++ b/lib/crypto/powerpc/Kconfig @@ -14,9 +14,3 @@ config CRYPTO_POLY1305_P10 default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC - -config CRYPTO_SHA256_PPC_SPE - tristate - depends on SPE - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 diff --git a/lib/crypto/powerpc/Makefile b/lib/crypto/powerpc/Makefile index 27f231f8e334..5709ae14258a 100644 --- a/lib/crypto/powerpc/Makefile +++ b/lib/crypto/powerpc/Makefile @@ -5,6 +5,3 @@ chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o - -obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o -sha256-ppc-spe-y := sha256.o sha256-spe-asm.o diff --git a/lib/crypto/powerpc/sha256.c b/lib/crypto/powerpc/sha256.h similarity index 80% rename from lib/crypto/powerpc/sha256.c rename to lib/crypto/powerpc/sha256.h index 14b8adcdcfc2..50d355441c7e 100644 --- a/lib/crypto/powerpc/sha256.c +++ b/lib/crypto/powerpc/sha256.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SHA-256 Secure Hash Algorithm, SPE optimized * @@ -9,9 +9,6 @@ */ #include -#include -#include -#include #include /* @@ -43,8 +40,8 @@ static void spe_end(void) preempt_enable(); } -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks) +static void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { do { /* cut input data into smaller blocks */ @@ -59,7 +56,3 @@ void sha256_blocks_arch(struct sha256_block_state *state, nblocks -= unit; } while (nblocks); } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm, SPE optimized"); diff --git a/lib/crypto/riscv/Kconfig b/lib/crypto/riscv/Kconfig index c100571feb7e..bc7a43f33eb3 100644 --- a/lib/crypto/riscv/Kconfig +++ b/lib/crypto/riscv/Kconfig @@ -6,10 +6,3 @@ config CRYPTO_CHACHA_RISCV64 default CRYPTO_LIB_CHACHA select CRYPTO_ARCH_HAVE_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC - -config CRYPTO_SHA256_RISCV64 - tristate - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_LIB_SHA256_GENERIC diff --git a/lib/crypto/riscv/Makefile b/lib/crypto/riscv/Makefile index b7cb877a2c07..e27b78f317fc 100644 --- a/lib/crypto/riscv/Makefile +++ b/lib/crypto/riscv/Makefile @@ -2,6 +2,3 @@ obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o - -obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o -sha256-riscv64-y := sha256.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o diff --git a/lib/crypto/riscv/sha256.c b/lib/crypto/riscv/sha256.h similarity index 63% rename from lib/crypto/riscv/sha256.c rename to lib/crypto/riscv/sha256.h index 01004cb9c6e9..c0f79c18f119 100644 --- a/lib/crypto/riscv/sha256.c +++ b/lib/crypto/riscv/sha256.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SHA-256 (RISC-V accelerated) * @@ -10,10 +10,7 @@ */ #include -#include #include -#include -#include asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state, @@ -21,8 +18,8 @@ sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks) +static void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { if (static_branch_likely(&have_extensions) && crypto_simd_usable()) { kernel_vector_begin(); @@ -32,9 +29,9 @@ void sha256_blocks_arch(struct sha256_block_state *state, sha256_blocks_generic(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); -static int __init riscv64_sha256_mod_init(void) +#define sha256_mod_init_arch sha256_mod_init_arch +static inline void sha256_mod_init_arch(void) { /* Both zvknha and zvknhb provide the SHA-256 instructions. */ if ((riscv_isa_extension_available(NULL, ZVKNHA) || @@ -42,15 +39,4 @@ static int __init riscv64_sha256_mod_init(void) riscv_isa_extension_available(NULL, ZVKB) && riscv_vector_vlen() >= 128) static_branch_enable(&have_extensions); - return 0; } -subsys_initcall(riscv64_sha256_mod_init); - -static void __exit riscv64_sha256_mod_exit(void) -{ -} -module_exit(riscv64_sha256_mod_exit); - -MODULE_DESCRIPTION("SHA-256 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); diff --git a/lib/crypto/s390/Kconfig b/lib/crypto/s390/Kconfig index e3f855ef4393..069b355fe51a 100644 --- a/lib/crypto/s390/Kconfig +++ b/lib/crypto/s390/Kconfig @@ -5,9 +5,3 @@ config CRYPTO_CHACHA_S390 default CRYPTO_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA - -config CRYPTO_SHA256_S390 - tristate - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_LIB_SHA256_GENERIC diff --git a/lib/crypto/s390/Makefile b/lib/crypto/s390/Makefile index 5df30f1e7930..06c2cf77178e 100644 --- a/lib/crypto/s390/Makefile +++ b/lib/crypto/s390/Makefile @@ -2,6 +2,3 @@ obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o chacha_s390-y := chacha-glue.o chacha-s390.o - -obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256-s390.o -sha256-s390-y := sha256.o diff --git a/lib/crypto/s390/sha256.c b/lib/crypto/s390/sha256.h similarity index 50% rename from lib/crypto/s390/sha256.c rename to lib/crypto/s390/sha256.h index 6ebfd35a5d44..70a81cbc06b2 100644 --- a/lib/crypto/s390/sha256.c +++ b/lib/crypto/s390/sha256.h @@ -1,19 +1,16 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SHA-256 optimized using the CP Assist for Cryptographic Functions (CPACF) * * Copyright 2025 Google LLC */ #include -#include #include -#include -#include static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha256); -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks) +static void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { if (static_branch_likely(&have_cpacf_sha256)) cpacf_kimd(CPACF_KIMD_SHA_256, state, data, @@ -21,21 +18,11 @@ void sha256_blocks_arch(struct sha256_block_state *state, else sha256_blocks_generic(state, data, nblocks); } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); -static int __init sha256_s390_mod_init(void) +#define sha256_mod_init_arch sha256_mod_init_arch +static inline void sha256_mod_init_arch(void) { if (cpu_have_feature(S390_CPU_FEATURE_MSA) && cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_256)) static_branch_enable(&have_cpacf_sha256); - return 0; } -subsys_initcall(sha256_s390_mod_init); - -static void __exit sha256_s390_mod_exit(void) -{ -} -module_exit(sha256_s390_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-256 using the CP Assist for Cryptographic Functions (CPACF)"); diff --git a/lib/crypto/sha256-generic.c b/lib/crypto/sha256-generic.c deleted file mode 100644 index 99f904033c26..000000000000 --- a/lib/crypto/sha256-generic.c +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SHA-256, as specified in - * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf - * - * SHA-256 code by Jean-Luc Cooke . - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2002 James Morris - * Copyright (c) 2014 Red Hat Inc. - */ - -#include -#include -#include -#include -#include -#include - -static const u32 SHA256_K[] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, - 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, - 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, - 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, - 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, -}; - -static inline u32 Ch(u32 x, u32 y, u32 z) -{ - return z ^ (x & (y ^ z)); -} - -static inline u32 Maj(u32 x, u32 y, u32 z) -{ - return (x & y) | (z & (x | y)); -} - -#define e0(x) (ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22)) -#define e1(x) (ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25)) -#define s0(x) (ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3)) -#define s1(x) (ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10)) - -static inline void LOAD_OP(int I, u32 *W, const u8 *input) -{ - W[I] = get_unaligned_be32((__u32 *)input + I); -} - -static inline void BLEND_OP(int I, u32 *W) -{ - W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; -} - -#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) do { \ - u32 t1, t2; \ - t1 = h + e1(e) + Ch(e, f, g) + SHA256_K[i] + W[i]; \ - t2 = e0(a) + Maj(a, b, c); \ - d += t1; \ - h = t1 + t2; \ -} while (0) - -static void sha256_block_generic(struct sha256_block_state *state, - const u8 *input, u32 W[64]) -{ - u32 a, b, c, d, e, f, g, h; - int i; - - /* load the input */ - for (i = 0; i < 16; i += 8) { - LOAD_OP(i + 0, W, input); - LOAD_OP(i + 1, W, input); - LOAD_OP(i + 2, W, input); - LOAD_OP(i + 3, W, input); - LOAD_OP(i + 4, W, input); - LOAD_OP(i + 5, W, input); - LOAD_OP(i + 6, W, input); - LOAD_OP(i + 7, W, input); - } - - /* now blend */ - for (i = 16; i < 64; i += 8) { - BLEND_OP(i + 0, W); - BLEND_OP(i + 1, W); - BLEND_OP(i + 2, W); - BLEND_OP(i + 3, W); - BLEND_OP(i + 4, W); - BLEND_OP(i + 5, W); - BLEND_OP(i + 6, W); - BLEND_OP(i + 7, W); - } - - /* load the state into our registers */ - a = state->h[0]; - b = state->h[1]; - c = state->h[2]; - d = state->h[3]; - e = state->h[4]; - f = state->h[5]; - g = state->h[6]; - h = state->h[7]; - - /* now iterate */ - for (i = 0; i < 64; i += 8) { - SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h); - SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g); - SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f); - SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e); - SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d); - SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c); - SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b); - SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a); - } - - state->h[0] += a; - state->h[1] += b; - state->h[2] += c; - state->h[3] += d; - state->h[4] += e; - state->h[5] += f; - state->h[6] += g; - state->h[7] += h; -} - -void sha256_blocks_generic(struct sha256_block_state *state, - const u8 *data, size_t nblocks) -{ - u32 W[64]; - - do { - sha256_block_generic(state, data, W); - data += SHA256_BLOCK_SIZE; - } while (--nblocks); - - memzero_explicit(W, sizeof(W)); -} -EXPORT_SYMBOL_GPL(sha256_blocks_generic); - -MODULE_DESCRIPTION("SHA-256 Algorithm (generic implementation)"); -MODULE_LICENSE("GPL"); diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 12b4b59052c4..68936d5cd774 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -6,15 +6,17 @@ * Copyright (c) Andrew McDonald * Copyright (c) 2002 James Morris * Copyright (c) 2014 Red Hat Inc. + * Copyright 2025 Google LLC */ #include #include -#include +#include #include #include #include #include +#include #include static const struct sha256_block_state sha224_iv = { @@ -31,26 +33,128 @@ static const struct sha256_block_state sha256_iv = { }, }; -/* - * If __DISABLE_EXPORTS is defined, then this file is being compiled for a - * pre-boot environment. In that case, ignore the kconfig options, pull the - * generic code into the same translation unit, and use that only. - */ -#ifdef __DISABLE_EXPORTS -#include "sha256-generic.c" +static const u32 sha256_K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) +#define Maj(x, y, z) (((x) & (y)) | ((z) & ((x) | (y)))) +#define e0(x) (ror32((x), 2) ^ ror32((x), 13) ^ ror32((x), 22)) +#define e1(x) (ror32((x), 6) ^ ror32((x), 11) ^ ror32((x), 25)) +#define s0(x) (ror32((x), 7) ^ ror32((x), 18) ^ ((x) >> 3)) +#define s1(x) (ror32((x), 17) ^ ror32((x), 19) ^ ((x) >> 10)) + +static inline void LOAD_OP(int I, u32 *W, const u8 *input) +{ + W[I] = get_unaligned_be32((__u32 *)input + I); +} + +static inline void BLEND_OP(int I, u32 *W) +{ + W[I] = s1(W[I - 2]) + W[I - 7] + s0(W[I - 15]) + W[I - 16]; +} + +#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) \ + do { \ + u32 t1, t2; \ + t1 = h + e1(e) + Ch(e, f, g) + sha256_K[i] + W[i]; \ + t2 = e0(a) + Maj(a, b, c); \ + d += t1; \ + h = t1 + t2; \ + } while (0) + +static void sha256_block_generic(struct sha256_block_state *state, + const u8 *input, u32 W[64]) +{ + u32 a, b, c, d, e, f, g, h; + int i; + + /* load the input */ + for (i = 0; i < 16; i += 8) { + LOAD_OP(i + 0, W, input); + LOAD_OP(i + 1, W, input); + LOAD_OP(i + 2, W, input); + LOAD_OP(i + 3, W, input); + LOAD_OP(i + 4, W, input); + LOAD_OP(i + 5, W, input); + LOAD_OP(i + 6, W, input); + LOAD_OP(i + 7, W, input); + } + + /* now blend */ + for (i = 16; i < 64; i += 8) { + BLEND_OP(i + 0, W); + BLEND_OP(i + 1, W); + BLEND_OP(i + 2, W); + BLEND_OP(i + 3, W); + BLEND_OP(i + 4, W); + BLEND_OP(i + 5, W); + BLEND_OP(i + 6, W); + BLEND_OP(i + 7, W); + } + + /* load the state into our registers */ + a = state->h[0]; + b = state->h[1]; + c = state->h[2]; + d = state->h[3]; + e = state->h[4]; + f = state->h[5]; + g = state->h[6]; + h = state->h[7]; + + /* now iterate */ + for (i = 0; i < 64; i += 8) { + SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h); + SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g); + SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f); + SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e); + SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d); + SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c); + SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b); + SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a); + } + + state->h[0] += a; + state->h[1] += b; + state->h[2] += c; + state->h[3] += d; + state->h[4] += e; + state->h[5] += f; + state->h[6] += g; + state->h[7] += h; +} + +static void __maybe_unused +sha256_blocks_generic(struct sha256_block_state *state, + const u8 *data, size_t nblocks) +{ + u32 W[64]; + + do { + sha256_block_generic(state, data, W); + data += SHA256_BLOCK_SIZE; + } while (--nblocks); + + memzero_explicit(W, sizeof(W)); +} + +#if defined(CONFIG_CRYPTO_LIB_SHA256_ARCH) && !defined(__DISABLE_EXPORTS) +#include "sha256.h" /* $(SRCARCH)/sha256.h */ +#else +#define sha256_blocks sha256_blocks_generic #endif -static inline bool sha256_purgatory(void) -{ - return __is_defined(__DISABLE_EXPORTS); -} - -static inline void sha256_blocks(struct sha256_block_state *state, - const u8 *data, size_t nblocks) -{ - sha256_choose_blocks(state->h, data, nblocks, sha256_purgatory(), false); -} - static void __sha256_init(struct __sha256_ctx *ctx, const struct sha256_block_state *iv, u64 initial_bytecount) @@ -273,5 +377,19 @@ void hmac_sha256_usingrawkey(const u8 *raw_key, size_t raw_key_len, EXPORT_SYMBOL_GPL(hmac_sha256_usingrawkey); #endif /* !__DISABLE_EXPORTS */ +#ifdef sha256_mod_init_arch +static int __init sha256_mod_init(void) +{ + sha256_mod_init_arch(); + return 0; +} +subsys_initcall(sha256_mod_init); + +static void __exit sha256_mod_exit(void) +{ +} +module_exit(sha256_mod_exit); +#endif + MODULE_DESCRIPTION("SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 library functions"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/sparc/Kconfig b/lib/crypto/sparc/Kconfig deleted file mode 100644 index e5c3e4d3dba6..000000000000 --- a/lib/crypto/sparc/Kconfig +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - -config CRYPTO_SHA256_SPARC64 - tristate - depends on SPARC64 - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_LIB_SHA256_GENERIC diff --git a/lib/crypto/sparc/Makefile b/lib/crypto/sparc/Makefile deleted file mode 100644 index 75ee244ad6f7..000000000000 --- a/lib/crypto/sparc/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - -obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o -sha256-sparc64-y := sha256.o sha256_asm.o diff --git a/lib/crypto/sparc/sha256.c b/lib/crypto/sparc/sha256.h similarity index 62% rename from lib/crypto/sparc/sha256.c rename to lib/crypto/sparc/sha256.h index f41c109c1c18..1d10108eb195 100644 --- a/lib/crypto/sparc/sha256.c +++ b/lib/crypto/sparc/sha256.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only +/* SPDX-License-Identifier: GPL-2.0-only */ /* * SHA-256 accelerated using the sparc64 sha256 opcodes * @@ -8,51 +8,36 @@ * SHA224 Support Copyright 2007 Intel Corporation */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include -#include -#include -#include static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_opcodes); asmlinkage void sha256_sparc64_transform(struct sha256_block_state *state, const u8 *data, size_t nblocks); -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks) +static void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { if (static_branch_likely(&have_sha256_opcodes)) sha256_sparc64_transform(state, data, nblocks); else sha256_blocks_generic(state, data, nblocks); } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); -static int __init sha256_sparc64_mod_init(void) +#define sha256_mod_init_arch sha256_mod_init_arch +static inline void sha256_mod_init_arch(void) { unsigned long cfr; if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return 0; + return; __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); if (!(cfr & CFR_SHA256)) - return 0; + return; static_branch_enable(&have_sha256_opcodes); pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); - return 0; } -subsys_initcall(sha256_sparc64_mod_init); - -static void __exit sha256_sparc64_mod_exit(void) -{ -} -module_exit(sha256_sparc64_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-256 accelerated using the sparc64 sha256 opcodes"); diff --git a/lib/crypto/x86/Kconfig b/lib/crypto/x86/Kconfig index e344579db3d8..546fe2afe0b5 100644 --- a/lib/crypto/x86/Kconfig +++ b/lib/crypto/x86/Kconfig @@ -24,10 +24,3 @@ config CRYPTO_POLY1305_X86_64 depends on 64BIT default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 - -config CRYPTO_SHA256_X86_64 - tristate - depends on 64BIT - default CRYPTO_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_LIB_SHA256_GENERIC diff --git a/lib/crypto/x86/Makefile b/lib/crypto/x86/Makefile index abceca3d31c0..c2ff8c5f1046 100644 --- a/lib/crypto/x86/Makefile +++ b/lib/crypto/x86/Makefile @@ -10,9 +10,6 @@ obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o targets += poly1305-x86_64-cryptogams.S -obj-$(CONFIG_CRYPTO_SHA256_X86_64) += sha256-x86_64.o -sha256-x86_64-y := sha256.o sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256-ni-asm.o - quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $< > $@ diff --git a/lib/crypto/x86/sha256.c b/lib/crypto/x86/sha256.h similarity index 70% rename from lib/crypto/x86/sha256.c rename to lib/crypto/x86/sha256.h index 9ee38d2b3d57..3b5456c222ba 100644 --- a/lib/crypto/x86/sha256.c +++ b/lib/crypto/x86/sha256.h @@ -1,14 +1,11 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * SHA-256 optimized for x86_64 * * Copyright 2025 Google LLC */ #include -#include #include -#include -#include #include asmlinkage void sha256_transform_ssse3(struct sha256_block_state *state, @@ -24,8 +21,8 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); -void sha256_blocks_arch(struct sha256_block_state *state, - const u8 *data, size_t nblocks) +static void sha256_blocks(struct sha256_block_state *state, + const u8 *data, size_t nblocks) { if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { kernel_fpu_begin(); @@ -35,14 +32,14 @@ void sha256_blocks_arch(struct sha256_block_state *state, sha256_blocks_generic(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_arch); -static int __init sha256_x86_mod_init(void) +#define sha256_mod_init_arch sha256_mod_init_arch +static inline void sha256_mod_init_arch(void) { if (boot_cpu_has(X86_FEATURE_SHA_NI)) { static_call_update(sha256_blocks_x86, sha256_ni_transform); - } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | - XFEATURE_MASK_YMM, NULL) && + } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, + NULL) && boot_cpu_has(X86_FEATURE_AVX)) { if (boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2)) @@ -52,17 +49,7 @@ static int __init sha256_x86_mod_init(void) static_call_update(sha256_blocks_x86, sha256_transform_avx); } else if (!boot_cpu_has(X86_FEATURE_SSSE3)) { - return 0; + return; } static_branch_enable(&have_sha256_x86); - return 0; } -subsys_initcall(sha256_x86_mod_init); - -static void __exit sha256_x86_mod_exit(void) -{ -} -module_exit(sha256_x86_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-256 optimized for x86_64"); From 773d2b99bb76131f65f98828221142ccd2cec856 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:44 -0700 Subject: [PATCH 40/67] lib/crypto: sha256: Sync sha256_update() with sha512_update() The BLOCK_HASH_UPDATE_BLOCKS macro is difficult to read. For now, let's just write the update explicitly in the straightforward way, mirroring sha512_update(). It's possible that we'll bring back a macro for this later, but it needs to be properly justified and hopefully a bit more readable. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-14-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/sha256.c | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 68936d5cd774..808438d4f427 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -10,7 +10,6 @@ */ #include -#include #include #include #include @@ -180,8 +179,31 @@ void __sha256_update(struct __sha256_ctx *ctx, const u8 *data, size_t len) size_t partial = ctx->bytecount % SHA256_BLOCK_SIZE; ctx->bytecount += len; - BLOCK_HASH_UPDATE_BLOCKS(sha256_blocks, &ctx->state, data, len, - SHA256_BLOCK_SIZE, ctx->buf, partial); + + if (partial + len >= SHA256_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l = SHA256_BLOCK_SIZE - partial; + + memcpy(&ctx->buf[partial], data, l); + data += l; + len -= l; + + sha256_blocks(&ctx->state, ctx->buf, 1); + } + + nblocks = len / SHA256_BLOCK_SIZE; + len %= SHA256_BLOCK_SIZE; + + if (nblocks) { + sha256_blocks(&ctx->state, data, nblocks); + data += nblocks * SHA256_BLOCK_SIZE; + } + partial = 0; + } + if (len) + memcpy(&ctx->buf[partial], data, len); } EXPORT_SYMBOL(__sha256_update); From b34c9803aabd85189ffacc0d3cdb9ce4515c2b4d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 09:06:45 -0700 Subject: [PATCH 41/67] lib/crypto: sha256: Document the SHA-224 and SHA-256 API Add kerneldoc comments, consistent with the kerneldoc comments of the SHA-384 and SHA-512 API. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630160645.3198-15-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 76 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 2e3fc2cf4aa0..e0a08f6addd0 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -155,13 +155,51 @@ struct sha224_ctx { struct __sha256_ctx ctx; }; +/** + * sha224_init() - Initialize a SHA-224 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha224() instead. + * + * Context: Any context. + */ void sha224_init(struct sha224_ctx *ctx); + +/** + * sha224_update() - Update a SHA-224 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ static inline void sha224_update(struct sha224_ctx *ctx, const u8 *data, size_t len) { __sha256_update(&ctx->ctx, data, len); } + +/** + * sha224_final() - Finish computing a SHA-224 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-224 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ void sha224_final(struct sha224_ctx *ctx, u8 out[SHA224_DIGEST_SIZE]); + +/** + * sha224() - Compute SHA-224 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-224 message digest + * + * Context: Any context. + */ void sha224(const u8 *data, size_t len, u8 out[SHA224_DIGEST_SIZE]); /** @@ -275,13 +313,51 @@ struct sha256_ctx { struct __sha256_ctx ctx; }; +/** + * sha256_init() - Initialize a SHA-256 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha256() instead. + * + * Context: Any context. + */ void sha256_init(struct sha256_ctx *ctx); + +/** + * sha256_update() - Update a SHA-256 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ static inline void sha256_update(struct sha256_ctx *ctx, const u8 *data, size_t len) { __sha256_update(&ctx->ctx, data, len); } + +/** + * sha256_final() - Finish computing a SHA-256 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-256 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ void sha256_final(struct sha256_ctx *ctx, u8 out[SHA256_DIGEST_SIZE]); + +/** + * sha256() - Compute SHA-256 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-256 message digest + * + * Context: Any context. + */ void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); /** From a8c60a9aca778d7fd22d6c9b1af702d6f952b87f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 3 Jul 2025 19:39:57 -0700 Subject: [PATCH 42/67] lib/crypto: x86/sha256: Move static_call above kernel-mode FPU section As I did for sha512_blocks(), reorganize x86's sha256_blocks() to be just a static_call. To achieve that, for each assembly function add a C function that handles the kernel-mode FPU section and fallback. While this increases total code size slightly, the amount of code actually executed on a given system does not increase, and it is slightly more efficient since it eliminates the extra static_key. It also makes the assembly functions be called with standard direct calls instead of static calls, eliminating the need for ANNOTATE_NOENDBR. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250704023958.73274-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/x86/sha256-avx-asm.S | 3 -- lib/crypto/x86/sha256-avx2-asm.S | 3 -- lib/crypto/x86/sha256-ni-asm.S | 2 -- lib/crypto/x86/sha256-ssse3-asm.S | 3 -- lib/crypto/x86/sha256.h | 48 ++++++++++++++++--------------- 5 files changed, 25 insertions(+), 34 deletions(-) diff --git a/lib/crypto/x86/sha256-avx-asm.S b/lib/crypto/x86/sha256-avx-asm.S index 73bcff2b548f..798a7f07fa01 100644 --- a/lib/crypto/x86/sha256-avx-asm.S +++ b/lib/crypto/x86/sha256-avx-asm.S @@ -48,7 +48,6 @@ ######################################################################## #include -#include ## assume buffers not aligned #define VMOVDQ vmovdqu @@ -346,8 +345,6 @@ a = TMP_ ######################################################################## .text SYM_FUNC_START(sha256_transform_avx) - ANNOTATE_NOENDBR # since this is called only via static_call - pushq %rbx pushq %r12 pushq %r13 diff --git a/lib/crypto/x86/sha256-avx2-asm.S b/lib/crypto/x86/sha256-avx2-asm.S index 45787570387f..62a46993359e 100644 --- a/lib/crypto/x86/sha256-avx2-asm.S +++ b/lib/crypto/x86/sha256-avx2-asm.S @@ -49,7 +49,6 @@ ######################################################################## #include -#include ## assume buffers not aligned #define VMOVDQ vmovdqu @@ -523,8 +522,6 @@ STACK_SIZE = _CTX + _CTX_SIZE ######################################################################## .text SYM_FUNC_START(sha256_transform_rorx) - ANNOTATE_NOENDBR # since this is called only via static_call - pushq %rbx pushq %r12 pushq %r13 diff --git a/lib/crypto/x86/sha256-ni-asm.S b/lib/crypto/x86/sha256-ni-asm.S index 4af7d22e29e4..9ebbacbb9c13 100644 --- a/lib/crypto/x86/sha256-ni-asm.S +++ b/lib/crypto/x86/sha256-ni-asm.S @@ -54,7 +54,6 @@ */ #include -#include #define STATE_PTR %rdi /* 1st arg */ #define DATA_PTR %rsi /* 2nd arg */ @@ -111,7 +110,6 @@ */ .text SYM_FUNC_START(sha256_ni_transform) - ANNOTATE_NOENDBR # since this is called only via static_call shl $6, NUM_BLKS /* convert to bytes */ jz .Ldone_hash diff --git a/lib/crypto/x86/sha256-ssse3-asm.S b/lib/crypto/x86/sha256-ssse3-asm.S index 407b30adcd37..820fc8bbc29f 100644 --- a/lib/crypto/x86/sha256-ssse3-asm.S +++ b/lib/crypto/x86/sha256-ssse3-asm.S @@ -47,7 +47,6 @@ ######################################################################## #include -#include ## assume buffers not aligned #define MOVDQ movdqu @@ -353,8 +352,6 @@ a = TMP_ ######################################################################## .text SYM_FUNC_START(sha256_transform_ssse3) - ANNOTATE_NOENDBR # since this is called only via static_call - pushq %rbx pushq %r12 pushq %r13 diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h index 3b5456c222ba..669bc06538b6 100644 --- a/lib/crypto/x86/sha256.h +++ b/lib/crypto/x86/sha256.h @@ -8,48 +8,50 @@ #include #include -asmlinkage void sha256_transform_ssse3(struct sha256_block_state *state, - const u8 *data, size_t nblocks); -asmlinkage void sha256_transform_avx(struct sha256_block_state *state, - const u8 *data, size_t nblocks); -asmlinkage void sha256_transform_rorx(struct sha256_block_state *state, - const u8 *data, size_t nblocks); -asmlinkage void sha256_ni_transform(struct sha256_block_state *state, - const u8 *data, size_t nblocks); +DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_blocks_generic); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); +#define DEFINE_X86_SHA256_FN(c_fn, asm_fn) \ + asmlinkage void asm_fn(struct sha256_block_state *state, \ + const u8 *data, size_t nblocks); \ + static void c_fn(struct sha256_block_state *state, const u8 *data, \ + size_t nblocks) \ + { \ + if (likely(crypto_simd_usable())) { \ + kernel_fpu_begin(); \ + asm_fn(state, data, nblocks); \ + kernel_fpu_end(); \ + } else { \ + sha256_blocks_generic(state, data, nblocks); \ + } \ + } -DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); +DEFINE_X86_SHA256_FN(sha256_blocks_ssse3, sha256_transform_ssse3); +DEFINE_X86_SHA256_FN(sha256_blocks_avx, sha256_transform_avx); +DEFINE_X86_SHA256_FN(sha256_blocks_avx2, sha256_transform_rorx); +DEFINE_X86_SHA256_FN(sha256_blocks_ni, sha256_ni_transform); static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { - if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { - kernel_fpu_begin(); - static_call(sha256_blocks_x86)(state, data, nblocks); - kernel_fpu_end(); - } else { - sha256_blocks_generic(state, data, nblocks); - } + static_call(sha256_blocks_x86)(state, data, nblocks); } #define sha256_mod_init_arch sha256_mod_init_arch static inline void sha256_mod_init_arch(void) { if (boot_cpu_has(X86_FEATURE_SHA_NI)) { - static_call_update(sha256_blocks_x86, sha256_ni_transform); + static_call_update(sha256_blocks_x86, sha256_blocks_ni); } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && boot_cpu_has(X86_FEATURE_AVX)) { if (boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2)) static_call_update(sha256_blocks_x86, - sha256_transform_rorx); + sha256_blocks_avx2); else static_call_update(sha256_blocks_x86, - sha256_transform_avx); - } else if (!boot_cpu_has(X86_FEATURE_SSSE3)) { - return; + sha256_blocks_avx); + } else if (boot_cpu_has(X86_FEATURE_SSSE3)) { + static_call_update(sha256_blocks_x86, sha256_blocks_ssse3); } - static_branch_enable(&have_sha256_x86); } From 57b15e9260a31438e91cf83dbfcb63333b24c684 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 3 Jul 2025 19:39:58 -0700 Subject: [PATCH 43/67] lib/crypto: x86/sha256: Remove unnecessary checks for nblocks==0 Since sha256_blocks() is called only with nblocks >= 1, remove unnecessary checks for nblocks == 0 from the x86 SHA-256 assembly code. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250704023958.73274-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/x86/sha256-avx-asm.S | 3 --- lib/crypto/x86/sha256-avx2-asm.S | 1 - lib/crypto/x86/sha256-ni-asm.S | 3 --- lib/crypto/x86/sha256-ssse3-asm.S | 3 --- 4 files changed, 10 deletions(-) diff --git a/lib/crypto/x86/sha256-avx-asm.S b/lib/crypto/x86/sha256-avx-asm.S index 798a7f07fa01..c1aceb3ba3a3 100644 --- a/lib/crypto/x86/sha256-avx-asm.S +++ b/lib/crypto/x86/sha256-avx-asm.S @@ -357,7 +357,6 @@ SYM_FUNC_START(sha256_transform_avx) and $~15, %rsp # align stack pointer shl $6, NUM_BLKS # convert to bytes - jz .Ldone_hash add INP, NUM_BLKS # pointer to end of data mov NUM_BLKS, _INP_END(%rsp) @@ -446,8 +445,6 @@ SYM_FUNC_START(sha256_transform_avx) cmp _INP_END(%rsp), INP jne .Lloop0 -.Ldone_hash: - mov %rbp, %rsp popq %rbp popq %r15 diff --git a/lib/crypto/x86/sha256-avx2-asm.S b/lib/crypto/x86/sha256-avx2-asm.S index 62a46993359e..eb8836fb9695 100644 --- a/lib/crypto/x86/sha256-avx2-asm.S +++ b/lib/crypto/x86/sha256-avx2-asm.S @@ -535,7 +535,6 @@ SYM_FUNC_START(sha256_transform_rorx) and $-32, %rsp # align rsp to 32 byte boundary shl $6, NUM_BLKS # convert to bytes - jz .Ldone_hash lea -64(INP, NUM_BLKS), NUM_BLKS # pointer to last block mov NUM_BLKS, _INP_END(%rsp) diff --git a/lib/crypto/x86/sha256-ni-asm.S b/lib/crypto/x86/sha256-ni-asm.S index 9ebbacbb9c13..4bd9490ffc66 100644 --- a/lib/crypto/x86/sha256-ni-asm.S +++ b/lib/crypto/x86/sha256-ni-asm.S @@ -112,7 +112,6 @@ SYM_FUNC_START(sha256_ni_transform) shl $6, NUM_BLKS /* convert to bytes */ - jz .Ldone_hash add DATA_PTR, NUM_BLKS /* pointer to end of data */ /* @@ -163,8 +162,6 @@ SYM_FUNC_START(sha256_ni_transform) movdqu STATE1, 0*16(STATE_PTR) movdqu STATE0, 1*16(STATE_PTR) -.Ldone_hash: - RET SYM_FUNC_END(sha256_ni_transform) diff --git a/lib/crypto/x86/sha256-ssse3-asm.S b/lib/crypto/x86/sha256-ssse3-asm.S index 820fc8bbc29f..383b8eec7ebe 100644 --- a/lib/crypto/x86/sha256-ssse3-asm.S +++ b/lib/crypto/x86/sha256-ssse3-asm.S @@ -364,7 +364,6 @@ SYM_FUNC_START(sha256_transform_ssse3) and $~15, %rsp shl $6, NUM_BLKS # convert to bytes - jz .Ldone_hash add INP, NUM_BLKS mov NUM_BLKS, _INP_END(%rsp) # pointer to end of data @@ -457,8 +456,6 @@ SYM_FUNC_START(sha256_transform_ssse3) cmp _INP_END(%rsp), INP jne .Lloop0 -.Ldone_hash: - mov %rbp, %rsp popq %rbp popq %r15 From aacb37f597d0b50800a233ab2c29c195a1595147 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Jun 2025 10:22:23 -0700 Subject: [PATCH 44/67] lib/crypto: hash_info: Move hash_info.c into lib/crypto/ crypto/hash_info.c just contains a couple of arrays that map HASH_ALGO_* algorithm IDs to properties of those algorithms. It is compiled only when CRYPTO_HASH_INFO=y, but currently CRYPTO_HASH_INFO depends on CRYPTO. Since this can be useful without the old-school crypto API, move it into lib/crypto/ so that it no longer depends on CRYPTO. This eliminates the need for FS_VERITY to select CRYPTO after it's been converted to use lib/crypto/. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250630172224.46909-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 3 --- crypto/Makefile | 1 - lib/crypto/Kconfig | 3 +++ lib/crypto/Makefile | 2 ++ {crypto => lib/crypto}/hash_info.c | 0 5 files changed, 5 insertions(+), 4 deletions(-) rename {crypto => lib/crypto}/hash_info.c (100%) diff --git a/crypto/Kconfig b/crypto/Kconfig index 3ea1397214e0..5d4cf022c577 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1422,9 +1422,6 @@ config CRYPTO_USER_API_ENABLE_OBSOLETE endmenu -config CRYPTO_HASH_INFO - bool - if !KMSAN # avoid false positives from assembly if ARM source "arch/arm/crypto/Kconfig" diff --git a/crypto/Makefile b/crypto/Makefile index 5098fa6d5f39..816607e0e78c 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -204,7 +204,6 @@ obj-$(CONFIG_CRYPTO_ECRDSA) += ecrdsa_generic.o obj-$(CONFIG_XOR_BLOCKS) += xor.o obj-$(CONFIG_ASYNC_CORE) += async_tx/ obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/ -obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o crypto_simd-y := simd.o obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 3305c6908581..ee7e1e27ea6a 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -2,6 +2,9 @@ menu "Crypto library routines" +config CRYPTO_HASH_INFO + bool + config CRYPTO_LIB_UTILS tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index a887bf103bf0..533bb1533e19 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -8,6 +8,8 @@ quiet_cmd_perlasm = PERLASM $@ quiet_cmd_perlasm_with_args = PERLASM $@ cmd_perlasm_with_args = $(PERL) $(<) void $(@) +obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o + obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o libcryptoutils-y := memneq.o utils.o diff --git a/crypto/hash_info.c b/lib/crypto/hash_info.c similarity index 100% rename from crypto/hash_info.c rename to lib/crypto/hash_info.c From 52c3e242f4d0043186b70d65460ba1767f27494a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 6 Jul 2025 16:10:57 -0700 Subject: [PATCH 45/67] lib/crypto: arm/poly1305: Fix register corruption in no-SIMD contexts Restore the SIMD usability check that was removed by commit 773426f4771b ("crypto: arm/poly1305 - Add block-only interface"). This safety check is cheap and is well worth eliminating a footgun. While the Poly1305 functions should not be called when SIMD registers are unusable, if they are anyway, they should just do the right thing instead of corrupting random tasks' registers and/or computing incorrect MACs. Fixing this is also needed for poly1305_kunit to pass. Just use may_use_simd() instead of the original crypto_simd_usable(), since poly1305_kunit won't rely on crypto_simd_disabled_for_test. Fixes: 773426f4771b ("crypto: arm/poly1305 - Add block-only interface") Cc: stable@vger.kernel.org Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250706231100.176113-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm/poly1305-glue.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/crypto/arm/poly1305-glue.c b/lib/crypto/arm/poly1305-glue.c index 2603b0771f2c..ca6dc5533705 100644 --- a/lib/crypto/arm/poly1305-glue.c +++ b/lib/crypto/arm/poly1305-glue.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -39,7 +40,7 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, { len = round_down(len, POLY1305_BLOCK_SIZE); if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon)) { + static_branch_likely(&have_neon) && likely(may_use_simd())) { do { unsigned int todo = min_t(unsigned int, len, SZ_4K); From eec76ea5a7213c48529a46eed1b343e5cee3aaab Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 6 Jul 2025 16:10:58 -0700 Subject: [PATCH 46/67] lib/crypto: arm64/poly1305: Fix register corruption in no-SIMD contexts Restore the SIMD usability check that was removed by commit a59e5468a921 ("crypto: arm64/poly1305 - Add block-only interface"). This safety check is cheap and is well worth eliminating a footgun. While the Poly1305 functions should not be called when SIMD registers are unusable, if they are anyway, they should just do the right thing instead of corrupting random tasks' registers and/or computing incorrect MACs. Fixing this is also needed for poly1305_kunit to pass. Just use may_use_simd() instead of the original crypto_simd_usable(), since poly1305_kunit won't rely on crypto_simd_disabled_for_test. Fixes: a59e5468a921 ("crypto: arm64/poly1305 - Add block-only interface") Cc: stable@vger.kernel.org Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250706231100.176113-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/poly1305-glue.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/crypto/arm64/poly1305-glue.c b/lib/crypto/arm64/poly1305-glue.c index c9a74766785b..31aea21ce42f 100644 --- a/lib/crypto/arm64/poly1305-glue.c +++ b/lib/crypto/arm64/poly1305-glue.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -33,7 +34,7 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { len = round_down(len, POLY1305_BLOCK_SIZE); - if (static_branch_likely(&have_neon)) { + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { do { unsigned int todo = min_t(unsigned int, len, SZ_4K); From 16f2c30e290e04135b70ad374fb7e1d1ed9ff5e7 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 6 Jul 2025 16:10:59 -0700 Subject: [PATCH 47/67] lib/crypto: x86/poly1305: Fix register corruption in no-SIMD contexts Restore the SIMD usability check and base conversion that were removed by commit 318c53ae02f2 ("crypto: x86/poly1305 - Add block-only interface"). This safety check is cheap and is well worth eliminating a footgun. While the Poly1305 functions should not be called when SIMD registers are unusable, if they are anyway, they should just do the right thing instead of corrupting random tasks' registers and/or computing incorrect MACs. Fixing this is also needed for poly1305_kunit to pass. Just use irq_fpu_usable() instead of the original crypto_simd_usable(), since poly1305_kunit won't rely on crypto_simd_disabled_for_test. Fixes: 318c53ae02f2 ("crypto: x86/poly1305 - Add block-only interface") Cc: stable@vger.kernel.org Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250706231100.176113-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/x86/poly1305_glue.c | 40 +++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/crypto/x86/poly1305_glue.c b/lib/crypto/x86/poly1305_glue.c index b7e78a583e07..968d84677631 100644 --- a/lib/crypto/x86/poly1305_glue.c +++ b/lib/crypto/x86/poly1305_glue.c @@ -25,6 +25,42 @@ struct poly1305_arch_internal { struct { u32 r2, r1, r4, r3; } rn[9]; }; +/* + * The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit + * the unfortunate situation of using AVX and then having to go back to scalar + * -- because the user is silly and has called the update function from two + * separate contexts -- then we need to convert back to the original base before + * proceeding. It is possible to reason that the initial reduction below is + * sufficient given the implementation invariants. However, for an avoidance of + * doubt and because this is not performance critical, we do the full reduction + * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py + */ +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->hs[2] = state->h[4] >> 24; + /* Unsigned Less Than: branchlessly produces 1 if a < b, else 0. */ +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); + state->hs[2] &= 3; + state->hs[0] += cy; + state->hs[1] += (cy = ULT(state->hs[0], cy)); + state->hs[2] += ULT(state->hs[1], cy); +#undef ULT + state->is_base2_26 = 0; +} + asmlinkage void poly1305_block_init_arch( struct poly1305_block_state *state, const u8 raw_key[POLY1305_BLOCK_SIZE]); @@ -62,7 +98,9 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *inp, BUILD_BUG_ON(SZ_4K < POLY1305_BLOCK_SIZE || SZ_4K % POLY1305_BLOCK_SIZE); - if (!static_branch_likely(&poly1305_use_avx)) { + if (!static_branch_likely(&poly1305_use_avx) || + unlikely(!irq_fpu_usable())) { + convert_to_base2_64(ctx); poly1305_blocks_x86_64(ctx, inp, len, padbit); return; } From 9f65592b7e1f24569bb6ced064df5b4319f725ce Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 6 Jul 2025 16:11:00 -0700 Subject: [PATCH 48/67] lib/crypto: x86/poly1305: Fix performance regression on short messages Restore the len >= 288 condition on using the AVX implementation, which was incidentally removed by commit 318c53ae02f2 ("crypto: x86/poly1305 - Add block-only interface"). This check took into account the overhead in key power computation, kernel-mode "FPU", and tail handling associated with the AVX code. Indeed, restoring this check slightly improves performance for len < 256 as measured using poly1305_kunit on an "AMD Ryzen AI 9 365" (Zen 5) CPU: Length Before After ====== ========== ========== 1 30 MB/s 36 MB/s 16 516 MB/s 598 MB/s 64 1700 MB/s 1882 MB/s 127 2265 MB/s 2651 MB/s 128 2457 MB/s 2827 MB/s 200 2702 MB/s 3238 MB/s 256 3841 MB/s 3768 MB/s 511 4580 MB/s 4585 MB/s 512 5430 MB/s 5398 MB/s 1024 7268 MB/s 7305 MB/s 3173 8999 MB/s 8948 MB/s 4096 9942 MB/s 9921 MB/s 16384 10557 MB/s 10545 MB/s While the optimal threshold for this CPU might be slightly lower than 288 (see the len == 256 case), other CPUs would need to be tested too, and these sorts of benchmarks can underestimate the true cost of kernel-mode "FPU". Therefore, for now just restore the 288 threshold. Fixes: 318c53ae02f2 ("crypto: x86/poly1305 - Add block-only interface") Cc: stable@vger.kernel.org Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250706231100.176113-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/x86/poly1305_glue.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/crypto/x86/poly1305_glue.c b/lib/crypto/x86/poly1305_glue.c index 968d84677631..856d48fd422b 100644 --- a/lib/crypto/x86/poly1305_glue.c +++ b/lib/crypto/x86/poly1305_glue.c @@ -98,7 +98,15 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *inp, BUILD_BUG_ON(SZ_4K < POLY1305_BLOCK_SIZE || SZ_4K % POLY1305_BLOCK_SIZE); + /* + * The AVX implementations have significant setup overhead (e.g. key + * power computation, kernel FPU enabling) which makes them slower for + * short messages. Fall back to the scalar implementation for messages + * shorter than 288 bytes, unless the AVX-specific key setup has already + * been performed (indicated by ctx->is_base2_26). + */ if (!static_branch_likely(&poly1305_use_avx) || + (len < POLY1305_BLOCK_SIZE * 18 && !ctx->is_base2_26) || unlikely(!irq_fpu_usable())) { convert_to_base2_64(ctx); poly1305_blocks_x86_64(ctx, inp, len, padbit); From 6e07c5e166597de1d7943ecf2539cad18c0e2ce1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 11 Jul 2025 14:28:22 -0700 Subject: [PATCH 49/67] lib/crypto: arm/poly1305: Remove unneeded empty weak function Fix poly1305-armv4.pl to not do '.globl poly1305_blocks_neon' when poly1305_blocks_neon() is not defined. Then, remove the empty __weak definition of poly1305_blocks_neon(), which was still needed only because of that unnecessary globl statement. (It also used to be needed because the compiler could generate calls to it when CONFIG_KERNEL_MODE_NEON=n, but that has been fixed.) Thanks to Arnd Bergmann for reporting that the globl statement in the asm file was still depending on the weak symbol. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250711212822.6372-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm/poly1305-armv4.pl | 2 +- lib/crypto/arm/poly1305-glue.c | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/crypto/arm/poly1305-armv4.pl b/lib/crypto/arm/poly1305-armv4.pl index d57c6e2fc84a..dd7a996361a7 100644 --- a/lib/crypto/arm/poly1305-armv4.pl +++ b/lib/crypto/arm/poly1305-armv4.pl @@ -46,7 +46,6 @@ $code.=<<___; # define poly1305_init poly1305_block_init_arch # define poly1305_blocks poly1305_blocks_arm # define poly1305_emit poly1305_emit_arch -.globl poly1305_blocks_neon #endif #if defined(__thumb2__) @@ -722,6 +721,7 @@ poly1305_init_neon: ret @ bx lr .size poly1305_init_neon,.-poly1305_init_neon +.globl poly1305_blocks_neon .type poly1305_blocks_neon,%function .align 5 poly1305_blocks_neon: diff --git a/lib/crypto/arm/poly1305-glue.c b/lib/crypto/arm/poly1305-glue.c index ca6dc5533705..2d86c78af883 100644 --- a/lib/crypto/arm/poly1305-glue.c +++ b/lib/crypto/arm/poly1305-glue.c @@ -28,11 +28,6 @@ asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, const u32 nonce[4]); EXPORT_SYMBOL_GPL(poly1305_emit_arch); -void __weak poly1305_blocks_neon(struct poly1305_block_state *state, - const u8 *src, u32 len, u32 hibit) -{ -} - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, From 7941ad696506917fa6228f44be2df0c2f0909a62 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 11 Jul 2025 14:58:43 -0700 Subject: [PATCH 50/67] lib/crypto: sha2: Add hmac_sha*_init_usingrawkey() While the HMAC library functions support both incremental and one-shot computation and both prepared and raw keys, the combination of raw key + incremental was missing. It turns out that several potential users of the HMAC library functions (tpm2-sessions.c, smb2transport.c, trusted_tpm1.c) want exactly that. Therefore, add the missing functions hmac_sha*_init_usingrawkey(). Implement them in an optimized way that directly initializes the HMAC context without a separate key preparation step. Reimplement the one-shot raw key functions hmac_sha*_usingrawkey() on top of the new functions, which makes them a bit more efficient. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250711215844.41715-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 60 +++++++++++++++++++++++++++++++++++++++++++ lib/crypto/sha256.c | 53 ++++++++++++++++++++++++++------------ lib/crypto/sha512.c | 55 +++++++++++++++++++++++++++------------ 3 files changed, 134 insertions(+), 34 deletions(-) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index e0a08f6addd0..15e461e568cc 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -247,6 +247,21 @@ static inline void hmac_sha224_init(struct hmac_sha224_ctx *ctx, __hmac_sha256_init(&ctx->ctx, &key->key); } +/** + * hmac_sha224_init_usingrawkey() - Initialize an HMAC-SHA224 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA224 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha224_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha224_init_usingrawkey(struct hmac_sha224_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + /** * hmac_sha224_update() - Update an HMAC-SHA224 context with message data * @ctx: the HMAC context to update; must have been initialized @@ -405,6 +420,21 @@ static inline void hmac_sha256_init(struct hmac_sha256_ctx *ctx, __hmac_sha256_init(&ctx->ctx, &key->key); } +/** + * hmac_sha256_init_usingrawkey() - Initialize an HMAC-SHA256 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA256 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha256_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha256_init_usingrawkey(struct hmac_sha256_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + /** * hmac_sha256_update() - Update an HMAC-SHA256 context with message data * @ctx: the HMAC context to update; must have been initialized @@ -597,6 +627,21 @@ static inline void hmac_sha384_init(struct hmac_sha384_ctx *ctx, __hmac_sha512_init(&ctx->ctx, &key->key); } +/** + * hmac_sha384_init_usingrawkey() - Initialize an HMAC-SHA384 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA384 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha384_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha384_init_usingrawkey(struct hmac_sha384_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + /** * hmac_sha384_update() - Update an HMAC-SHA384 context with message data * @ctx: the HMAC context to update; must have been initialized @@ -755,6 +800,21 @@ static inline void hmac_sha512_init(struct hmac_sha512_ctx *ctx, __hmac_sha512_init(&ctx->ctx, &key->key); } +/** + * hmac_sha512_init_usingrawkey() - Initialize an HMAC-SHA512 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA512 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha512_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha512_init_usingrawkey(struct hmac_sha512_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + /** * hmac_sha512_update() - Update an HMAC-SHA512 context with message data * @ctx: the HMAC context to update; must have been initialized diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 808438d4f427..8fa15165d23e 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -263,7 +263,8 @@ EXPORT_SYMBOL(sha256); /* pre-boot environment (as indicated by __DISABLE_EXPORTS) doesn't need HMAC */ #ifndef __DISABLE_EXPORTS -static void __hmac_sha256_preparekey(struct __hmac_sha256_key *key, +static void __hmac_sha256_preparekey(struct sha256_block_state *istate, + struct sha256_block_state *ostate, const u8 *raw_key, size_t raw_key_len, const struct sha256_block_state *iv) { @@ -283,14 +284,14 @@ static void __hmac_sha256_preparekey(struct __hmac_sha256_key *key, for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) derived_key.w[i] ^= REPEAT_BYTE(HMAC_IPAD_VALUE); - key->istate = *iv; - sha256_blocks(&key->istate, derived_key.b, 1); + *istate = *iv; + sha256_blocks(istate, derived_key.b, 1); for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) derived_key.w[i] ^= REPEAT_BYTE(HMAC_OPAD_VALUE ^ HMAC_IPAD_VALUE); - key->ostate = *iv; - sha256_blocks(&key->ostate, derived_key.b, 1); + *ostate = *iv; + sha256_blocks(ostate, derived_key.b, 1); memzero_explicit(&derived_key, sizeof(derived_key)); } @@ -298,14 +299,16 @@ static void __hmac_sha256_preparekey(struct __hmac_sha256_key *key, void hmac_sha224_preparekey(struct hmac_sha224_key *key, const u8 *raw_key, size_t raw_key_len) { - __hmac_sha256_preparekey(&key->key, raw_key, raw_key_len, &sha224_iv); + __hmac_sha256_preparekey(&key->key.istate, &key->key.ostate, + raw_key, raw_key_len, &sha224_iv); } EXPORT_SYMBOL_GPL(hmac_sha224_preparekey); void hmac_sha256_preparekey(struct hmac_sha256_key *key, const u8 *raw_key, size_t raw_key_len) { - __hmac_sha256_preparekey(&key->key, raw_key, raw_key_len, &sha256_iv); + __hmac_sha256_preparekey(&key->key.istate, &key->key.ostate, + raw_key, raw_key_len, &sha256_iv); } EXPORT_SYMBOL_GPL(hmac_sha256_preparekey); @@ -317,6 +320,24 @@ void __hmac_sha256_init(struct __hmac_sha256_ctx *ctx, } EXPORT_SYMBOL_GPL(__hmac_sha256_init); +void hmac_sha224_init_usingrawkey(struct hmac_sha224_ctx *ctx, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha256_preparekey(&ctx->ctx.sha_ctx.state, &ctx->ctx.ostate, + raw_key, raw_key_len, &sha224_iv); + ctx->ctx.sha_ctx.bytecount = SHA256_BLOCK_SIZE; +} +EXPORT_SYMBOL_GPL(hmac_sha224_init_usingrawkey); + +void hmac_sha256_init_usingrawkey(struct hmac_sha256_ctx *ctx, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha256_preparekey(&ctx->ctx.sha_ctx.state, &ctx->ctx.ostate, + raw_key, raw_key_len, &sha256_iv); + ctx->ctx.sha_ctx.bytecount = SHA256_BLOCK_SIZE; +} +EXPORT_SYMBOL_GPL(hmac_sha256_init_usingrawkey); + static void __hmac_sha256_final(struct __hmac_sha256_ctx *ctx, u8 *out, size_t digest_size) { @@ -376,12 +397,11 @@ void hmac_sha224_usingrawkey(const u8 *raw_key, size_t raw_key_len, const u8 *data, size_t data_len, u8 out[SHA224_DIGEST_SIZE]) { - struct hmac_sha224_key key; + struct hmac_sha224_ctx ctx; - hmac_sha224_preparekey(&key, raw_key, raw_key_len); - hmac_sha224(&key, data, data_len, out); - - memzero_explicit(&key, sizeof(key)); + hmac_sha224_init_usingrawkey(&ctx, raw_key, raw_key_len); + hmac_sha224_update(&ctx, data, data_len); + hmac_sha224_final(&ctx, out); } EXPORT_SYMBOL_GPL(hmac_sha224_usingrawkey); @@ -389,12 +409,11 @@ void hmac_sha256_usingrawkey(const u8 *raw_key, size_t raw_key_len, const u8 *data, size_t data_len, u8 out[SHA256_DIGEST_SIZE]) { - struct hmac_sha256_key key; + struct hmac_sha256_ctx ctx; - hmac_sha256_preparekey(&key, raw_key, raw_key_len); - hmac_sha256(&key, data, data_len, out); - - memzero_explicit(&key, sizeof(key)); + hmac_sha256_init_usingrawkey(&ctx, raw_key, raw_key_len); + hmac_sha256_update(&ctx, data, data_len); + hmac_sha256_final(&ctx, out); } EXPORT_SYMBOL_GPL(hmac_sha256_usingrawkey); #endif /* !__DISABLE_EXPORTS */ diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c index d514721491ca..d8062188be98 100644 --- a/lib/crypto/sha512.c +++ b/lib/crypto/sha512.c @@ -249,7 +249,8 @@ void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]) } EXPORT_SYMBOL_GPL(sha512); -static void __hmac_sha512_preparekey(struct __hmac_sha512_key *key, +static void __hmac_sha512_preparekey(struct sha512_block_state *istate, + struct sha512_block_state *ostate, const u8 *raw_key, size_t raw_key_len, const struct sha512_block_state *iv) { @@ -269,14 +270,14 @@ static void __hmac_sha512_preparekey(struct __hmac_sha512_key *key, for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) derived_key.w[i] ^= REPEAT_BYTE(HMAC_IPAD_VALUE); - key->istate = *iv; - sha512_blocks(&key->istate, derived_key.b, 1); + *istate = *iv; + sha512_blocks(istate, derived_key.b, 1); for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) derived_key.w[i] ^= REPEAT_BYTE(HMAC_OPAD_VALUE ^ HMAC_IPAD_VALUE); - key->ostate = *iv; - sha512_blocks(&key->ostate, derived_key.b, 1); + *ostate = *iv; + sha512_blocks(ostate, derived_key.b, 1); memzero_explicit(&derived_key, sizeof(derived_key)); } @@ -284,14 +285,16 @@ static void __hmac_sha512_preparekey(struct __hmac_sha512_key *key, void hmac_sha384_preparekey(struct hmac_sha384_key *key, const u8 *raw_key, size_t raw_key_len) { - __hmac_sha512_preparekey(&key->key, raw_key, raw_key_len, &sha384_iv); + __hmac_sha512_preparekey(&key->key.istate, &key->key.ostate, + raw_key, raw_key_len, &sha384_iv); } EXPORT_SYMBOL_GPL(hmac_sha384_preparekey); void hmac_sha512_preparekey(struct hmac_sha512_key *key, const u8 *raw_key, size_t raw_key_len) { - __hmac_sha512_preparekey(&key->key, raw_key, raw_key_len, &sha512_iv); + __hmac_sha512_preparekey(&key->key.istate, &key->key.ostate, + raw_key, raw_key_len, &sha512_iv); } EXPORT_SYMBOL_GPL(hmac_sha512_preparekey); @@ -303,6 +306,26 @@ void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx, } EXPORT_SYMBOL_GPL(__hmac_sha512_init); +void hmac_sha384_init_usingrawkey(struct hmac_sha384_ctx *ctx, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha512_preparekey(&ctx->ctx.sha_ctx.state, &ctx->ctx.ostate, + raw_key, raw_key_len, &sha384_iv); + ctx->ctx.sha_ctx.bytecount_lo = SHA512_BLOCK_SIZE; + ctx->ctx.sha_ctx.bytecount_hi = 0; +} +EXPORT_SYMBOL_GPL(hmac_sha384_init_usingrawkey); + +void hmac_sha512_init_usingrawkey(struct hmac_sha512_ctx *ctx, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha512_preparekey(&ctx->ctx.sha_ctx.state, &ctx->ctx.ostate, + raw_key, raw_key_len, &sha512_iv); + ctx->ctx.sha_ctx.bytecount_lo = SHA512_BLOCK_SIZE; + ctx->ctx.sha_ctx.bytecount_hi = 0; +} +EXPORT_SYMBOL_GPL(hmac_sha512_init_usingrawkey); + static void __hmac_sha512_final(struct __hmac_sha512_ctx *ctx, u8 *out, size_t digest_size) { @@ -362,12 +385,11 @@ void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len, const u8 *data, size_t data_len, u8 out[SHA384_DIGEST_SIZE]) { - struct hmac_sha384_key key; + struct hmac_sha384_ctx ctx; - hmac_sha384_preparekey(&key, raw_key, raw_key_len); - hmac_sha384(&key, data, data_len, out); - - memzero_explicit(&key, sizeof(key)); + hmac_sha384_init_usingrawkey(&ctx, raw_key, raw_key_len); + hmac_sha384_update(&ctx, data, data_len); + hmac_sha384_final(&ctx, out); } EXPORT_SYMBOL_GPL(hmac_sha384_usingrawkey); @@ -375,12 +397,11 @@ void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len, const u8 *data, size_t data_len, u8 out[SHA512_DIGEST_SIZE]) { - struct hmac_sha512_key key; + struct hmac_sha512_ctx ctx; - hmac_sha512_preparekey(&key, raw_key, raw_key_len); - hmac_sha512(&key, data, data_len, out); - - memzero_explicit(&key, sizeof(key)); + hmac_sha512_init_usingrawkey(&ctx, raw_key, raw_key_len); + hmac_sha512_update(&ctx, data, data_len); + hmac_sha512_final(&ctx, out); } EXPORT_SYMBOL_GPL(hmac_sha512_usingrawkey); From 56119446f89fbab40f3a160e9c3de33687cd85cf Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:22:52 -0700 Subject: [PATCH 51/67] crypto: x86/sha1 - Rename conflicting symbol Rename x86's sha1_update() to sha1_update_x86(), since it conflicts with the upcoming sha1_update() library function. Note: the affected code will be superseded by later commits that migrate the arch-optimized SHA-1 code into the library. This commit simply keeps the kernel building for the initial introduction of the library. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/x86/crypto/sha1_ssse3_glue.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 0a912bfc86c5..826579a7473c 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -34,7 +34,7 @@ static const struct x86_cpu_id module_cpu_ids[] = { }; MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); -static inline int sha1_update(struct shash_desc *desc, const u8 *data, +static inline int sha1_update_x86(struct shash_desc *desc, const u8 *data, unsigned int len, sha1_block_fn *sha1_xform) { int remain; @@ -69,7 +69,7 @@ asmlinkage void sha1_transform_ssse3(struct sha1_state *state, static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha1_update(desc, data, len, sha1_transform_ssse3); + return sha1_update_x86(desc, data, len, sha1_transform_ssse3); } static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, @@ -113,7 +113,7 @@ asmlinkage void sha1_transform_avx(struct sha1_state *state, static int sha1_avx_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha1_update(desc, data, len, sha1_transform_avx); + return sha1_update_x86(desc, data, len, sha1_transform_avx); } static int sha1_avx_finup(struct shash_desc *desc, const u8 *data, @@ -190,7 +190,7 @@ static inline void sha1_apply_transform_avx2(struct sha1_state *state, static int sha1_avx2_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha1_update(desc, data, len, sha1_apply_transform_avx2); + return sha1_update_x86(desc, data, len, sha1_apply_transform_avx2); } static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data, @@ -234,7 +234,7 @@ asmlinkage void sha1_ni_transform(struct sha1_state *digest, const u8 *data, static int sha1_ni_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha1_update(desc, data, len, sha1_ni_transform); + return sha1_update_x86(desc, data, len, sha1_ni_transform); } static int sha1_ni_finup(struct shash_desc *desc, const u8 *data, From 9503ca2ccafec51ee9e533d6f3aef14a589fc106 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:22:53 -0700 Subject: [PATCH 52/67] lib/crypto: sha1: Rename sha1_init() to sha1_init_raw() Rename the existing sha1_init() to sha1_init_raw(), since it conflicts with the upcoming library function. This will later be removed, but this keeps the kernel building for the introduction of the library. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha1.h | 2 +- kernel/bpf/core.c | 2 +- lib/crypto/sha1.c | 6 +++--- net/ipv6/addrconf.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h index f48230b1413c..d853d3b93169 100644 --- a/include/crypto/sha1.h +++ b/include/crypto/sha1.h @@ -33,7 +33,7 @@ struct sha1_state { */ #define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) #define SHA1_WORKSPACE_WORDS 16 -void sha1_init(__u32 *buf); +void sha1_init_raw(__u32 *buf); void sha1_transform(__u32 *digest, const char *data, __u32 *W); #endif /* _CRYPTO_SHA1_H */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index c20babbf998f..dae281a1286d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -304,7 +304,7 @@ int bpf_prog_calc_tag(struct bpf_prog *fp) if (!raw) return -ENOMEM; - sha1_init(digest); + sha1_init_raw(digest); memset(ws, 0, sizeof(ws)); /* We need to take out the map fd for the digest calculation diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index 6d809c3088be..813ad96daa25 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -124,10 +124,10 @@ void sha1_transform(__u32 *digest, const char *data, __u32 *array) EXPORT_SYMBOL(sha1_transform); /** - * sha1_init - initialize the vectors for a SHA1 digest + * sha1_init_raw - initialize the vectors for a SHA1 digest * @buf: vector to initialize */ -void sha1_init(__u32 *buf) +void sha1_init_raw(__u32 *buf) { buf[0] = 0x67452301; buf[1] = 0xefcdab89; @@ -135,7 +135,7 @@ void sha1_init(__u32 *buf) buf[3] = 0x10325476; buf[4] = 0xc3d2e1f0; } -EXPORT_SYMBOL(sha1_init); +EXPORT_SYMBOL(sha1_init_raw); MODULE_DESCRIPTION("SHA-1 Algorithm"); MODULE_LICENSE("GPL"); diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index ba2ec7c870cc..d0e5b94c10af 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -3367,7 +3367,7 @@ static int ipv6_generate_stable_address(struct in6_addr *address, retry: spin_lock_bh(&lock); - sha1_init(digest); + sha1_init_raw(digest); memset(&data, 0, sizeof(data)); memset(workspace, 0, sizeof(workspace)); memcpy(data.hwaddr, idev->dev->perm_addr, idev->dev->addr_len); From 90860aef630c5c9e58d05044f2866fcbfa7aa4d9 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:22:54 -0700 Subject: [PATCH 53/67] lib/crypto: sha1: Add SHA-1 library functions Add a library interface for SHA-1, following the SHA-2 one. As was the case with SHA-2, this will be useful for various in-kernel users. The crypto_shash interface will be reimplemented on top of it as well. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha1.h | 60 ++++++++++++++++++++++ lib/crypto/Kconfig | 7 +++ lib/crypto/Makefile | 9 +++- lib/crypto/sha1.c | 115 ++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 184 insertions(+), 7 deletions(-) diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h index d853d3b93169..387f6123a05e 100644 --- a/include/crypto/sha1.h +++ b/include/crypto/sha1.h @@ -36,4 +36,64 @@ struct sha1_state { void sha1_init_raw(__u32 *buf); void sha1_transform(__u32 *digest, const char *data, __u32 *W); +/* State for the SHA-1 compression function */ +struct sha1_block_state { + u32 h[SHA1_DIGEST_SIZE / 4]; +}; + +/** + * struct sha1_ctx - Context for hashing a message with SHA-1 + * @state: the compression function state + * @bytecount: number of bytes processed so far + * @buf: partial block buffer; bytecount % SHA1_BLOCK_SIZE bytes are valid + */ +struct sha1_ctx { + struct sha1_block_state state; + u64 bytecount; + u8 buf[SHA1_BLOCK_SIZE]; +}; + +/** + * sha1_init() - Initialize a SHA-1 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha1() instead. + * + * Context: Any context. + */ +void sha1_init(struct sha1_ctx *ctx); + +/** + * sha1_update() - Update a SHA-1 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +void sha1_update(struct sha1_ctx *ctx, const u8 *data, size_t len); + +/** + * sha1_final() - Finish computing a SHA-1 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-1 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha1_final(struct sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]); + +/** + * sha1() - Compute SHA-1 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-1 message digest + * + * Context: Any context. + */ +void sha1(const u8 *data, size_t len, u8 out[SHA1_DIGEST_SIZE]); + #endif /* _CRYPTO_SHA1_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index ee7e1e27ea6a..5aaf484fc9de 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -139,6 +139,13 @@ config CRYPTO_LIB_CHACHA20POLY1305 config CRYPTO_LIB_SHA1 tristate + help + The SHA-1 library functions. Select this if your module uses any of + the functions from . + +config CRYPTO_LIB_SHA1_ARCH + bool + depends on CRYPTO_LIB_SHA1 && !UML config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 533bb1533e19..0eb0906d693f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -65,8 +65,13 @@ libpoly1305-generic-y := poly1305-donna32.o libpoly1305-generic-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o libpoly1305-generic-y += poly1305-generic.o -obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o -libsha1-y := sha1.o +################################################################################ + +obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o +libsha1-y := sha1.o +ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y) +CFLAGS_sha1.o += -I$(src)/$(SRCARCH) +endif # CONFIG_CRYPTO_LIB_SHA1_ARCH ################################################################################ diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index 813ad96daa25..0fe9ca8d0653 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -1,9 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * SHA1 routine optimized to do word accesses rather than byte accesses, - * and to avoid unnecessary copies into the context array. - * - * This was based on the git SHA1 implementation. + * SHA-1 library functions */ #include @@ -14,6 +11,10 @@ #include #include +static const struct sha1_block_state sha1_iv = { + .h = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, +}; + /* * If you have 32 registers or more, the compiler can (and should) * try to change the array[] accesses into registers. However, on @@ -137,5 +138,109 @@ void sha1_init_raw(__u32 *buf) } EXPORT_SYMBOL(sha1_init_raw); -MODULE_DESCRIPTION("SHA-1 Algorithm"); +static void __maybe_unused sha1_blocks_generic(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + u32 workspace[SHA1_WORKSPACE_WORDS]; + + do { + sha1_transform(state->h, data, workspace); + data += SHA1_BLOCK_SIZE; + } while (--nblocks); + + memzero_explicit(workspace, sizeof(workspace)); +} + +#ifdef CONFIG_CRYPTO_LIB_SHA1_ARCH +#include "sha1.h" /* $(SRCARCH)/sha1.h */ +#else +#define sha1_blocks sha1_blocks_generic +#endif + +void sha1_init(struct sha1_ctx *ctx) +{ + ctx->state = sha1_iv; + ctx->bytecount = 0; +} +EXPORT_SYMBOL_GPL(sha1_init); + +void sha1_update(struct sha1_ctx *ctx, const u8 *data, size_t len) +{ + size_t partial = ctx->bytecount % SHA1_BLOCK_SIZE; + + ctx->bytecount += len; + + if (partial + len >= SHA1_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l = SHA1_BLOCK_SIZE - partial; + + memcpy(&ctx->buf[partial], data, l); + data += l; + len -= l; + + sha1_blocks(&ctx->state, ctx->buf, 1); + } + + nblocks = len / SHA1_BLOCK_SIZE; + len %= SHA1_BLOCK_SIZE; + + if (nblocks) { + sha1_blocks(&ctx->state, data, nblocks); + data += nblocks * SHA1_BLOCK_SIZE; + } + partial = 0; + } + if (len) + memcpy(&ctx->buf[partial], data, len); +} +EXPORT_SYMBOL_GPL(sha1_update); + +void sha1_final(struct sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]) +{ + u64 bitcount = ctx->bytecount << 3; + size_t partial = ctx->bytecount % SHA1_BLOCK_SIZE; + + ctx->buf[partial++] = 0x80; + if (partial > SHA1_BLOCK_SIZE - 8) { + memset(&ctx->buf[partial], 0, SHA1_BLOCK_SIZE - partial); + sha1_blocks(&ctx->state, ctx->buf, 1); + partial = 0; + } + memset(&ctx->buf[partial], 0, SHA1_BLOCK_SIZE - 8 - partial); + *(__be64 *)&ctx->buf[SHA1_BLOCK_SIZE - 8] = cpu_to_be64(bitcount); + sha1_blocks(&ctx->state, ctx->buf, 1); + + for (size_t i = 0; i < SHA1_DIGEST_SIZE; i += 4) + put_unaligned_be32(ctx->state.h[i / 4], out + i); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sha1_final); + +void sha1(const u8 *data, size_t len, u8 out[SHA1_DIGEST_SIZE]) +{ + struct sha1_ctx ctx; + + sha1_init(&ctx); + sha1_update(&ctx, data, len); + sha1_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sha1); + +#ifdef sha1_mod_init_arch +static int __init sha1_mod_init(void) +{ + sha1_mod_init_arch(); + return 0; +} +subsys_initcall(sha1_mod_init); + +static void __exit sha1_mod_exit(void) +{ +} +module_exit(sha1_mod_exit); +#endif + +MODULE_DESCRIPTION("SHA-1 library functions"); MODULE_LICENSE("GPL"); From 4cbc84471bb606ddfaf424709dd8d56b56d7ae7b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:22:55 -0700 Subject: [PATCH 54/67] lib/crypto: sha1: Add HMAC support Add HMAC support to the SHA-1 library, again following what was done for SHA-2. Besides providing the basis for a more streamlined "hmac(sha1)" shash, this will also be useful for multiple in-kernel users such as net/sctp/auth.c, net/ipv6/seg6_hmac.c, and security/keys/trusted-keys/trusted_tpm1.c. Those are currently using crypto_shash, but using the library functions would be much simpler. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha1.h | 118 ++++++++++++++++++++++++++++++++++++++++++ lib/crypto/sha1.c | 108 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 223 insertions(+), 3 deletions(-) diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h index 387f6123a05e..162a529ec841 100644 --- a/include/crypto/sha1.h +++ b/include/crypto/sha1.h @@ -96,4 +96,122 @@ void sha1_final(struct sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]); */ void sha1(const u8 *data, size_t len, u8 out[SHA1_DIGEST_SIZE]); +/** + * struct hmac_sha1_key - Prepared key for HMAC-SHA1 + * @istate: private + * @ostate: private + */ +struct hmac_sha1_key { + struct sha1_block_state istate; + struct sha1_block_state ostate; +}; + +/** + * struct hmac_sha1_ctx - Context for computing HMAC-SHA1 of a message + * @sha_ctx: private + * @ostate: private + */ +struct hmac_sha1_ctx { + struct sha1_ctx sha_ctx; + struct sha1_block_state ostate; +}; + +/** + * hmac_sha1_preparekey() - Prepare a key for HMAC-SHA1 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA1 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha1_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha1_preparekey(struct hmac_sha1_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha1_init() - Initialize an HMAC-SHA1 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha1() instead. + * + * Context: Any context. + */ +void hmac_sha1_init(struct hmac_sha1_ctx *ctx, const struct hmac_sha1_key *key); + +/** + * hmac_sha1_init_usingrawkey() - Initialize an HMAC-SHA1 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA1 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha1_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha1_init_usingrawkey(struct hmac_sha1_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha1_update() - Update an HMAC-SHA1 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha1_update(struct hmac_sha1_ctx *ctx, + const u8 *data, size_t data_len) +{ + sha1_update(&ctx->sha_ctx, data, data_len); +} + +/** + * hmac_sha1_final() - Finish computing an HMAC-SHA1 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA1 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha1_final(struct hmac_sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]); + +/** + * hmac_sha1() - Compute HMAC-SHA1 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA1 value + * + * If you're using the key only once, consider using hmac_sha1_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha1(const struct hmac_sha1_key *key, + const u8 *data, size_t data_len, u8 out[SHA1_DIGEST_SIZE]); + +/** + * hmac_sha1_usingrawkey() - Compute HMAC-SHA1 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA1 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA1 value + * + * If you're using the key multiple times, prefer to use hmac_sha1_preparekey() + * followed by multiple calls to hmac_sha1() instead. + * + * Context: Any context. + */ +void hmac_sha1_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA1_DIGEST_SIZE]); + #endif /* _CRYPTO_SHA1_H */ diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index 0fe9ca8d0653..5904e4ae85d2 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -1,8 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 /* - * SHA-1 library functions + * SHA-1 and HMAC-SHA1 library functions */ +#include #include #include #include @@ -10,6 +11,7 @@ #include #include #include +#include static const struct sha1_block_state sha1_iv = { .h = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, @@ -197,7 +199,7 @@ void sha1_update(struct sha1_ctx *ctx, const u8 *data, size_t len) } EXPORT_SYMBOL_GPL(sha1_update); -void sha1_final(struct sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]) +static void __sha1_final(struct sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]) { u64 bitcount = ctx->bytecount << 3; size_t partial = ctx->bytecount % SHA1_BLOCK_SIZE; @@ -214,6 +216,11 @@ void sha1_final(struct sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]) for (size_t i = 0; i < SHA1_DIGEST_SIZE; i += 4) put_unaligned_be32(ctx->state.h[i / 4], out + i); +} + +void sha1_final(struct sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]) +{ + __sha1_final(ctx, out); memzero_explicit(ctx, sizeof(*ctx)); } EXPORT_SYMBOL_GPL(sha1_final); @@ -228,6 +235,101 @@ void sha1(const u8 *data, size_t len, u8 out[SHA1_DIGEST_SIZE]) } EXPORT_SYMBOL_GPL(sha1); +static void __hmac_sha1_preparekey(struct sha1_block_state *istate, + struct sha1_block_state *ostate, + const u8 *raw_key, size_t raw_key_len) +{ + union { + u8 b[SHA1_BLOCK_SIZE]; + unsigned long w[SHA1_BLOCK_SIZE / sizeof(unsigned long)]; + } derived_key = { 0 }; + + if (unlikely(raw_key_len > SHA1_BLOCK_SIZE)) + sha1(raw_key, raw_key_len, derived_key.b); + else + memcpy(derived_key.b, raw_key, raw_key_len); + + for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) + derived_key.w[i] ^= REPEAT_BYTE(HMAC_IPAD_VALUE); + *istate = sha1_iv; + sha1_blocks(istate, derived_key.b, 1); + + for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++) + derived_key.w[i] ^= REPEAT_BYTE(HMAC_OPAD_VALUE ^ + HMAC_IPAD_VALUE); + *ostate = sha1_iv; + sha1_blocks(ostate, derived_key.b, 1); + + memzero_explicit(&derived_key, sizeof(derived_key)); +} + +void hmac_sha1_preparekey(struct hmac_sha1_key *key, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha1_preparekey(&key->istate, &key->ostate, + raw_key, raw_key_len); +} +EXPORT_SYMBOL_GPL(hmac_sha1_preparekey); + +void hmac_sha1_init(struct hmac_sha1_ctx *ctx, const struct hmac_sha1_key *key) +{ + ctx->sha_ctx.state = key->istate; + ctx->sha_ctx.bytecount = SHA1_BLOCK_SIZE; + ctx->ostate = key->ostate; +} +EXPORT_SYMBOL_GPL(hmac_sha1_init); + +void hmac_sha1_init_usingrawkey(struct hmac_sha1_ctx *ctx, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha1_preparekey(&ctx->sha_ctx.state, &ctx->ostate, + raw_key, raw_key_len); + ctx->sha_ctx.bytecount = SHA1_BLOCK_SIZE; +} +EXPORT_SYMBOL_GPL(hmac_sha1_init_usingrawkey); + +void hmac_sha1_final(struct hmac_sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE]) +{ + /* Generate the padded input for the outer hash in ctx->sha_ctx.buf. */ + __sha1_final(&ctx->sha_ctx, ctx->sha_ctx.buf); + memset(&ctx->sha_ctx.buf[SHA1_DIGEST_SIZE], 0, + SHA1_BLOCK_SIZE - SHA1_DIGEST_SIZE); + ctx->sha_ctx.buf[SHA1_DIGEST_SIZE] = 0x80; + *(__be32 *)&ctx->sha_ctx.buf[SHA1_BLOCK_SIZE - 4] = + cpu_to_be32(8 * (SHA1_BLOCK_SIZE + SHA1_DIGEST_SIZE)); + + /* Compute the outer hash, which gives the HMAC value. */ + sha1_blocks(&ctx->ostate, ctx->sha_ctx.buf, 1); + for (size_t i = 0; i < SHA1_DIGEST_SIZE; i += 4) + put_unaligned_be32(ctx->ostate.h[i / 4], out + i); + + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(hmac_sha1_final); + +void hmac_sha1(const struct hmac_sha1_key *key, + const u8 *data, size_t data_len, u8 out[SHA1_DIGEST_SIZE]) +{ + struct hmac_sha1_ctx ctx; + + hmac_sha1_init(&ctx, key); + hmac_sha1_update(&ctx, data, data_len); + hmac_sha1_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(hmac_sha1); + +void hmac_sha1_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA1_DIGEST_SIZE]) +{ + struct hmac_sha1_ctx ctx; + + hmac_sha1_init_usingrawkey(&ctx, raw_key, raw_key_len); + hmac_sha1_update(&ctx, data, data_len); + hmac_sha1_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(hmac_sha1_usingrawkey); + #ifdef sha1_mod_init_arch static int __init sha1_mod_init(void) { @@ -242,5 +344,5 @@ static void __exit sha1_mod_exit(void) module_exit(sha1_mod_exit); #endif -MODULE_DESCRIPTION("SHA-1 library functions"); +MODULE_DESCRIPTION("SHA-1 and HMAC-SHA1 library functions"); MODULE_LICENSE("GPL"); From 8bc79ab67d78e2991b9d6cf0b63789189212375a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:22:56 -0700 Subject: [PATCH 55/67] crypto: sha1 - Wrap library and add HMAC support Like I did for crypto/sha512.c, rework crypto/sha1_generic.c (renamed to crypto/sha1.c) to simply wrap the normal library functions instead of accessing the low-level block function directly. Also add support for HMAC-SHA1, again just wrapping the library functions. Since the replacement crypto_shash algorithms are implemented using the (potentially arch-optimized) library functions, give them driver names ending with "-lib" rather than "-generic". Update crypto/testmgr.c and an odd driver to take this change in driver name into account. Note: to see the diff from crypto/sha1_generic.c to crypto/sha1.c, view this commit with 'git show -M10'. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 3 +- crypto/Makefile | 2 +- crypto/sha1.c | 135 ++++++++++++++++++++++++++++++++++++++ crypto/sha1_generic.c | 87 ------------------------ crypto/testmgr.c | 6 ++ drivers/crypto/img-hash.c | 2 +- 6 files changed, 145 insertions(+), 90 deletions(-) create mode 100644 crypto/sha1.c delete mode 100644 crypto/sha1_generic.c diff --git a/crypto/Kconfig b/crypto/Kconfig index 5d4cf022c577..23bd98981ae8 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -986,7 +986,8 @@ config CRYPTO_SHA1 select CRYPTO_HASH select CRYPTO_LIB_SHA1 help - SHA-1 secure hash algorithm (FIPS 180, ISO/IEC 10118-3) + SHA-1 secure hash algorithm (FIPS 180, ISO/IEC 10118-3), including + HMAC support. config CRYPTO_SHA256 tristate "SHA-224 and SHA-256" diff --git a/crypto/Makefile b/crypto/Makefile index 816607e0e78c..9110f708e5a9 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -75,7 +75,7 @@ obj-$(CONFIG_CRYPTO_NULL) += crypto_null.o obj-$(CONFIG_CRYPTO_MD4) += md4.o obj-$(CONFIG_CRYPTO_MD5) += md5.o obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o -obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o +obj-$(CONFIG_CRYPTO_SHA1) += sha1.o obj-$(CONFIG_CRYPTO_SHA256) += sha256.o obj-$(CONFIG_CRYPTO_SHA512) += sha512.o obj-$(CONFIG_CRYPTO_SHA3) += sha3_generic.o diff --git a/crypto/sha1.c b/crypto/sha1.c new file mode 100644 index 000000000000..00e273b0401d --- /dev/null +++ b/crypto/sha1.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Crypto API support for SHA-1 and HMAC-SHA1 + * + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald + * Copyright (c) Jean-Francois Dive + * Copyright 2025 Google LLC + */ +#include +#include +#include +#include + +const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = { + 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, + 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, + 0xaf, 0xd8, 0x07, 0x09 +}; +EXPORT_SYMBOL_GPL(sha1_zero_message_hash); + +#define SHA1_CTX(desc) ((struct sha1_ctx *)shash_desc_ctx(desc)) + +static int crypto_sha1_init(struct shash_desc *desc) +{ + sha1_init(SHA1_CTX(desc)); + return 0; +} + +static int crypto_sha1_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sha1_update(SHA1_CTX(desc), data, len); + return 0; +} + +static int crypto_sha1_final(struct shash_desc *desc, u8 *out) +{ + sha1_final(SHA1_CTX(desc), out); + return 0; +} + +static int crypto_sha1_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sha1(data, len, out); + return 0; +} + +#define HMAC_SHA1_KEY(tfm) ((struct hmac_sha1_key *)crypto_shash_ctx(tfm)) +#define HMAC_SHA1_CTX(desc) ((struct hmac_sha1_ctx *)shash_desc_ctx(desc)) + +static int crypto_hmac_sha1_setkey(struct crypto_shash *tfm, + const u8 *raw_key, unsigned int keylen) +{ + hmac_sha1_preparekey(HMAC_SHA1_KEY(tfm), raw_key, keylen); + return 0; +} + +static int crypto_hmac_sha1_init(struct shash_desc *desc) +{ + hmac_sha1_init(HMAC_SHA1_CTX(desc), HMAC_SHA1_KEY(desc->tfm)); + return 0; +} + +static int crypto_hmac_sha1_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + hmac_sha1_update(HMAC_SHA1_CTX(desc), data, len); + return 0; +} + +static int crypto_hmac_sha1_final(struct shash_desc *desc, u8 *out) +{ + hmac_sha1_final(HMAC_SHA1_CTX(desc), out); + return 0; +} + +static int crypto_hmac_sha1_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + hmac_sha1(HMAC_SHA1_KEY(desc->tfm), data, len, out); + return 0; +} + +static struct shash_alg algs[] = { + { + .base.cra_name = "sha1", + .base.cra_driver_name = "sha1-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SHA1_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SHA1_DIGEST_SIZE, + .init = crypto_sha1_init, + .update = crypto_sha1_update, + .final = crypto_sha1_final, + .digest = crypto_sha1_digest, + .descsize = sizeof(struct sha1_ctx), + }, + { + .base.cra_name = "hmac(sha1)", + .base.cra_driver_name = "hmac-sha1-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SHA1_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct hmac_sha1_key), + .base.cra_module = THIS_MODULE, + .digestsize = SHA1_DIGEST_SIZE, + .setkey = crypto_hmac_sha1_setkey, + .init = crypto_hmac_sha1_init, + .update = crypto_hmac_sha1_update, + .final = crypto_hmac_sha1_final, + .digest = crypto_hmac_sha1_digest, + .descsize = sizeof(struct hmac_sha1_ctx), + }, +}; + +static int __init crypto_sha1_mod_init(void) +{ + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} +module_init(crypto_sha1_mod_init); + +static void __exit crypto_sha1_mod_exit(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} +module_exit(crypto_sha1_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Crypto API support for SHA-1 and HMAC-SHA1"); + +MODULE_ALIAS_CRYPTO("sha1"); +MODULE_ALIAS_CRYPTO("sha1-lib"); +MODULE_ALIAS_CRYPTO("hmac(sha1)"); +MODULE_ALIAS_CRYPTO("hmac-sha1-lib"); diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c deleted file mode 100644 index 024e8043bab0..000000000000 --- a/crypto/sha1_generic.c +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * SHA1 Secure Hash Algorithm. - * - * Derived from cryptoapi implementation, adapted for in-place - * scatterlist interface. - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - */ -#include -#include -#include -#include -#include -#include - -const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = { - 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, - 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, - 0xaf, 0xd8, 0x07, 0x09 -}; -EXPORT_SYMBOL_GPL(sha1_zero_message_hash); - -static void sha1_generic_block_fn(struct sha1_state *sst, u8 const *src, - int blocks) -{ - u32 temp[SHA1_WORKSPACE_WORDS]; - - while (blocks--) { - sha1_transform(sst->state, src, temp); - src += SHA1_BLOCK_SIZE; - } - memzero_explicit(temp, sizeof(temp)); -} - -static int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha1_base_do_update_blocks(desc, data, len, - sha1_generic_block_fn); -} - -static int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha1_base_do_finup(desc, data, len, sha1_generic_block_fn); - return sha1_base_finish(desc, out); -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = crypto_sha1_update, - .finup = crypto_sha1_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha1_generic_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit sha1_generic_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(sha1_generic_mod_init); -module_exit(sha1_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm"); - -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_ALIAS_CRYPTO("sha1-generic"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 4e95567f7ed1..be78e3930769 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4237,6 +4237,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "authenc(hmac(sha1),cbc(aes))", + .generic_driver = "authenc(hmac-sha1-lib,cbc(aes-generic))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -4244,12 +4245,14 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "authenc(hmac(sha1),cbc(des))", + .generic_driver = "authenc(hmac-sha1-lib,cbc(des-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha1_des_cbc_tv_temp) } }, { .alg = "authenc(hmac(sha1),cbc(des3_ede))", + .generic_driver = "authenc(hmac-sha1-lib,cbc(des3_ede-generic))", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp) @@ -4260,6 +4263,7 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, }, { .alg = "authenc(hmac(sha1),ecb(cipher_null))", + .generic_driver = "authenc(hmac-sha1-lib,ecb-cipher_null)", .test = alg_test_aead, .suite = { .aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp) @@ -5122,6 +5126,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "hmac(sha1)", + .generic_driver = "hmac-sha1-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { @@ -5462,6 +5467,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "sha1", + .generic_driver = "sha1-lib", .test = alg_test_hash, .fips_allowed = 1, .suite = { diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index f312eb075fec..a8f735390f0d 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -705,7 +705,7 @@ static int img_hash_cra_md5_init(struct crypto_tfm *tfm) static int img_hash_cra_sha1_init(struct crypto_tfm *tfm) { - return img_hash_cra_init(tfm, "sha1-generic"); + return img_hash_cra_init(tfm, "sha1-lib"); } static int img_hash_cra_sha224_init(struct crypto_tfm *tfm) From b10a74abcfc5c61afc63a567c457038be57eeb6e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:22:57 -0700 Subject: [PATCH 56/67] crypto: sha1 - Use same state format as legacy drivers Same as sha256 and sha512: Use the state format that the generic partial block handling code produces, as requested by Herbert, even though this is applicable only to legacy drivers. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-7-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/sha1.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/crypto/sha1.c b/crypto/sha1.c index 00e273b0401d..ecef4bf2d9c0 100644 --- a/crypto/sha1.c +++ b/crypto/sha1.c @@ -12,6 +12,43 @@ #include #include +/* + * Export and import functions. crypto_shash wants a particular format that + * matches that used by some legacy drivers. It currently is the same as the + * library SHA context, except the value in bytecount must be block-aligned and + * the remainder must be stored in an extra u8 appended to the struct. + */ + +#define SHA1_SHASH_STATE_SIZE (sizeof(struct sha1_ctx) + 1) +static_assert(sizeof(struct sha1_ctx) == sizeof(struct sha1_state)); +static_assert(offsetof(struct sha1_ctx, state) == offsetof(struct sha1_state, state)); +static_assert(offsetof(struct sha1_ctx, bytecount) == offsetof(struct sha1_state, count)); +static_assert(offsetof(struct sha1_ctx, buf) == offsetof(struct sha1_state, buffer)); + +static int __crypto_sha1_export(const struct sha1_ctx *ctx0, void *out) +{ + struct sha1_ctx ctx = *ctx0; + unsigned int partial; + u8 *p = out; + + partial = ctx.bytecount % SHA1_BLOCK_SIZE; + ctx.bytecount -= partial; + memcpy(p, &ctx, sizeof(ctx)); + p += sizeof(ctx); + *p = partial; + return 0; +} + +static int __crypto_sha1_import(struct sha1_ctx *ctx, const void *in) +{ + const u8 *p = in; + + memcpy(ctx, p, sizeof(*ctx)); + p += sizeof(*ctx); + ctx->bytecount += *p; + return 0; +} + const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = { 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, @@ -47,6 +84,16 @@ static int crypto_sha1_digest(struct shash_desc *desc, return 0; } +static int crypto_sha1_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha1_export(SHA1_CTX(desc), out); +} + +static int crypto_sha1_import(struct shash_desc *desc, const void *in) +{ + return __crypto_sha1_import(SHA1_CTX(desc), in); +} + #define HMAC_SHA1_KEY(tfm) ((struct hmac_sha1_key *)crypto_shash_ctx(tfm)) #define HMAC_SHA1_CTX(desc) ((struct hmac_sha1_ctx *)shash_desc_ctx(desc)) @@ -83,6 +130,19 @@ static int crypto_hmac_sha1_digest(struct shash_desc *desc, return 0; } +static int crypto_hmac_sha1_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha1_export(&HMAC_SHA1_CTX(desc)->sha_ctx, out); +} + +static int crypto_hmac_sha1_import(struct shash_desc *desc, const void *in) +{ + struct hmac_sha1_ctx *ctx = HMAC_SHA1_CTX(desc); + + ctx->ostate = HMAC_SHA1_KEY(desc->tfm)->ostate; + return __crypto_sha1_import(&ctx->sha_ctx, in); +} + static struct shash_alg algs[] = { { .base.cra_name = "sha1", @@ -95,7 +155,10 @@ static struct shash_alg algs[] = { .update = crypto_sha1_update, .final = crypto_sha1_final, .digest = crypto_sha1_digest, + .export = crypto_sha1_export, + .import = crypto_sha1_import, .descsize = sizeof(struct sha1_ctx), + .statesize = SHA1_SHASH_STATE_SIZE, }, { .base.cra_name = "hmac(sha1)", @@ -110,7 +173,10 @@ static struct shash_alg algs[] = { .update = crypto_hmac_sha1_update, .final = crypto_hmac_sha1_final, .digest = crypto_hmac_sha1_digest, + .export = crypto_hmac_sha1_export, + .import = crypto_hmac_sha1_import, .descsize = sizeof(struct hmac_sha1_ctx), + .statesize = SHA1_SHASH_STATE_SIZE, }, }; From 70cb6ca58fddb02e269fe743ba75d53d577b5b1c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:22:58 -0700 Subject: [PATCH 57/67] lib/crypto: arm/sha1: Migrate optimized code into library Instead of exposing the arm-optimized SHA-1 code via arm-specific crypto_shash algorithms, instead just implement the sha1_blocks() library function. This is much simpler, it makes the SHA-1 library functions be arm-optimized, and it fixes the longstanding issue where the arm-optimized SHA-1 code was disabled by default. SHA-1 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha1_blocks(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-8-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm/configs/exynos_defconfig | 1 - arch/arm/configs/milbeaut_m10v_defconfig | 2 - arch/arm/configs/multi_v7_defconfig | 2 - arch/arm/configs/omap2plus_defconfig | 1 - arch/arm/configs/pxa_defconfig | 1 - arch/arm/crypto/Kconfig | 31 ------- arch/arm/crypto/Makefile | 6 -- arch/arm/crypto/sha1-ce-glue.c | 72 ---------------- arch/arm/crypto/sha1_glue.c | 75 ----------------- arch/arm/crypto/sha1_neon_glue.c | 83 ------------------- lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 5 ++ .../crypto/arm}/sha1-armv4-large.S | 0 .../crypto/arm}/sha1-armv7-neon.S | 13 ++- .../crypto => lib/crypto/arm}/sha1-ce-core.S | 4 +- lib/crypto/arm/sha1.h | 46 ++++++++++ 16 files changed, 60 insertions(+), 283 deletions(-) delete mode 100644 arch/arm/crypto/sha1-ce-glue.c delete mode 100644 arch/arm/crypto/sha1_glue.c delete mode 100644 arch/arm/crypto/sha1_neon_glue.c rename {arch/arm/crypto => lib/crypto/arm}/sha1-armv4-large.S (100%) rename {arch/arm/crypto => lib/crypto/arm}/sha1-armv7-neon.S (98%) rename {arch/arm/crypto => lib/crypto/arm}/sha1-ce-core.S (96%) create mode 100644 lib/crypto/arm/sha1.h diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index d58e30069304..6915c766923a 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -363,7 +363,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_CHACHA20_NEON=m CONFIG_CRYPTO_DEV_EXYNOS_RNG=y diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 8ebf8bd872fe..a3be0b2ede09 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -98,8 +98,6 @@ CONFIG_CRYPTO_SELFTESTS=y CONFIG_CRYPTO_AES=y CONFIG_CRYPTO_SEQIV=m CONFIG_CRYPTO_GHASH_ARM_CE=m -CONFIG_CRYPTO_SHA1_ARM_NEON=m -CONFIG_CRYPTO_SHA1_ARM_CE=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_AES_ARM_CE=m diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index 3fd07e864ca8..fb63f487a623 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -1280,8 +1280,6 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_GHASH_ARM_CE=m -CONFIG_CRYPTO_SHA1_ARM_NEON=m -CONFIG_CRYPTO_SHA1_ARM_CE=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_AES_ARM_CE=m diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index 530dfb8338c9..046467637901 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -704,7 +704,6 @@ CONFIG_NLS_ISO8859_1=y CONFIG_SECURITY=y CONFIG_CRYPTO_MICHAEL_MIC=y CONFIG_CRYPTO_GHASH_ARM_CE=m -CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_AES_ARM=m CONFIG_CRYPTO_AES_ARM_BS=m CONFIG_CRYPTO_CHACHA20_NEON=m diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index eaa44574d4a6..1a80602c1284 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -658,7 +658,6 @@ CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_DEFLATE=y CONFIG_CRYPTO_LZO=y -CONFIG_CRYPTO_SHA1_ARM=m CONFIG_CRYPTO_AES_ARM=m CONFIG_FONTS=y CONFIG_FONT_8x8=y diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index a18f97f1597c..1e5f3cdf691c 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -62,37 +62,6 @@ config CRYPTO_BLAKE2B_NEON much faster than the SHA-2 family and slightly faster than SHA-1. -config CRYPTO_SHA1_ARM - tristate "Hash functions: SHA-1" - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: arm - -config CRYPTO_SHA1_ARM_NEON - tristate "Hash functions: SHA-1 (NEON)" - depends on KERNEL_MODE_NEON - select CRYPTO_SHA1_ARM - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: arm using - - NEON (Advanced SIMD) extensions - -config CRYPTO_SHA1_ARM_CE - tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_SHA1_ARM - select CRYPTO_HASH - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: arm using ARMv8 Crypto Extensions - config CRYPTO_AES_ARM tristate "Ciphers: AES" select CRYPTO_ALGAPI diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 78a4042d8761..4f23999ae17d 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -5,22 +5,16 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o -obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o -obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o obj-$(CONFIG_CRYPTO_CURVE25519_NEON) += curve25519-neon.o obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o -obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o aes-arm-y := aes-cipher-core.o aes-cipher-glue.o aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o -sha1-arm-y := sha1-armv4-large.o sha1_glue.o -sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o -sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c deleted file mode 100644 index fac07a4799de..000000000000 --- a/arch/arm/crypto/sha1-ce-glue.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions - * - * Copyright (C) 2015 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); - -asmlinkage void sha1_ce_transform(struct sha1_state *sst, u8 const *src, - int blocks); - -static int sha1_ce_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - kernel_neon_begin(); - remain = sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); - kernel_neon_end(); - - return remain; -} - -static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_neon_begin(); - sha1_base_do_finup(desc, data, len, sha1_ce_transform); - kernel_neon_end(); - - return sha1_base_finish(desc, out); -} - -static struct shash_alg alg = { - .init = sha1_base_init, - .update = sha1_ce_update, - .finup = sha1_ce_finup, - .descsize = SHA1_STATE_SIZE, - .digestsize = SHA1_DIGEST_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-ce", - .cra_priority = 200, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha1_ce_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit sha1_ce_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_cpu_feature_match(SHA1, sha1_ce_mod_init); -module_exit(sha1_ce_mod_fini); diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c deleted file mode 100644 index 255da00c7d98..000000000000 --- a/arch/arm/crypto/sha1_glue.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * Glue code for the SHA1 Secure Hash Algorithm assembler implementation - * - * This file is based on sha1_generic.c and sha1_ssse3_glue.c - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - * Copyright (c) Mathias Krause - */ - -#include -#include -#include -#include -#include - -asmlinkage void sha1_block_data_order(struct sha1_state *digest, - const u8 *data, int rounds); - -static int sha1_update_arm(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - /* make sure signature matches sha1_block_fn() */ - BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); - - return sha1_base_do_update_blocks(desc, data, len, - sha1_block_data_order); -} - -static int sha1_finup_arm(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha1_base_do_finup(desc, data, len, sha1_block_data_order); - return sha1_base_finish(desc, out); -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = sha1_update_arm, - .finup = sha1_finup_arm, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-asm", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - - -static int __init sha1_mod_init(void) -{ - return crypto_register_shash(&alg); -} - - -static void __exit sha1_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - - -module_init(sha1_mod_init); -module_exit(sha1_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm (ARM)"); -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_AUTHOR("David McCullough "); diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c deleted file mode 100644 index d321850f22a6..000000000000 --- a/arch/arm/crypto/sha1_neon_glue.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using - * ARM NEON instructions. - * - * Copyright © 2014 Jussi Kivilinna - * - * This file is based on sha1_generic.c and sha1_ssse3_glue.c: - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - * Copyright (c) Mathias Krause - * Copyright (c) Chandramouli Narayanan - */ - -#include -#include -#include -#include -#include -#include - -asmlinkage void sha1_transform_neon(struct sha1_state *state_h, - const u8 *data, int rounds); - -static int sha1_neon_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - kernel_neon_begin(); - remain = sha1_base_do_update_blocks(desc, data, len, - sha1_transform_neon); - kernel_neon_end(); - - return remain; -} - -static int sha1_neon_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_neon_begin(); - sha1_base_do_finup(desc, data, len, sha1_transform_neon); - kernel_neon_end(); - - return sha1_base_finish(desc, out); -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = sha1_neon_update, - .finup = sha1_neon_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-neon", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha1_neon_mod_init(void) -{ - if (!cpu_has_neon()) - return -ENODEV; - - return crypto_register_shash(&alg); -} - -static void __exit sha1_neon_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(sha1_neon_mod_init); -module_exit(sha1_neon_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, NEON accelerated"); -MODULE_ALIAS_CRYPTO("sha1"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 5aaf484fc9de..519c5d6a050f 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -146,6 +146,7 @@ config CRYPTO_LIB_SHA1 config CRYPTO_LIB_SHA1_ARCH bool depends on CRYPTO_LIB_SHA1 && !UML + default y if ARM config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 0eb0906d693f..699a42133927 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -71,6 +71,11 @@ obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o libsha1-y := sha1.o ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y) CFLAGS_sha1.o += -I$(src)/$(SRCARCH) +ifeq ($(CONFIG_ARM),y) +libsha1-y += arm/sha1-armv4-large.o +libsha1-$(CONFIG_KERNEL_MODE_NEON) += arm/sha1-armv7-neon.o \ + arm/sha1-ce-core.o +endif endif # CONFIG_CRYPTO_LIB_SHA1_ARCH ################################################################################ diff --git a/arch/arm/crypto/sha1-armv4-large.S b/lib/crypto/arm/sha1-armv4-large.S similarity index 100% rename from arch/arm/crypto/sha1-armv4-large.S rename to lib/crypto/arm/sha1-armv4-large.S diff --git a/arch/arm/crypto/sha1-armv7-neon.S b/lib/crypto/arm/sha1-armv7-neon.S similarity index 98% rename from arch/arm/crypto/sha1-armv7-neon.S rename to lib/crypto/arm/sha1-armv7-neon.S index 28d816a6a530..6edba3ab62e8 100644 --- a/arch/arm/crypto/sha1-armv7-neon.S +++ b/lib/crypto/arm/sha1-armv7-neon.S @@ -282,18 +282,17 @@ /* - * Transform nblks*64 bytes (nblks*16 32-bit words) at DATA. + * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA. * - * unsigned int - * sha1_transform_neon (void *ctx, const unsigned char *data, - * unsigned int nblks) + * void sha1_transform_neon(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ .align 3 ENTRY(sha1_transform_neon) /* input: - * r0: ctx, CTX - * r1: data (64*nblks bytes) - * r2: nblks + * r0: state + * r1: data (64*nblocks bytes) + * r2: nblocks */ cmp RNBLKS, #0; diff --git a/arch/arm/crypto/sha1-ce-core.S b/lib/crypto/arm/sha1-ce-core.S similarity index 96% rename from arch/arm/crypto/sha1-ce-core.S rename to lib/crypto/arm/sha1-ce-core.S index 8a702e051738..2de40dd25e47 100644 --- a/arch/arm/crypto/sha1-ce-core.S +++ b/lib/crypto/arm/sha1-ce-core.S @@ -59,8 +59,8 @@ .word 0xca62c1d6, 0xca62c1d6, 0xca62c1d6, 0xca62c1d6 /* - * void sha1_ce_transform(struct sha1_state *sst, u8 const *src, - * int blocks); + * void sha1_ce_transform(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ ENTRY(sha1_ce_transform) /* load round constants */ diff --git a/lib/crypto/arm/sha1.h b/lib/crypto/arm/sha1.h new file mode 100644 index 000000000000..fa1e92419000 --- /dev/null +++ b/lib/crypto/arm/sha1.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-1 optimized for ARM + * + * Copyright 2025 Google LLC + */ +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); + +asmlinkage void sha1_block_data_order(struct sha1_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage void sha1_transform_neon(struct sha1_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage void sha1_ce_transform(struct sha1_block_state *state, + const u8 *data, size_t nblocks); + +static void sha1_blocks(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_neon) && likely(may_use_simd())) { + kernel_neon_begin(); + if (static_branch_likely(&have_ce)) + sha1_ce_transform(state, data, nblocks); + else + sha1_transform_neon(state, data, nblocks); + kernel_neon_end(); + } else { + sha1_block_data_order(state, data, nblocks); + } +} + +#ifdef CONFIG_KERNEL_MODE_NEON +#define sha1_mod_init_arch sha1_mod_init_arch +static inline void sha1_mod_init_arch(void) +{ + if (elf_hwcap & HWCAP_NEON) { + static_branch_enable(&have_neon); + if (elf_hwcap2 & HWCAP2_SHA1) + static_branch_enable(&have_ce); + } +} +#endif /* CONFIG_KERNEL_MODE_NEON */ From 00d549bb89e471b7df550459fcb51ffbded39cbf Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:22:59 -0700 Subject: [PATCH 58/67] lib/crypto: arm64/sha1: Migrate optimized code into library Instead of exposing the arm64-optimized SHA-1 code via arm64-specific crypto_shash algorithms, instead just implement the sha1_blocks() library function. This is much simpler, it makes the SHA-1 library functions be arm64-optimized, and it fixes the longstanding issue where the arm64-optimized SHA-1 code was disabled by default. SHA-1 still remains available through crypto_shash, but individual architectures no longer need to handle it. Remove support for SHA-1 finalization from assembly code, since the library does not yet support architecture-specific overrides of the finalization. (Support for that has been omitted for now, for simplicity and because usually it isn't performance-critical.) To match sha1_blocks(), change the type of the nblocks parameter and the return value of __sha1_ce_transform() from int to size_t. Update the assembly code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-9-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/configs/defconfig | 1 - arch/arm64/crypto/Kconfig | 11 -- arch/arm64/crypto/Makefile | 3 - arch/arm64/crypto/sha1-ce-glue.c | 118 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 1 + .../crypto/arm64}/sha1-ce-core.S | 40 ++---- lib/crypto/arm64/sha1.h | 39 ++++++ 8 files changed, 51 insertions(+), 163 deletions(-) delete mode 100644 arch/arm64/crypto/sha1-ce-glue.c rename {arch/arm64/crypto => lib/crypto/arm64}/sha1-ce-core.S (76%) create mode 100644 lib/crypto/arm64/sha1.h diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index b612b78b3b09..31681206b49c 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1743,7 +1743,6 @@ CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_ANSI_CPRNG=y CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_GHASH_ARM64_CE=y -CONFIG_CRYPTO_SHA1_ARM64_CE=y CONFIG_CRYPTO_SHA3_ARM64=m CONFIG_CRYPTO_SM3_ARM64_CE=m CONFIG_CRYPTO_AES_ARM64_CE_BLK=y diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index a9ead99f72c2..3bb5b513d5ae 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -25,17 +25,6 @@ config CRYPTO_NHPOLY1305_NEON Architecture: arm64 using: - NEON (Advanced SIMD) extensions -config CRYPTO_SHA1_ARM64_CE - tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_SHA1 - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: arm64 using: - - ARMv8 Crypto Extensions - config CRYPTO_SHA3_ARM64 tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index 228101f125d5..a8b2cdbe202c 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -5,9 +5,6 @@ # Copyright (C) 2014 Linaro Ltd # -obj-$(CONFIG_CRYPTO_SHA1_ARM64_CE) += sha1-ce.o -sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o - obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c deleted file mode 100644 index 65b6980817e5..000000000000 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions - * - * Copyright (C) 2014 - 2017 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("sha1"); - -struct sha1_ce_state { - struct sha1_state sst; - u32 finalize; -}; - -extern const u32 sha1_ce_offsetof_count; -extern const u32 sha1_ce_offsetof_finalize; - -asmlinkage int __sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, - int blocks); - -static void sha1_ce_transform(struct sha1_state *sst, u8 const *src, - int blocks) -{ - while (blocks) { - int rem; - - kernel_neon_begin(); - rem = __sha1_ce_transform(container_of(sst, - struct sha1_ce_state, - sst), src, blocks); - kernel_neon_end(); - src += (blocks - rem) * SHA1_BLOCK_SIZE; - blocks = rem; - } -} - -const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count); -const u32 sha1_ce_offsetof_finalize = offsetof(struct sha1_ce_state, finalize); - -static int sha1_ce_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha1_ce_state *sctx = shash_desc_ctx(desc); - - sctx->finalize = 0; - return sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); -} - -static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - struct sha1_ce_state *sctx = shash_desc_ctx(desc); - bool finalized = false; - - /* - * Allow the asm code to perform the finalization if there is no - * partial data and the input is a round multiple of the block size. - */ - if (len >= SHA1_BLOCK_SIZE) { - unsigned int remain = len - round_down(len, SHA1_BLOCK_SIZE); - - finalized = !remain; - sctx->finalize = finalized; - sha1_base_do_update_blocks(desc, data, len, sha1_ce_transform); - data += len - remain; - len = remain; - } - if (!finalized) { - sctx->finalize = 0; - sha1_base_do_finup(desc, data, len, sha1_ce_transform); - } - return sha1_base_finish(desc, out); -} - -static struct shash_alg alg = { - .init = sha1_base_init, - .update = sha1_ce_update, - .finup = sha1_ce_finup, - .descsize = sizeof(struct sha1_ce_state), - .statesize = SHA1_STATE_SIZE, - .digestsize = SHA1_DIGEST_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-ce", - .cra_priority = 200, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha1_ce_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit sha1_ce_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_cpu_feature_match(SHA1, sha1_ce_mod_init); -module_exit(sha1_ce_mod_fini); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 519c5d6a050f..05cce143af31 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -147,6 +147,7 @@ config CRYPTO_LIB_SHA1_ARCH bool depends on CRYPTO_LIB_SHA1 && !UML default y if ARM + default y if ARM64 && KERNEL_MODE_NEON config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 699a42133927..1da13c9e2f71 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -76,6 +76,7 @@ libsha1-y += arm/sha1-armv4-large.o libsha1-$(CONFIG_KERNEL_MODE_NEON) += arm/sha1-armv7-neon.o \ arm/sha1-ce-core.o endif +libsha1-$(CONFIG_ARM64) += arm64/sha1-ce-core.o endif # CONFIG_CRYPTO_LIB_SHA1_ARCH ################################################################################ diff --git a/arch/arm64/crypto/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-core.S similarity index 76% rename from arch/arm64/crypto/sha1-ce-core.S rename to lib/crypto/arm64/sha1-ce-core.S index 9b1f2d82a6fe..21efbbafd7d6 100644 --- a/arch/arm64/crypto/sha1-ce-core.S +++ b/lib/crypto/arm64/sha1-ce-core.S @@ -62,8 +62,8 @@ .endm /* - * int __sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, - * int blocks) + * size_t __sha1_ce_transform(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ SYM_FUNC_START(__sha1_ce_transform) /* load round constants */ @@ -76,20 +76,16 @@ SYM_FUNC_START(__sha1_ce_transform) ld1 {dgav.4s}, [x0] ldr dgb, [x0, #16] - /* load sha1_ce_state::finalize */ - ldr_l w4, sha1_ce_offsetof_finalize, x4 - ldr w4, [x0, x4] - /* load input */ 0: ld1 {v8.4s-v11.4s}, [x1], #64 - sub w2, w2, #1 + sub x2, x2, #1 CPU_LE( rev32 v8.16b, v8.16b ) CPU_LE( rev32 v9.16b, v9.16b ) CPU_LE( rev32 v10.16b, v10.16b ) CPU_LE( rev32 v11.16b, v11.16b ) -1: add t0.4s, v8.4s, k0.4s + add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b add_update c, ev, k0, 8, 9, 10, 11, dgb @@ -120,31 +116,15 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - cbz w2, 2f - cond_yield 3f, x5, x6 - b 0b + /* return early if voluntary preemption is needed */ + cond_yield 1f, x5, x6 - /* - * Final block: add padding and total bit count. - * Skip if the input size was not a round multiple of the block size, - * the padding is handled by the C code in that case. - */ -2: cbz x4, 3f - ldr_l w4, sha1_ce_offsetof_count, x4 - ldr x4, [x0, x4] - movi v9.2d, #0 - mov x8, #0x80000000 - movi v10.2d, #0 - ror x7, x4, #29 // ror(lsl(x4, 3), 32) - fmov d8, x8 - mov x4, #0 - mov v11.d[0], xzr - mov v11.d[1], x7 - b 1b + /* handled all input blocks? */ + cbnz x2, 0b /* store new state */ -3: st1 {dgav.4s}, [x0] +1: st1 {dgav.4s}, [x0] str dgb, [x0, #16] - mov w0, w2 + mov x0, x2 ret SYM_FUNC_END(__sha1_ce_transform) diff --git a/lib/crypto/arm64/sha1.h b/lib/crypto/arm64/sha1.h new file mode 100644 index 000000000000..f822563538cc --- /dev/null +++ b/lib/crypto/arm64/sha1.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-1 optimized for ARM64 + * + * Copyright 2025 Google LLC + */ +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); + +asmlinkage size_t __sha1_ce_transform(struct sha1_block_state *state, + const u8 *data, size_t nblocks); + +static void sha1_blocks(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_ce) && likely(may_use_simd())) { + do { + size_t rem; + + kernel_neon_begin(); + rem = __sha1_ce_transform(state, data, nblocks); + kernel_neon_end(); + data += (nblocks - rem) * SHA1_BLOCK_SIZE; + nblocks = rem; + } while (nblocks); + } else { + sha1_blocks_generic(state, data, nblocks); + } +} + +#define sha1_mod_init_arch sha1_mod_init_arch +static inline void sha1_mod_init_arch(void) +{ + if (cpu_have_named_feature(SHA1)) + static_branch_enable(&have_ce); +} From b6ac1dac2f18d1f9714804654ad0643d5aeef4d5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:23:00 -0700 Subject: [PATCH 59/67] lib/crypto: mips/sha1: Migrate optimized code into library Instead of exposing the mips-optimized SHA-1 code via mips-specific crypto_shash algorithms, instead just implement the sha1_blocks() library function. This is much simpler, it makes the SHA-1 library functions be mips-optimized, and it fixes the longstanding issue where the mips-optimized SHA-1 code was disabled by default. SHA-1 still remains available through crypto_shash, but individual architectures no longer need to handle it. Note: to see the diff from arch/mips/cavium-octeon/crypto/octeon-sha1.c to lib/crypto/mips/sha1.h, view this commit with 'git show -M10'. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-10-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/mips/cavium-octeon/crypto/Makefile | 1 - arch/mips/cavium-octeon/crypto/octeon-sha1.c | 146 ------------------- arch/mips/configs/cavium_octeon_defconfig | 1 - arch/mips/crypto/Kconfig | 10 -- lib/crypto/Kconfig | 1 + lib/crypto/mips/sha1.h | 81 ++++++++++ 6 files changed, 82 insertions(+), 158 deletions(-) delete mode 100644 arch/mips/cavium-octeon/crypto/octeon-sha1.c create mode 100644 lib/crypto/mips/sha1.h diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-octeon/crypto/Makefile index db428e4b30bc..83f2f5dd93cc 100644 --- a/arch/mips/cavium-octeon/crypto/Makefile +++ b/arch/mips/cavium-octeon/crypto/Makefile @@ -6,4 +6,3 @@ obj-y += octeon-crypto.o obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o -obj-$(CONFIG_CRYPTO_SHA1_OCTEON) += octeon-sha1.o diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c deleted file mode 100644 index e4a369a7764f..000000000000 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * SHA1 Secure Hash Algorithm. - * - * Adapted for OCTEON by Aaro Koskinen . - * - * Based on crypto/sha1_generic.c, which is: - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * We pass everything as 64-bit. OCTEON can handle misaligned data. - */ - -static void octeon_sha1_store_hash(struct sha1_state *sctx) -{ - u64 *hash = (u64 *)sctx->state; - union { - u32 word[2]; - u64 dword; - } hash_tail = { { sctx->state[4], } }; - - write_octeon_64bit_hash_dword(hash[0], 0); - write_octeon_64bit_hash_dword(hash[1], 1); - write_octeon_64bit_hash_dword(hash_tail.dword, 2); - memzero_explicit(&hash_tail.word[0], sizeof(hash_tail.word[0])); -} - -static void octeon_sha1_read_hash(struct sha1_state *sctx) -{ - u64 *hash = (u64 *)sctx->state; - union { - u32 word[2]; - u64 dword; - } hash_tail; - - hash[0] = read_octeon_64bit_hash_dword(0); - hash[1] = read_octeon_64bit_hash_dword(1); - hash_tail.dword = read_octeon_64bit_hash_dword(2); - sctx->state[4] = hash_tail.word[0]; - memzero_explicit(&hash_tail.dword, sizeof(hash_tail.dword)); -} - -static void octeon_sha1_transform(struct sha1_state *sctx, const u8 *src, - int blocks) -{ - do { - const u64 *block = (const u64 *)src; - - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_sha1_start(block[7]); - - src += SHA1_BLOCK_SIZE; - } while (--blocks); -} - -static int octeon_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - struct octeon_cop2_state state; - unsigned long flags; - int remain; - - flags = octeon_crypto_enable(&state); - octeon_sha1_store_hash(sctx); - - remain = sha1_base_do_update_blocks(desc, data, len, - octeon_sha1_transform); - - octeon_sha1_read_hash(sctx); - octeon_crypto_disable(&state, flags); - return remain; -} - -static int octeon_sha1_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - struct octeon_cop2_state state; - unsigned long flags; - - flags = octeon_crypto_enable(&state); - octeon_sha1_store_hash(sctx); - - sha1_base_do_finup(desc, src, len, octeon_sha1_transform); - - octeon_sha1_read_hash(sctx); - octeon_crypto_disable(&state, flags); - return sha1_base_finish(desc, out); -} - -static struct shash_alg octeon_sha1_alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = octeon_sha1_update, - .finup = octeon_sha1_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name= "octeon-sha1", - .cra_priority = OCTEON_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init octeon_sha1_mod_init(void) -{ - if (!octeon_has_crypto()) - return -ENOTSUPP; - return crypto_register_shash(&octeon_sha1_alg); -} - -static void __exit octeon_sha1_mod_fini(void) -{ - crypto_unregister_shash(&octeon_sha1_alg); -} - -module_init(octeon_sha1_mod_init); -module_exit(octeon_sha1_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm (OCTEON)"); -MODULE_AUTHOR("Aaro Koskinen "); diff --git a/arch/mips/configs/cavium_octeon_defconfig b/arch/mips/configs/cavium_octeon_defconfig index effdfb2bb738..3f50e1d78894 100644 --- a/arch/mips/configs/cavium_octeon_defconfig +++ b/arch/mips/configs/cavium_octeon_defconfig @@ -156,7 +156,6 @@ CONFIG_SECURITY_NETWORK=y CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5_OCTEON=y -CONFIG_CRYPTO_SHA1_OCTEON=m CONFIG_CRYPTO_DES=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_FS=y diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 51a76a5ee3b1..7b91f4ec65bf 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -12,14 +12,4 @@ config CRYPTO_MD5_OCTEON Architecture: mips OCTEON using crypto instructions, when available -config CRYPTO_SHA1_OCTEON - tristate "Hash functions: SHA-1 (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: mips OCTEON - endmenu diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 05cce143af31..7c9dc432fb42 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -148,6 +148,7 @@ config CRYPTO_LIB_SHA1_ARCH depends on CRYPTO_LIB_SHA1 && !UML default y if ARM default y if ARM64 && KERNEL_MODE_NEON + default y if MIPS && CPU_CAVIUM_OCTEON config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/mips/sha1.h b/lib/crypto/mips/sha1.h new file mode 100644 index 000000000000..ba1965002e4a --- /dev/null +++ b/lib/crypto/mips/sha1.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Cryptographic API. + * + * SHA1 Secure Hash Algorithm. + * + * Adapted for OCTEON by Aaro Koskinen . + * + * Based on crypto/sha1_generic.c, which is: + * + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald + * Copyright (c) Jean-Francois Dive + */ + +#include +#include + +/* + * We pass everything as 64-bit. OCTEON can handle misaligned data. + */ + +static void octeon_sha1_store_hash(struct sha1_block_state *state) +{ + u64 *hash = (u64 *)&state->h[0]; + union { + u32 word[2]; + u64 dword; + } hash_tail = { { state->h[4], } }; + + write_octeon_64bit_hash_dword(hash[0], 0); + write_octeon_64bit_hash_dword(hash[1], 1); + write_octeon_64bit_hash_dword(hash_tail.dword, 2); + memzero_explicit(&hash_tail.word[0], sizeof(hash_tail.word[0])); +} + +static void octeon_sha1_read_hash(struct sha1_block_state *state) +{ + u64 *hash = (u64 *)&state->h[0]; + union { + u32 word[2]; + u64 dword; + } hash_tail; + + hash[0] = read_octeon_64bit_hash_dword(0); + hash[1] = read_octeon_64bit_hash_dword(1); + hash_tail.dword = read_octeon_64bit_hash_dword(2); + state->h[4] = hash_tail.word[0]; + memzero_explicit(&hash_tail.dword, sizeof(hash_tail.dword)); +} + +static void sha1_blocks(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + struct octeon_cop2_state cop2_state; + unsigned long flags; + + if (!octeon_has_crypto()) + return sha1_blocks_generic(state, data, nblocks); + + flags = octeon_crypto_enable(&cop2_state); + octeon_sha1_store_hash(state); + + do { + const u64 *block = (const u64 *)data; + + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_sha1_start(block[7]); + + data += SHA1_BLOCK_SIZE; + } while (--nblocks); + + octeon_sha1_read_hash(state); + octeon_crypto_disable(&cop2_state, flags); +} From 6b9ae8cfaa7abc65f9fc8bd93f0c707c31b7ce85 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:23:01 -0700 Subject: [PATCH 60/67] lib/crypto: powerpc/sha1: Migrate optimized code into library Instead of exposing the powerpc-optimized SHA-1 code via powerpc-specific crypto_shash algorithms, instead just implement the sha1_blocks() library function. This is much simpler, it makes the SHA-1 library functions be powerpc-optimized, and it fixes the longstanding issue where the powerpc-optimized SHA-1 code was disabled by default. SHA-1 still remains available through crypto_shash, but individual architectures no longer need to handle it. Note: to see the diff from arch/powerpc/crypto/sha1-spe-glue.c to lib/crypto/powerpc/sha1.h, view this commit with 'git show -M10'. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-11-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/powerpc/configs/44x/akebono_defconfig | 1 - arch/powerpc/configs/powernv_defconfig | 1 - arch/powerpc/configs/ppc64_defconfig | 1 - arch/powerpc/crypto/Kconfig | 16 --- arch/powerpc/crypto/Makefile | 4 - arch/powerpc/crypto/sha1-spe-glue.c | 107 ------------------ arch/powerpc/crypto/sha1.c | 78 ------------- lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 4 + .../crypto/powerpc}/sha1-powerpc-asm.S | 0 .../crypto/powerpc}/sha1-spe-asm.S | 0 lib/crypto/powerpc/sha1.h | 67 +++++++++++ 12 files changed, 72 insertions(+), 208 deletions(-) delete mode 100644 arch/powerpc/crypto/sha1-spe-glue.c delete mode 100644 arch/powerpc/crypto/sha1.c rename {arch/powerpc/crypto => lib/crypto/powerpc}/sha1-powerpc-asm.S (100%) rename {arch/powerpc/crypto => lib/crypto/powerpc}/sha1-spe-asm.S (100%) create mode 100644 lib/crypto/powerpc/sha1.h diff --git a/arch/powerpc/configs/44x/akebono_defconfig b/arch/powerpc/configs/44x/akebono_defconfig index fde4824f235e..1882eb2da354 100644 --- a/arch/powerpc/configs/44x/akebono_defconfig +++ b/arch/powerpc/configs/44x/akebono_defconfig @@ -128,6 +128,5 @@ CONFIG_PPC_EARLY_DEBUG_44x_PHYSLOW=0x00010000 CONFIG_PPC_EARLY_DEBUG_44x_PHYSHIGH=0x33f CONFIG_CRYPTO_PCBC=y CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_SHA1_PPC=y CONFIG_CRYPTO_DES=y # CONFIG_CRYPTO_HW is not set diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig index 379229c982a4..98f56e63ad21 100644 --- a/arch/powerpc/configs/powernv_defconfig +++ b/arch/powerpc/configs/powernv_defconfig @@ -322,7 +322,6 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5_PPC=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_SHA1_PPC=m CONFIG_CRYPTO_SHA256=y CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 3423c405cad4..dca67aae5da3 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -388,7 +388,6 @@ CONFIG_CRYPTO_SHA256=y CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_MD5_PPC=m -CONFIG_CRYPTO_SHA1_PPC=m CONFIG_CRYPTO_AES_GCM_P10=m CONFIG_CRYPTO_DEV_NX=y CONFIG_CRYPTO_DEV_NX_ENCRYPT=m diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index caaa359f4742..cfe39fc221cf 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -23,22 +23,6 @@ config CRYPTO_MD5_PPC Architecture: powerpc -config CRYPTO_SHA1_PPC - tristate "Hash functions: SHA-1" - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: powerpc - -config CRYPTO_SHA1_PPC_SPE - tristate "Hash functions: SHA-1 (SPE)" - depends on SPE - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: powerpc using - - SPE (Signal Processing Engine) extensions - config CRYPTO_AES_PPC_SPE tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)" depends on SPE diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 8c2936ae466f..bc8fd27344b8 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -7,16 +7,12 @@ obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o obj-$(CONFIG_CRYPTO_MD5_PPC) += md5-ppc.o -obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o -obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o obj-$(CONFIG_CRYPTO_CURVE25519_PPC64) += curve25519-ppc64le.o aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o md5-ppc-y := md5-asm.o md5-glue.o -sha1-powerpc-y := sha1-powerpc-asm.o sha1.o -sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o curve25519-ppc64le-y := curve25519-ppc64le-core.o curve25519-ppc64le_asm.o diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c deleted file mode 100644 index 04c88e173ce1..000000000000 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Glue code for SHA-1 implementation for SPE instructions (PPC) - * - * Based on generic implementation. - * - * Copyright (c) 2015 Markus Stockhausen - */ - -#include -#include -#include -#include -#include -#include -#include - -/* - * MAX_BYTES defines the number of bytes that are allowed to be processed - * between preempt_disable() and preempt_enable(). SHA1 takes ~1000 - * operations per 64 bytes. e500 cores can issue two arithmetic instructions - * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2). - * Thus 2KB of input data will need an estimated maximum of 18,000 cycles. - * Headroom for cache misses included. Even with the low end model clocked - * at 667 MHz this equals to a critical time window of less than 27us. - * - */ -#define MAX_BYTES 2048 - -asmlinkage void ppc_spe_sha1_transform(u32 *state, const u8 *src, u32 blocks); - -static void spe_begin(void) -{ - /* We just start SPE operations and will save SPE registers later. */ - preempt_disable(); - enable_kernel_spe(); -} - -static void spe_end(void) -{ - disable_kernel_spe(); - /* reenable preemption */ - preempt_enable(); -} - -static void ppc_spe_sha1_block(struct sha1_state *sctx, const u8 *src, - int blocks) -{ - do { - int unit = min(blocks, MAX_BYTES / SHA1_BLOCK_SIZE); - - spe_begin(); - ppc_spe_sha1_transform(sctx->state, src, unit); - spe_end(); - - src += unit * SHA1_BLOCK_SIZE; - blocks -= unit; - } while (blocks); -} - -static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha1_base_do_update_blocks(desc, data, len, ppc_spe_sha1_block); -} - -static int ppc_spe_sha1_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - sha1_base_do_finup(desc, src, len, ppc_spe_sha1_block); - return sha1_base_finish(desc, out); -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = ppc_spe_sha1_update, - .finup = ppc_spe_sha1_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-ppc-spe", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init ppc_spe_sha1_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit ppc_spe_sha1_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(ppc_spe_sha1_mod_init); -module_exit(ppc_spe_sha1_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, SPE optimized"); - -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_ALIAS_CRYPTO("sha1-ppc-spe"); diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c deleted file mode 100644 index 4593946aa9b3..000000000000 --- a/arch/powerpc/crypto/sha1.c +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * powerpc implementation of the SHA1 Secure Hash Algorithm. - * - * Derived from cryptoapi implementation, adapted for in-place - * scatterlist interface. - * - * Derived from "crypto/sha1.c" - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - */ -#include -#include -#include -#include -#include - -asmlinkage void powerpc_sha_transform(u32 *state, const u8 *src); - -static void powerpc_sha_block(struct sha1_state *sctx, const u8 *data, - int blocks) -{ - do { - powerpc_sha_transform(sctx->state, data); - data += 64; - } while (--blocks); -} - -static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha1_base_do_update_blocks(desc, data, len, powerpc_sha_block); -} - -/* Add padding and return the message digest. */ -static int powerpc_sha1_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - sha1_base_do_finup(desc, src, len, powerpc_sha_block); - return sha1_base_finish(desc, out); -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = powerpc_sha1_update, - .finup = powerpc_sha1_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-powerpc", - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha1_powerpc_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit sha1_powerpc_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(sha1_powerpc_mod_init); -module_exit(sha1_powerpc_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm"); - -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_ALIAS_CRYPTO("sha1-powerpc"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 7c9dc432fb42..22ffbdab82d6 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -149,6 +149,7 @@ config CRYPTO_LIB_SHA1_ARCH default y if ARM default y if ARM64 && KERNEL_MODE_NEON default y if MIPS && CPU_CAVIUM_OCTEON + default y if PPC config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 1da13c9e2f71..02f672562928 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -77,6 +77,10 @@ libsha1-$(CONFIG_KERNEL_MODE_NEON) += arm/sha1-armv7-neon.o \ arm/sha1-ce-core.o endif libsha1-$(CONFIG_ARM64) += arm64/sha1-ce-core.o +ifeq ($(CONFIG_PPC),y) +libsha1-y += powerpc/sha1-powerpc-asm.o +libsha1-$(CONFIG_SPE) += powerpc/sha1-spe-asm.o +endif endif # CONFIG_CRYPTO_LIB_SHA1_ARCH ################################################################################ diff --git a/arch/powerpc/crypto/sha1-powerpc-asm.S b/lib/crypto/powerpc/sha1-powerpc-asm.S similarity index 100% rename from arch/powerpc/crypto/sha1-powerpc-asm.S rename to lib/crypto/powerpc/sha1-powerpc-asm.S diff --git a/arch/powerpc/crypto/sha1-spe-asm.S b/lib/crypto/powerpc/sha1-spe-asm.S similarity index 100% rename from arch/powerpc/crypto/sha1-spe-asm.S rename to lib/crypto/powerpc/sha1-spe-asm.S diff --git a/lib/crypto/powerpc/sha1.h b/lib/crypto/powerpc/sha1.h new file mode 100644 index 000000000000..e2c010f0370b --- /dev/null +++ b/lib/crypto/powerpc/sha1.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-1 optimized for PowerPC + * + * Copyright (c) 2015 Markus Stockhausen + */ + +#include +#include + +#ifdef CONFIG_SPE +/* + * MAX_BYTES defines the number of bytes that are allowed to be processed + * between preempt_disable() and preempt_enable(). SHA1 takes ~1000 + * operations per 64 bytes. e500 cores can issue two arithmetic instructions + * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2). + * Thus 2KB of input data will need an estimated maximum of 18,000 cycles. + * Headroom for cache misses included. Even with the low end model clocked + * at 667 MHz this equals to a critical time window of less than 27us. + * + */ +#define MAX_BYTES 2048 + +asmlinkage void ppc_spe_sha1_transform(struct sha1_block_state *state, + const u8 *data, u32 nblocks); + +static void spe_begin(void) +{ + /* We just start SPE operations and will save SPE registers later. */ + preempt_disable(); + enable_kernel_spe(); +} + +static void spe_end(void) +{ + disable_kernel_spe(); + /* reenable preemption */ + preempt_enable(); +} + +static void sha1_blocks(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + do { + u32 unit = min_t(size_t, nblocks, MAX_BYTES / SHA1_BLOCK_SIZE); + + spe_begin(); + ppc_spe_sha1_transform(state, data, unit); + spe_end(); + + data += unit * SHA1_BLOCK_SIZE; + nblocks -= unit; + } while (nblocks); +} +#else /* CONFIG_SPE */ +asmlinkage void powerpc_sha_transform(struct sha1_block_state *state, + const u8 data[SHA1_BLOCK_SIZE]); + +static void sha1_blocks(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + do { + powerpc_sha_transform(state, data); + data += SHA1_BLOCK_SIZE; + } while (--nblocks); +} +#endif /* !CONFIG_SPE */ From 377982d5618a7b80bf2ad3eed9aa62691e984d50 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:23:02 -0700 Subject: [PATCH 61/67] lib/crypto: s390/sha1: Migrate optimized code into library Instead of exposing the s390-optimized SHA-1 code via s390-specific crypto_shash algorithms, instead just implement the sha1_blocks() library function. This is much simpler, it makes the SHA-1 library functions be s390-optimized, and it fixes the longstanding issue where the s390-optimized SHA-1 code was disabled by default. SHA-1 still remains available through crypto_shash, but individual architectures no longer need to handle it. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-12-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/s390/configs/debug_defconfig | 1 - arch/s390/configs/defconfig | 1 - arch/s390/crypto/Kconfig | 10 --- arch/s390/crypto/Makefile | 1 - arch/s390/crypto/sha1_s390.c | 103 ------------------------------ lib/crypto/Kconfig | 1 + lib/crypto/s390/sha1.h | 28 ++++++++ 7 files changed, 29 insertions(+), 116 deletions(-) delete mode 100644 arch/s390/crypto/sha1_s390.c create mode 100644 lib/crypto/s390/sha1.h diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index ef313c30b375..a7db7ed28720 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -804,7 +804,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA3_256_S390=m CONFIG_CRYPTO_SHA3_512_S390=m CONFIG_CRYPTO_GHASH_S390=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index b6fa341bb03b..0217ed5616bf 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -791,7 +791,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA3_256_S390=m CONFIG_CRYPTO_SHA3_512_S390=m CONFIG_CRYPTO_GHASH_S390=m diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 4557514fbac3..03f73fbd38b6 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -2,16 +2,6 @@ menu "Accelerated Cryptographic Algorithms for CPU (s390)" -config CRYPTO_SHA1_S390 - tristate "Hash functions: SHA-1" - select CRYPTO_HASH - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: s390 - - It is available as of z990. - config CRYPTO_SHA3_256_S390 tristate "Hash functions: SHA3-224 and SHA3-256" select CRYPTO_HASH diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 473d64c0982a..1e5a1038d491 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -3,7 +3,6 @@ # Cryptographic API # -obj-$(CONFIG_CRYPTO_SHA1_S390) += sha1_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_256_S390) += sha3_256_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_512_S390) += sha3_512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c deleted file mode 100644 index d229cbd2ba22..000000000000 --- a/arch/s390/crypto/sha1_s390.c +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Cryptographic API. - * - * s390 implementation of the SHA1 Secure Hash Algorithm. - * - * Derived from cryptoapi implementation, adapted for in-place - * scatterlist interface. Originally based on the public domain - * implementation written by Steve Reid. - * - * s390 Version: - * Copyright IBM Corp. 2003, 2007 - * Author(s): Thomas Spatzier - * Jan Glauber (jan.glauber@de.ibm.com) - * - * Derived from "crypto/sha1_generic.c" - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - */ -#include -#include -#include -#include -#include -#include - -#include "sha.h" - -static int s390_sha1_init(struct shash_desc *desc) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA1_H0; - sctx->state[1] = SHA1_H1; - sctx->state[2] = SHA1_H2; - sctx->state[3] = SHA1_H3; - sctx->state[4] = SHA1_H4; - sctx->count = 0; - sctx->func = CPACF_KIMD_SHA_1; - - return 0; -} - -static int s390_sha1_export(struct shash_desc *desc, void *out) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - struct sha1_state *octx = out; - - octx->count = sctx->count; - memcpy(octx->state, sctx->state, sizeof(octx->state)); - return 0; -} - -static int s390_sha1_import(struct shash_desc *desc, const void *in) -{ - struct s390_sha_ctx *sctx = shash_desc_ctx(desc); - const struct sha1_state *ictx = in; - - sctx->count = ictx->count; - memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - sctx->func = CPACF_KIMD_SHA_1; - return 0; -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = s390_sha1_init, - .update = s390_sha_update_blocks, - .finup = s390_sha_finup, - .export = s390_sha1_export, - .import = s390_sha1_import, - .descsize = S390_SHA_CTX_SIZE, - .statesize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha1_s390_init(void) -{ - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_1)) - return -ENODEV; - return crypto_register_shash(&alg); -} - -static void __exit sha1_s390_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, sha1_s390_init); -module_exit(sha1_s390_fini); - -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 22ffbdab82d6..7da0432d8164 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -150,6 +150,7 @@ config CRYPTO_LIB_SHA1_ARCH default y if ARM64 && KERNEL_MODE_NEON default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC + default y if S390 config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/s390/sha1.h b/lib/crypto/s390/sha1.h new file mode 100644 index 000000000000..08bd138e881c --- /dev/null +++ b/lib/crypto/s390/sha1.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-1 optimized using the CP Assist for Cryptographic Functions (CPACF) + * + * Copyright 2025 Google LLC + */ +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha1); + +static void sha1_blocks(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_cpacf_sha1)) + cpacf_kimd(CPACF_KIMD_SHA_1, state, data, + nblocks * SHA1_BLOCK_SIZE); + else + sha1_blocks_generic(state, data, nblocks); +} + +#define sha1_mod_init_arch sha1_mod_init_arch +static inline void sha1_mod_init_arch(void) +{ + if (cpu_have_feature(S390_CPU_FEATURE_MSA) && + cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_1)) + static_branch_enable(&have_cpacf_sha1); +} From c751059985e02467c7fa6b14676c1d56d089b3cc Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:23:03 -0700 Subject: [PATCH 62/67] lib/crypto: sparc/sha1: Migrate optimized code into library Instead of exposing the sparc-optimized SHA-1 code via sparc-specific crypto_shash algorithms, instead just implement the sha1_blocks() library function. This is much simpler, it makes the SHA-1 library functions be sparc-optimized, and it fixes the longstanding issue where the sparc-optimized SHA-1 code was disabled by default. SHA-1 still remains available through crypto_shash, but individual architectures no longer need to handle it. Note: to see the diff from arch/sparc/crypto/sha1_glue.c to lib/crypto/sparc/sha1.h, view this commit with 'git show -M10'. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/sparc/crypto/Kconfig | 10 -- arch/sparc/crypto/Makefile | 2 - arch/sparc/crypto/sha1_glue.c | 94 ------------------- lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 1 + lib/crypto/sparc/sha1.h | 43 +++++++++ .../crypto => lib/crypto/sparc}/sha1_asm.S | 0 7 files changed, 45 insertions(+), 106 deletions(-) delete mode 100644 arch/sparc/crypto/sha1_glue.c create mode 100644 lib/crypto/sparc/sha1.h rename {arch/sparc/crypto => lib/crypto/sparc}/sha1_asm.S (100%) diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index 9d8da9aef3a4..f5b2e720fec3 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -26,16 +26,6 @@ config CRYPTO_MD5_SPARC64 Architecture: sparc64 using crypto instructions, when available -config CRYPTO_SHA1_SPARC64 - tristate "Hash functions: SHA-1" - depends on SPARC64 - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: sparc64 - config CRYPTO_AES_SPARC64 tristate "Ciphers: AES, modes: ECB, CBC, CTR" depends on SPARC64 diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile index 99a7e8fd13bc..0d05a17988c4 100644 --- a/arch/sparc/crypto/Makefile +++ b/arch/sparc/crypto/Makefile @@ -3,14 +3,12 @@ # Arch-specific CryptoAPI modules. # -obj-$(CONFIG_CRYPTO_SHA1_SPARC64) += sha1-sparc64.o obj-$(CONFIG_CRYPTO_MD5_SPARC64) += md5-sparc64.o obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o -sha1-sparc64-y := sha1_asm.o sha1_glue.o md5-sparc64-y := md5_asm.o md5_glue.o aes-sparc64-y := aes_asm.o aes_glue.o diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c deleted file mode 100644 index ef19d5023b1b..000000000000 --- a/arch/sparc/crypto/sha1_glue.c +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Glue code for SHA1 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon arch/x86/crypto/sha1_ssse3_glue.c - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - * Copyright (c) Mathias Krause - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha1_sparc64_transform(struct sha1_state *digest, - const u8 *data, int rounds); - -static int sha1_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha1_base_do_update_blocks(desc, data, len, - sha1_sparc64_transform); -} - -/* Add padding and return the message digest. */ -static int sha1_sparc64_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - sha1_base_do_finup(desc, src, len, sha1_sparc64_transform); - return sha1_base_finish(desc, out); -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = sha1_sparc64_update, - .finup = sha1_sparc64_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool __init sparc64_has_sha1_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_SHA1)) - return false; - - return true; -} - -static int __init sha1_sparc64_mod_init(void) -{ - if (sparc64_has_sha1_opcode()) { - pr_info("Using sparc64 sha1 opcode optimized SHA-1 implementation\n"); - return crypto_register_shash(&alg); - } - pr_info("sparc64 sha1 opcode not available.\n"); - return -ENODEV; -} - -static void __exit sha1_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(sha1_sparc64_mod_init); -module_exit(sha1_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, sparc64 sha1 opcode accelerated"); - -MODULE_ALIAS_CRYPTO("sha1"); - -#include "crop_devid.c" diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 7da0432d8164..019034f2bb53 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -151,6 +151,7 @@ config CRYPTO_LIB_SHA1_ARCH default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC default y if S390 + default y if SPARC64 config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 02f672562928..7897da2de623 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -81,6 +81,7 @@ ifeq ($(CONFIG_PPC),y) libsha1-y += powerpc/sha1-powerpc-asm.o libsha1-$(CONFIG_SPE) += powerpc/sha1-spe-asm.o endif +libsha1-$(CONFIG_SPARC) += sparc/sha1_asm.o endif # CONFIG_CRYPTO_LIB_SHA1_ARCH ################################################################################ diff --git a/lib/crypto/sparc/sha1.h b/lib/crypto/sparc/sha1.h new file mode 100644 index 000000000000..5015f93584b7 --- /dev/null +++ b/lib/crypto/sparc/sha1.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * SHA-1 accelerated using the sparc64 crypto opcodes + * + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald + * Copyright (c) Jean-Francois Dive + * Copyright (c) Mathias Krause + */ + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha1_opcodes); + +asmlinkage void sha1_sparc64_transform(struct sha1_block_state *state, + const u8 *data, size_t nblocks); + +static void sha1_blocks(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_sha1_opcodes)) + sha1_sparc64_transform(state, data, nblocks); + else + sha1_blocks_generic(state, data, nblocks); +} + +#define sha1_mod_init_arch sha1_mod_init_arch +static inline void sha1_mod_init_arch(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_SHA1)) + return; + + static_branch_enable(&have_sha1_opcodes); + pr_info("Using sparc64 sha1 opcode optimized SHA-1 implementation\n"); +} diff --git a/arch/sparc/crypto/sha1_asm.S b/lib/crypto/sparc/sha1_asm.S similarity index 100% rename from arch/sparc/crypto/sha1_asm.S rename to lib/crypto/sparc/sha1_asm.S From f3d6cb3dc0394b866bc0d1e15157ce45844cf3d3 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:23:04 -0700 Subject: [PATCH 63/67] lib/crypto: x86/sha1: Migrate optimized code into library Instead of exposing the x86-optimized SHA-1 code via x86-specific crypto_shash algorithms, instead just implement the sha1_blocks() library function. This is much simpler, it makes the SHA-1 library functions be x86-optimized, and it fixes the longstanding issue where the x86-optimized SHA-1 code was disabled by default. SHA-1 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha1_blocks(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-14-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/x86/crypto/Kconfig | 14 - arch/x86/crypto/Makefile | 3 - arch/x86/crypto/sha1_ssse3_glue.c | 324 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 3 + .../crypto/x86/sha1-avx2-asm.S | 7 +- .../crypto/x86/sha1-ni-asm.S | 23 +- .../crypto/x86/sha1-ssse3-and-avx.S | 13 +- lib/crypto/x86/sha1.h | 74 ++++ 9 files changed, 94 insertions(+), 368 deletions(-) delete mode 100644 arch/x86/crypto/sha1_ssse3_glue.c rename arch/x86/crypto/sha1_avx2_x86_64_asm.S => lib/crypto/x86/sha1-avx2-asm.S (98%) rename arch/x86/crypto/sha1_ni_asm.S => lib/crypto/x86/sha1-ni-asm.S (90%) rename arch/x86/crypto/sha1_ssse3_asm.S => lib/crypto/x86/sha1-ssse3-and-avx.S (97%) create mode 100644 lib/crypto/x86/sha1.h diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index eb641a300154..94016c60561e 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -376,20 +376,6 @@ config CRYPTO_POLYVAL_CLMUL_NI Architecture: x86_64 using: - CLMUL-NI (carry-less multiplication new instructions) -config CRYPTO_SHA1_SSSE3 - tristate "Hash functions: SHA-1 (SSSE3/AVX/AVX2/SHA-NI)" - depends on 64BIT - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: x86_64 using: - - SSSE3 (Supplemental SSE3) - - AVX (Advanced Vector Extensions) - - AVX2 (Advanced Vector Extensions 2) - - SHA-NI (SHA Extensions New Instructions) - config CRYPTO_SM3_AVX_X86_64 tristate "Hash functions: SM3 (AVX)" depends on 64BIT diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index d31348be8370..d402963d6b57 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -51,9 +51,6 @@ ifeq ($(CONFIG_AS_VAES)$(CONFIG_AS_VPCLMULQDQ),yy) aesni-intel-$(CONFIG_64BIT) += aes-gcm-avx10-x86_64.o endif -obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o -sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sha1_ssse3_glue.o - obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c deleted file mode 100644 index 826579a7473c..000000000000 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ /dev/null @@ -1,324 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * Glue code for the SHA1 Secure Hash Algorithm assembler implementations - * using SSSE3, AVX, AVX2, and SHA-NI instructions. - * - * This file is based on sha1_generic.c - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - * Copyright (c) Mathias Krause - * Copyright (c) Chandramouli Narayanan - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include - -static const struct x86_cpu_id module_cpu_ids[] = { - X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), - X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), - X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), - X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); - -static inline int sha1_update_x86(struct shash_desc *desc, const u8 *data, - unsigned int len, sha1_block_fn *sha1_xform) -{ - int remain; - - /* - * Make sure struct sha1_state begins directly with the SHA1 - * 160-bit internal state, as this is what the asm functions expect. - */ - BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); - - kernel_fpu_begin(); - remain = sha1_base_do_update_blocks(desc, data, len, sha1_xform); - kernel_fpu_end(); - - return remain; -} - -static inline int sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, - sha1_block_fn *sha1_xform) -{ - kernel_fpu_begin(); - sha1_base_do_finup(desc, data, len, sha1_xform); - kernel_fpu_end(); - - return sha1_base_finish(desc, out); -} - -asmlinkage void sha1_transform_ssse3(struct sha1_state *state, - const u8 *data, int blocks); - -static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha1_update_x86(desc, data, len, sha1_transform_ssse3); -} - -static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha1_finup(desc, data, len, out, sha1_transform_ssse3); -} - -static struct shash_alg sha1_ssse3_alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = sha1_ssse3_update, - .finup = sha1_ssse3_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-ssse3", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int register_sha1_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - return crypto_register_shash(&sha1_ssse3_alg); - return 0; -} - -static void unregister_sha1_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_shash(&sha1_ssse3_alg); -} - -asmlinkage void sha1_transform_avx(struct sha1_state *state, - const u8 *data, int blocks); - -static int sha1_avx_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha1_update_x86(desc, data, len, sha1_transform_avx); -} - -static int sha1_avx_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha1_finup(desc, data, len, out, sha1_transform_avx); -} - -static struct shash_alg sha1_avx_alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = sha1_avx_update, - .finup = sha1_avx_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-avx", - .cra_priority = 160, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool avx_usable(void) -{ - if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { - if (boot_cpu_has(X86_FEATURE_AVX)) - pr_info("AVX detected but unusable.\n"); - return false; - } - - return true; -} - -static int register_sha1_avx(void) -{ - if (avx_usable()) - return crypto_register_shash(&sha1_avx_alg); - return 0; -} - -static void unregister_sha1_avx(void) -{ - if (avx_usable()) - crypto_unregister_shash(&sha1_avx_alg); -} - -#define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ - -asmlinkage void sha1_transform_avx2(struct sha1_state *state, - const u8 *data, int blocks); - -static bool avx2_usable(void) -{ - if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) - && boot_cpu_has(X86_FEATURE_BMI1) - && boot_cpu_has(X86_FEATURE_BMI2)) - return true; - - return false; -} - -static inline void sha1_apply_transform_avx2(struct sha1_state *state, - const u8 *data, int blocks) -{ - /* Select the optimal transform based on data block size */ - if (blocks >= SHA1_AVX2_BLOCK_OPTSIZE) - sha1_transform_avx2(state, data, blocks); - else - sha1_transform_avx(state, data, blocks); -} - -static int sha1_avx2_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha1_update_x86(desc, data, len, sha1_apply_transform_avx2); -} - -static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha1_finup(desc, data, len, out, sha1_apply_transform_avx2); -} - -static struct shash_alg sha1_avx2_alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = sha1_avx2_update, - .finup = sha1_avx2_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-avx2", - .cra_priority = 170, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int register_sha1_avx2(void) -{ - if (avx2_usable()) - return crypto_register_shash(&sha1_avx2_alg); - return 0; -} - -static void unregister_sha1_avx2(void) -{ - if (avx2_usable()) - crypto_unregister_shash(&sha1_avx2_alg); -} - -asmlinkage void sha1_ni_transform(struct sha1_state *digest, const u8 *data, - int rounds); - -static int sha1_ni_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha1_update_x86(desc, data, len, sha1_ni_transform); -} - -static int sha1_ni_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha1_finup(desc, data, len, out, sha1_ni_transform); -} - -static struct shash_alg sha1_ni_alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = sha1_ni_update, - .finup = sha1_ni_finup, - .descsize = SHA1_STATE_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-ni", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int register_sha1_ni(void) -{ - if (boot_cpu_has(X86_FEATURE_SHA_NI)) - return crypto_register_shash(&sha1_ni_alg); - return 0; -} - -static void unregister_sha1_ni(void) -{ - if (boot_cpu_has(X86_FEATURE_SHA_NI)) - crypto_unregister_shash(&sha1_ni_alg); -} - -static int __init sha1_ssse3_mod_init(void) -{ - if (!x86_match_cpu(module_cpu_ids)) - return -ENODEV; - - if (register_sha1_ssse3()) - goto fail; - - if (register_sha1_avx()) { - unregister_sha1_ssse3(); - goto fail; - } - - if (register_sha1_avx2()) { - unregister_sha1_avx(); - unregister_sha1_ssse3(); - goto fail; - } - - if (register_sha1_ni()) { - unregister_sha1_avx2(); - unregister_sha1_avx(); - unregister_sha1_ssse3(); - goto fail; - } - - return 0; -fail: - return -ENODEV; -} - -static void __exit sha1_ssse3_mod_fini(void) -{ - unregister_sha1_ni(); - unregister_sha1_avx2(); - unregister_sha1_avx(); - unregister_sha1_ssse3(); -} - -module_init(sha1_ssse3_mod_init); -module_exit(sha1_ssse3_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated"); - -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_ALIAS_CRYPTO("sha1-ssse3"); -MODULE_ALIAS_CRYPTO("sha1-avx"); -MODULE_ALIAS_CRYPTO("sha1-avx2"); -MODULE_ALIAS_CRYPTO("sha1-ni"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 019034f2bb53..dc34441a80bd 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -152,6 +152,7 @@ config CRYPTO_LIB_SHA1_ARCH default y if PPC default y if S390 default y if SPARC64 + default y if X86_64 config CRYPTO_LIB_SHA256 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 7897da2de623..b09fdb7f4b4c 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -82,6 +82,9 @@ libsha1-y += powerpc/sha1-powerpc-asm.o libsha1-$(CONFIG_SPE) += powerpc/sha1-spe-asm.o endif libsha1-$(CONFIG_SPARC) += sparc/sha1_asm.o +libsha1-$(CONFIG_X86) += x86/sha1-ssse3-and-avx.o \ + x86/sha1-avx2-asm.o \ + x86/sha1-ni-asm.o endif # CONFIG_CRYPTO_LIB_SHA1_ARCH ################################################################################ diff --git a/arch/x86/crypto/sha1_avx2_x86_64_asm.S b/lib/crypto/x86/sha1-avx2-asm.S similarity index 98% rename from arch/x86/crypto/sha1_avx2_x86_64_asm.S rename to lib/crypto/x86/sha1-avx2-asm.S index 4b49bdc95265..91b2dc6db179 100644 --- a/arch/x86/crypto/sha1_avx2_x86_64_asm.S +++ b/lib/crypto/x86/sha1-avx2-asm.S @@ -62,11 +62,8 @@ *Visit http://software.intel.com/en-us/articles/ *and refer to improving-the-performance-of-the-secure-hash-algorithm-1/ * - *Updates 20-byte SHA-1 record at start of 'state', from 'input', for - *even number of 'blocks' consecutive 64-byte blocks. - * - *extern "C" void sha1_transform_avx2( - * struct sha1_state *state, const u8* input, int blocks ); + * void sha1_transform_avx2(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ #include diff --git a/arch/x86/crypto/sha1_ni_asm.S b/lib/crypto/x86/sha1-ni-asm.S similarity index 90% rename from arch/x86/crypto/sha1_ni_asm.S rename to lib/crypto/x86/sha1-ni-asm.S index cade913d4882..3989b0642ff5 100644 --- a/arch/x86/crypto/sha1_ni_asm.S +++ b/lib/crypto/x86/sha1-ni-asm.S @@ -54,7 +54,6 @@ */ #include -#include #define DIGEST_PTR %rdi /* 1st arg */ #define DATA_PTR %rsi /* 2nd arg */ @@ -74,26 +73,22 @@ /* - * Intel SHA Extensions optimized implementation of a SHA-1 update function + * Intel SHA Extensions optimized implementation of a SHA-1 block function * - * The function takes a pointer to the current hash values, a pointer to the - * input data, and a number of 64 byte blocks to process. Once all blocks have - * been processed, the digest pointer is updated with the resulting hash value. - * The function only processes complete blocks, there is no functionality to - * store partial blocks. All message padding and hash value initialization must - * be done outside the update function. + * This function takes a pointer to the current SHA-1 state, a pointer to the + * input data, and the number of 64-byte blocks to process. Once all blocks + * have been processed, the state is updated with the new state. This function + * only processes complete blocks. State initialization, buffering of partial + * blocks, and digest finalization are expected to be handled elsewhere. * * The indented lines in the loop are instructions related to rounds processing. * The non-indented lines are instructions related to the message schedule. * - * void sha1_ni_transform(uint32_t *digest, const void *data, - uint32_t numBlocks) - * digest : pointer to digest - * data: pointer to input data - * numBlocks: Number of blocks to process + * void sha1_ni_transform(struct sha1_block_state *state, + * const u8 *data, size_t nblocks) */ .text -SYM_TYPED_FUNC_START(sha1_ni_transform) +SYM_FUNC_START(sha1_ni_transform) push %rbp mov %rsp, %rbp sub $FRAME_SIZE, %rsp diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/lib/crypto/x86/sha1-ssse3-and-avx.S similarity index 97% rename from arch/x86/crypto/sha1_ssse3_asm.S rename to lib/crypto/x86/sha1-ssse3-and-avx.S index f54988c80eb4..78b48cb09c5b 100644 --- a/arch/x86/crypto/sha1_ssse3_asm.S +++ b/lib/crypto/x86/sha1-ssse3-and-avx.S @@ -25,7 +25,6 @@ */ #include -#include #define CTX %rdi // arg1 #define BUF %rsi // arg2 @@ -68,7 +67,7 @@ * param: function's name */ .macro SHA1_VECTOR_ASM name - SYM_TYPED_FUNC_START(\name) + SYM_FUNC_START(\name) push %rbx push %r12 @@ -461,10 +460,8 @@ W_PRECALC_SSSE3 /* * SSSE3 optimized implementation: * - * extern "C" void sha1_transform_ssse3(struct sha1_state *state, - * const u8 *data, int blocks); - * - * Note that struct sha1_state is assumed to begin with u32 state[5]. + * void sha1_transform_ssse3(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ SHA1_VECTOR_ASM sha1_transform_ssse3 @@ -548,7 +545,7 @@ W_PRECALC_AVX /* AVX optimized implementation: - * extern "C" void sha1_transform_avx(struct sha1_state *state, - * const u8 *data, int blocks); + * void sha1_transform_avx(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ SHA1_VECTOR_ASM sha1_transform_avx diff --git a/lib/crypto/x86/sha1.h b/lib/crypto/x86/sha1.h new file mode 100644 index 000000000000..e308379d89bc --- /dev/null +++ b/lib/crypto/x86/sha1.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-1 optimized for x86_64 + * + * Copyright 2025 Google LLC + */ +#include +#include + +DEFINE_STATIC_CALL(sha1_blocks_x86, sha1_blocks_generic); + +#define DEFINE_X86_SHA1_FN(c_fn, asm_fn) \ + asmlinkage void asm_fn(struct sha1_block_state *state, \ + const u8 *data, size_t nblocks); \ + static void c_fn(struct sha1_block_state *state, \ + const u8 *data, size_t nblocks) \ + { \ + if (likely(irq_fpu_usable())) { \ + kernel_fpu_begin(); \ + asm_fn(state, data, nblocks); \ + kernel_fpu_end(); \ + } else { \ + sha1_blocks_generic(state, data, nblocks); \ + } \ + } + +DEFINE_X86_SHA1_FN(sha1_blocks_ssse3, sha1_transform_ssse3); +DEFINE_X86_SHA1_FN(sha1_blocks_avx, sha1_transform_avx); +DEFINE_X86_SHA1_FN(sha1_blocks_ni, sha1_ni_transform); + +#define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ + +asmlinkage void sha1_transform_avx2(struct sha1_block_state *state, + const u8 *data, size_t nblocks); +static void sha1_blocks_avx2(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + if (likely(irq_fpu_usable())) { + kernel_fpu_begin(); + /* Select the optimal transform based on the number of blocks */ + if (nblocks >= SHA1_AVX2_BLOCK_OPTSIZE) + sha1_transform_avx2(state, data, nblocks); + else + sha1_transform_avx(state, data, nblocks); + kernel_fpu_end(); + } else { + sha1_blocks_generic(state, data, nblocks); + } +} + +static void sha1_blocks(struct sha1_block_state *state, + const u8 *data, size_t nblocks) +{ + static_call(sha1_blocks_x86)(state, data, nblocks); +} + +#define sha1_mod_init_arch sha1_mod_init_arch +static inline void sha1_mod_init_arch(void) +{ + if (boot_cpu_has(X86_FEATURE_SHA_NI)) { + static_call_update(sha1_blocks_x86, sha1_blocks_ni); + } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, + NULL) && + boot_cpu_has(X86_FEATURE_AVX)) { + if (boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_BMI1) && + boot_cpu_has(X86_FEATURE_BMI2)) + static_call_update(sha1_blocks_x86, sha1_blocks_avx2); + else + static_call_update(sha1_blocks_x86, sha1_blocks_avx); + } else if (boot_cpu_has(X86_FEATURE_SSSE3)) { + static_call_update(sha1_blocks_x86, sha1_blocks_ssse3); + } +} From c76ed8790b3018fe36647d9aae96e0373f321184 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 12 Jul 2025 16:23:05 -0700 Subject: [PATCH 64/67] crypto: sha1 - Remove sha1_base.h sha1_base.h is no longer used, so remove it. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250712232329.818226-15-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sha1_base.h | 82 -------------------------------------- 1 file changed, 82 deletions(-) delete mode 100644 include/crypto/sha1_base.h diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h deleted file mode 100644 index 62701d136c79..000000000000 --- a/include/crypto/sha1_base.h +++ /dev/null @@ -1,82 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sha1_base.h - core logic for SHA-1 implementations - * - * Copyright (C) 2015 Linaro Ltd - */ - -#ifndef _CRYPTO_SHA1_BASE_H -#define _CRYPTO_SHA1_BASE_H - -#include -#include -#include -#include -#include -#include - -typedef void (sha1_block_fn)(struct sha1_state *sst, u8 const *src, int blocks); - -static inline int sha1_base_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA1_H0; - sctx->state[1] = SHA1_H1; - sctx->state[2] = SHA1_H2; - sctx->state[3] = SHA1_H3; - sctx->state[4] = SHA1_H4; - sctx->count = 0; - - return 0; -} - -static inline int sha1_base_do_update_blocks(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha1_block_fn *block_fn) -{ - unsigned int remain = len - round_down(len, SHA1_BLOCK_SIZE); - struct sha1_state *sctx = shash_desc_ctx(desc); - - sctx->count += len - remain; - block_fn(sctx, data, len / SHA1_BLOCK_SIZE); - return remain; -} - -static inline int sha1_base_do_finup(struct shash_desc *desc, - const u8 *src, unsigned int len, - sha1_block_fn *block_fn) -{ - unsigned int bit_offset = SHA1_BLOCK_SIZE / 8 - 1; - struct sha1_state *sctx = shash_desc_ctx(desc); - union { - __be64 b64[SHA1_BLOCK_SIZE / 4]; - u8 u8[SHA1_BLOCK_SIZE * 2]; - } block = {}; - - if (len >= bit_offset * 8) - bit_offset += SHA1_BLOCK_SIZE / 8; - memcpy(&block, src, len); - block.u8[len] = 0x80; - sctx->count += len; - block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); - block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SHA1_BLOCK_SIZE); - memzero_explicit(&block, sizeof(block)); - - return 0; -} - -static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - __be32 *digest = (__be32 *)out; - int i; - - for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], digest++); - - return 0; -} - -#endif /* _CRYPTO_SHA1_BASE_H */ From f88ed14aa0ef64ff5633605114efe313a0bed84b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 18 Jul 2025 12:18:59 -0700 Subject: [PATCH 65/67] lib/crypto: x86/sha1-ni: Minor optimizations and cleanup - Store the previous state in %xmm8-%xmm9 instead of spilling it to the stack. There are plenty of unused XMM registers here, so there is no reason to spill to the stack. (While 32-bit code is limited to %xmm0-%xmm7, this is 64-bit code, so it's free to use %xmm8-%xmm15.) - Remove the unnecessary check for nblocks == 0. sha1_ni_transform() is always passed a positive nblocks. - To get an XMM register with 'e' in the high dword and the rest zeroes, just zeroize the register using pxor, then load 'e'. Previously the code loaded 'e', then zeroized the lower dwords by AND-ing with a constant, which was slightly less efficient. - Instead of computing &DATA_PTR[NBLOCKS << 6] and stopping when DATA_PTR reaches that value, instead just decrement NBLOCKS on each iteration and stop when it reaches 0. This is fewer instructions. - Rename DIGEST_PTR to STATE_PTR. It points to the SHA-1 internal state, not a SHA-1 digest value. This commit shrinks the code size of sha1_ni_transform() from 624 bytes to 589 bytes and also shrinks rodata by 16 bytes. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250718191900.42877-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/x86/sha1-ni-asm.S | 66 +++++++++++++----------------------- 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/lib/crypto/x86/sha1-ni-asm.S b/lib/crypto/x86/sha1-ni-asm.S index 3989b0642ff5..1d08b2f364ce 100644 --- a/lib/crypto/x86/sha1-ni-asm.S +++ b/lib/crypto/x86/sha1-ni-asm.S @@ -55,13 +55,10 @@ #include -#define DIGEST_PTR %rdi /* 1st arg */ +#define STATE_PTR %rdi /* 1st arg */ #define DATA_PTR %rsi /* 2nd arg */ #define NUM_BLKS %rdx /* 3rd arg */ -/* gcc conversion */ -#define FRAME_SIZE 32 /* space for 2x16 bytes */ - #define ABCD %xmm0 #define E0 %xmm1 /* Need two E's b/c they ping pong */ #define E1 %xmm2 @@ -70,15 +67,17 @@ #define MSG2 %xmm5 #define MSG3 %xmm6 #define SHUF_MASK %xmm7 - +#define ABCD_SAVED %xmm8 +#define E0_SAVED %xmm9 /* * Intel SHA Extensions optimized implementation of a SHA-1 block function * * This function takes a pointer to the current SHA-1 state, a pointer to the - * input data, and the number of 64-byte blocks to process. Once all blocks - * have been processed, the state is updated with the new state. This function - * only processes complete blocks. State initialization, buffering of partial + * input data, and the number of 64-byte blocks to process. The number of + * blocks to process is assumed to be nonzero. Once all blocks have been + * processed, the state is updated with the new state. This function only + * processes complete blocks. State initialization, buffering of partial * blocks, and digest finalization are expected to be handled elsewhere. * * The indented lines in the loop are instructions related to rounds processing. @@ -89,27 +88,19 @@ */ .text SYM_FUNC_START(sha1_ni_transform) - push %rbp - mov %rsp, %rbp - sub $FRAME_SIZE, %rsp - and $~0xF, %rsp - shl $6, NUM_BLKS /* convert to bytes */ - jz .Ldone_hash - add DATA_PTR, NUM_BLKS /* pointer to end of data */ - - /* load initial hash values */ - pinsrd $3, 1*16(DIGEST_PTR), E0 - movdqu 0*16(DIGEST_PTR), ABCD - pand UPPER_WORD_MASK(%rip), E0 + /* Load the initial state from STATE_PTR. */ + pxor E0, E0 + pinsrd $3, 16(STATE_PTR), E0 + movdqu (STATE_PTR), ABCD pshufd $0x1B, ABCD, ABCD movdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK -.Lloop0: - /* Save hash values for addition after rounds */ - movdqa E0, (0*16)(%rsp) - movdqa ABCD, (1*16)(%rsp) +.Lnext_block: + /* Save the state for addition after the rounds. */ + movdqa E0, E0_SAVED + movdqa ABCD, ABCD_SAVED /* Rounds 0-3 */ movdqu 0*16(DATA_PTR), MSG0 @@ -267,23 +258,19 @@ SYM_FUNC_START(sha1_ni_transform) movdqa ABCD, E0 sha1rnds4 $3, E1, ABCD - /* Add current hash values with previously saved */ - sha1nexte (0*16)(%rsp), E0 - paddd (1*16)(%rsp), ABCD + /* Add the previous state (before the rounds) to the current state. */ + sha1nexte E0_SAVED, E0 + paddd ABCD_SAVED, ABCD - /* Increment data pointer and loop if more to process */ + /* Advance to the next block, or break if there are no more blocks. */ add $64, DATA_PTR - cmp NUM_BLKS, DATA_PTR - jne .Lloop0 + dec NUM_BLKS + jnz .Lnext_block - /* Write hash values back in the correct order */ + /* Store the new state to STATE_PTR. */ + pextrd $3, E0, 16(STATE_PTR) pshufd $0x1B, ABCD, ABCD - movdqu ABCD, 0*16(DIGEST_PTR) - pextrd $3, E0, 1*16(DIGEST_PTR) - -.Ldone_hash: - mov %rbp, %rsp - pop %rbp + movdqu ABCD, (STATE_PTR) RET SYM_FUNC_END(sha1_ni_transform) @@ -292,8 +279,3 @@ SYM_FUNC_END(sha1_ni_transform) .align 16 PSHUFFLE_BYTE_FLIP_MASK: .octa 0x000102030405060708090a0b0c0d0e0f - -.section .rodata.cst16.UPPER_WORD_MASK, "aM", @progbits, 16 -.align 16 -UPPER_WORD_MASK: - .octa 0xFFFFFFFF000000000000000000000000 From 42e3376e0954d7e589e5a53d6149835cad64e8e6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 18 Jul 2025 12:19:00 -0700 Subject: [PATCH 66/67] lib/crypto: x86/sha1-ni: Convert to use rounds macros The assembly code that does all 80 rounds of SHA-1 is highly repetitive. Replace it with 20 expansions of a macro that does 4 rounds, using the macro arguments and .if directives to handle the slight variations between rounds. This reduces the length of sha1-ni-asm.S by 129 lines while still producing the exact same object file. This mirrors sha256-ni-asm.S which uses this same strategy. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250718191900.42877-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/x86/sha1-ni-asm.S | 187 ++++++----------------------------- 1 file changed, 29 insertions(+), 158 deletions(-) diff --git a/lib/crypto/x86/sha1-ni-asm.S b/lib/crypto/x86/sha1-ni-asm.S index 1d08b2f364ce..428f9b960594 100644 --- a/lib/crypto/x86/sha1-ni-asm.S +++ b/lib/crypto/x86/sha1-ni-asm.S @@ -70,6 +70,29 @@ #define ABCD_SAVED %xmm8 #define E0_SAVED %xmm9 +.macro do_4rounds i, m0, m1, m2, m3, e0, e1 +.if \i < 16 + movdqu \i*4(DATA_PTR), \m0 + pshufb SHUF_MASK, \m0 +.endif +.if \i == 0 + paddd \m0, \e0 +.else + sha1nexte \m0, \e0 +.endif + movdqa ABCD, \e1 +.if \i >= 12 && \i < 76 + sha1msg2 \m0, \m1 +.endif + sha1rnds4 $\i / 20, \e0, ABCD +.if \i >= 4 && \i < 68 + sha1msg1 \m0, \m3 +.endif +.if \i >= 8 && \i < 72 + pxor \m0, \m2 +.endif +.endm + /* * Intel SHA Extensions optimized implementation of a SHA-1 block function * @@ -80,9 +103,6 @@ * processes complete blocks. State initialization, buffering of partial * blocks, and digest finalization are expected to be handled elsewhere. * - * The indented lines in the loop are instructions related to rounds processing. - * The non-indented lines are instructions related to the message schedule. - * * void sha1_ni_transform(struct sha1_block_state *state, * const u8 *data, size_t nblocks) */ @@ -102,161 +122,12 @@ SYM_FUNC_START(sha1_ni_transform) movdqa E0, E0_SAVED movdqa ABCD, ABCD_SAVED - /* Rounds 0-3 */ - movdqu 0*16(DATA_PTR), MSG0 - pshufb SHUF_MASK, MSG0 - paddd MSG0, E0 - movdqa ABCD, E1 - sha1rnds4 $0, E0, ABCD - - /* Rounds 4-7 */ - movdqu 1*16(DATA_PTR), MSG1 - pshufb SHUF_MASK, MSG1 - sha1nexte MSG1, E1 - movdqa ABCD, E0 - sha1rnds4 $0, E1, ABCD - sha1msg1 MSG1, MSG0 - - /* Rounds 8-11 */ - movdqu 2*16(DATA_PTR), MSG2 - pshufb SHUF_MASK, MSG2 - sha1nexte MSG2, E0 - movdqa ABCD, E1 - sha1rnds4 $0, E0, ABCD - sha1msg1 MSG2, MSG1 - pxor MSG2, MSG0 - - /* Rounds 12-15 */ - movdqu 3*16(DATA_PTR), MSG3 - pshufb SHUF_MASK, MSG3 - sha1nexte MSG3, E1 - movdqa ABCD, E0 - sha1msg2 MSG3, MSG0 - sha1rnds4 $0, E1, ABCD - sha1msg1 MSG3, MSG2 - pxor MSG3, MSG1 - - /* Rounds 16-19 */ - sha1nexte MSG0, E0 - movdqa ABCD, E1 - sha1msg2 MSG0, MSG1 - sha1rnds4 $0, E0, ABCD - sha1msg1 MSG0, MSG3 - pxor MSG0, MSG2 - - /* Rounds 20-23 */ - sha1nexte MSG1, E1 - movdqa ABCD, E0 - sha1msg2 MSG1, MSG2 - sha1rnds4 $1, E1, ABCD - sha1msg1 MSG1, MSG0 - pxor MSG1, MSG3 - - /* Rounds 24-27 */ - sha1nexte MSG2, E0 - movdqa ABCD, E1 - sha1msg2 MSG2, MSG3 - sha1rnds4 $1, E0, ABCD - sha1msg1 MSG2, MSG1 - pxor MSG2, MSG0 - - /* Rounds 28-31 */ - sha1nexte MSG3, E1 - movdqa ABCD, E0 - sha1msg2 MSG3, MSG0 - sha1rnds4 $1, E1, ABCD - sha1msg1 MSG3, MSG2 - pxor MSG3, MSG1 - - /* Rounds 32-35 */ - sha1nexte MSG0, E0 - movdqa ABCD, E1 - sha1msg2 MSG0, MSG1 - sha1rnds4 $1, E0, ABCD - sha1msg1 MSG0, MSG3 - pxor MSG0, MSG2 - - /* Rounds 36-39 */ - sha1nexte MSG1, E1 - movdqa ABCD, E0 - sha1msg2 MSG1, MSG2 - sha1rnds4 $1, E1, ABCD - sha1msg1 MSG1, MSG0 - pxor MSG1, MSG3 - - /* Rounds 40-43 */ - sha1nexte MSG2, E0 - movdqa ABCD, E1 - sha1msg2 MSG2, MSG3 - sha1rnds4 $2, E0, ABCD - sha1msg1 MSG2, MSG1 - pxor MSG2, MSG0 - - /* Rounds 44-47 */ - sha1nexte MSG3, E1 - movdqa ABCD, E0 - sha1msg2 MSG3, MSG0 - sha1rnds4 $2, E1, ABCD - sha1msg1 MSG3, MSG2 - pxor MSG3, MSG1 - - /* Rounds 48-51 */ - sha1nexte MSG0, E0 - movdqa ABCD, E1 - sha1msg2 MSG0, MSG1 - sha1rnds4 $2, E0, ABCD - sha1msg1 MSG0, MSG3 - pxor MSG0, MSG2 - - /* Rounds 52-55 */ - sha1nexte MSG1, E1 - movdqa ABCD, E0 - sha1msg2 MSG1, MSG2 - sha1rnds4 $2, E1, ABCD - sha1msg1 MSG1, MSG0 - pxor MSG1, MSG3 - - /* Rounds 56-59 */ - sha1nexte MSG2, E0 - movdqa ABCD, E1 - sha1msg2 MSG2, MSG3 - sha1rnds4 $2, E0, ABCD - sha1msg1 MSG2, MSG1 - pxor MSG2, MSG0 - - /* Rounds 60-63 */ - sha1nexte MSG3, E1 - movdqa ABCD, E0 - sha1msg2 MSG3, MSG0 - sha1rnds4 $3, E1, ABCD - sha1msg1 MSG3, MSG2 - pxor MSG3, MSG1 - - /* Rounds 64-67 */ - sha1nexte MSG0, E0 - movdqa ABCD, E1 - sha1msg2 MSG0, MSG1 - sha1rnds4 $3, E0, ABCD - sha1msg1 MSG0, MSG3 - pxor MSG0, MSG2 - - /* Rounds 68-71 */ - sha1nexte MSG1, E1 - movdqa ABCD, E0 - sha1msg2 MSG1, MSG2 - sha1rnds4 $3, E1, ABCD - pxor MSG1, MSG3 - - /* Rounds 72-75 */ - sha1nexte MSG2, E0 - movdqa ABCD, E1 - sha1msg2 MSG2, MSG3 - sha1rnds4 $3, E0, ABCD - - /* Rounds 76-79 */ - sha1nexte MSG3, E1 - movdqa ABCD, E0 - sha1rnds4 $3, E1, ABCD +.irp i, 0, 16, 32, 48, 64 + do_4rounds (\i + 0), MSG0, MSG1, MSG2, MSG3, E0, E1 + do_4rounds (\i + 4), MSG1, MSG2, MSG3, MSG0, E1, E0 + do_4rounds (\i + 8), MSG2, MSG3, MSG0, MSG1, E0, E1 + do_4rounds (\i + 12), MSG3, MSG0, MSG1, MSG2, E1, E0 +.endr /* Add the previous state (before the rounds) to the current state. */ sha1nexte E0_SAVED, E0 From debc1e5a431779c027a5752f247a4de2e4f702b2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 18 Jul 2025 15:07:06 -0700 Subject: [PATCH 67/67] lib/crypto: arm64/sha512-ce: Drop compatibility macros for older binutils Now that the oldest supported binutils version is 2.30, the macros that emit the SHA-512 instructions as '.inst' words are no longer needed. So drop them. No change in the generated machine code. Changed from the original patch by Ard Biesheuvel: (https://lore.kernel.org/r/20250515142702.2592942-2-ardb+git@google.com): - Reduced scope to just SHA-512 - Added comment that explains why "sha3" is used instead of "sha2" Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20250718220706.475240-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/sha512-ce-core.S | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S index 7d870a435ea3..22f1ded89bc8 100644 --- a/lib/crypto/arm64/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -12,26 +12,17 @@ #include #include - .irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19 - .set .Lq\b, \b - .set .Lv\b\().2d, \b - .endr - - .macro sha512h, rd, rn, rm - .inst 0xce608000 | .L\rd | (.L\rn << 5) | (.L\rm << 16) - .endm - - .macro sha512h2, rd, rn, rm - .inst 0xce608400 | .L\rd | (.L\rn << 5) | (.L\rm << 16) - .endm - - .macro sha512su0, rd, rn - .inst 0xcec08000 | .L\rd | (.L\rn << 5) - .endm - - .macro sha512su1, rd, rn, rm - .inst 0xce608800 | .L\rd | (.L\rn << 5) | (.L\rm << 16) - .endm + /* + * We have to specify the "sha3" feature here, since the GNU and clang + * assemblers both consider the SHA-512 instructions to be part of the + * "sha3" feature. (Except binutils 2.30 through 2.42, which used + * "sha2". But "sha3" implies "sha2", so "sha3" still works in those + * versions.) "sha3" doesn't make a lot of sense, since SHA-512 is part + * of the SHA-2 family of algorithms, and also the Arm Architecture + * Reference Manual defines FEAT_SHA512 and FEAT_SHA3 separately. + * Regardless, we must use "sha3" to be compatible with the assemblers. + */ + .arch armv8-a+sha3 /* * The SHA-512 round constants