mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-08-29 17:30:01 -04:00
37f57bd3fa
User mode tasks compiled with Zicfilp may call indirectly into the vdso (like hwprobe indirect calls). Add support for compiling landing pads into the vdso. Landing pad instructions in the vdso will be no-ops for tasks which have not enabled landing pads. Furthermore, add support for the C sources of the vdso to be compiled with shadow stack and landing pads enabled as well. Landing pad and shadow stack instructions are emitted only when the VDSO_CFI cflags option is defined during compile. Signed-off-by: Jim Shu <jim.shu@sifive.com> Reviewed-by: Zong Li <zong.li@sifive.com> Signed-off-by: Deepak Gupta <debug@rivosinc.com> Tested-by: Andreas Korb <andreas.korb@aisec.fraunhofer.de> # QEMU, custom CVA6 Tested-by: Valentin Haudiquet <valentin.haudiquet@canonical.com> Link: https://patch.msgid.link/20251112-v5_user_cfi_series-v23-23-b55691eacf4f@rivosinc.com [pjw@kernel.org: cleaned up patch description, issues reported by checkpatch] Signed-off-by: Paul Walmsley <pjw@kernel.org>
253 lines
5.5 KiB
ArmAsm
253 lines
5.5 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Copyright (C) 2025 Xi Ruoyao <xry111@xry111.site>. All Rights Reserved.
|
|
*
|
|
* Based on arch/loongarch/vdso/vgetrandom-chacha.S.
|
|
*/
|
|
|
|
#include <asm/asm.h>
|
|
#include <linux/linkage.h>
|
|
#include <asm/assembler.h>
|
|
|
|
.text
|
|
|
|
.macro ROTRI rd rs imm
|
|
slliw t0, \rs, 32 - \imm
|
|
srliw \rd, \rs, \imm
|
|
or \rd, \rd, t0
|
|
.endm
|
|
|
|
.macro OP_4REG op d0 d1 d2 d3 s0 s1 s2 s3
|
|
\op \d0, \d0, \s0
|
|
\op \d1, \d1, \s1
|
|
\op \d2, \d2, \s2
|
|
\op \d3, \d3, \s3
|
|
.endm
|
|
|
|
/*
|
|
* a0: output bytes
|
|
* a1: 32-byte key input
|
|
* a2: 8-byte counter input/output
|
|
* a3: number of 64-byte blocks to write to output
|
|
*/
|
|
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
|
|
|
|
#define output a0
|
|
#define key a1
|
|
#define counter a2
|
|
#define nblocks a3
|
|
#define i a4
|
|
#define state0 s0
|
|
#define state1 s1
|
|
#define state2 s2
|
|
#define state3 s3
|
|
#define state4 s4
|
|
#define state5 s5
|
|
#define state6 s6
|
|
#define state7 s7
|
|
#define state8 s8
|
|
#define state9 s9
|
|
#define state10 s10
|
|
#define state11 s11
|
|
#define state12 a5
|
|
#define state13 a6
|
|
#define state14 a7
|
|
#define state15 t1
|
|
#define cnt t2
|
|
#define copy0 t3
|
|
#define copy1 t4
|
|
#define copy2 t5
|
|
#define copy3 t6
|
|
|
|
/* Packs to be used with OP_4REG */
|
|
#define line0 state0, state1, state2, state3
|
|
#define line1 state4, state5, state6, state7
|
|
#define line2 state8, state9, state10, state11
|
|
#define line3 state12, state13, state14, state15
|
|
|
|
#define line1_perm state5, state6, state7, state4
|
|
#define line2_perm state10, state11, state8, state9
|
|
#define line3_perm state15, state12, state13, state14
|
|
|
|
#define copy copy0, copy1, copy2, copy3
|
|
|
|
#define _16 16, 16, 16, 16
|
|
#define _20 20, 20, 20, 20
|
|
#define _24 24, 24, 24, 24
|
|
#define _25 25, 25, 25, 25
|
|
vdso_lpad
|
|
/*
|
|
* The ABI requires s0-s9 saved.
|
|
* This does not violate the stack-less requirement: no sensitive data
|
|
* is spilled onto the stack.
|
|
*/
|
|
addi sp, sp, -12*SZREG
|
|
REG_S s0, (sp)
|
|
REG_S s1, SZREG(sp)
|
|
REG_S s2, 2*SZREG(sp)
|
|
REG_S s3, 3*SZREG(sp)
|
|
REG_S s4, 4*SZREG(sp)
|
|
REG_S s5, 5*SZREG(sp)
|
|
REG_S s6, 6*SZREG(sp)
|
|
REG_S s7, 7*SZREG(sp)
|
|
REG_S s8, 8*SZREG(sp)
|
|
REG_S s9, 9*SZREG(sp)
|
|
REG_S s10, 10*SZREG(sp)
|
|
REG_S s11, 11*SZREG(sp)
|
|
|
|
ld cnt, (counter)
|
|
|
|
li copy0, 0x61707865
|
|
li copy1, 0x3320646e
|
|
li copy2, 0x79622d32
|
|
li copy3, 0x6b206574
|
|
|
|
.Lblock:
|
|
/* state[0,1,2,3] = "expand 32-byte k" */
|
|
mv state0, copy0
|
|
mv state1, copy1
|
|
mv state2, copy2
|
|
mv state3, copy3
|
|
|
|
/* state[4,5,..,11] = key */
|
|
lw state4, (key)
|
|
lw state5, 4(key)
|
|
lw state6, 8(key)
|
|
lw state7, 12(key)
|
|
lw state8, 16(key)
|
|
lw state9, 20(key)
|
|
lw state10, 24(key)
|
|
lw state11, 28(key)
|
|
|
|
/* state[12,13] = counter */
|
|
mv state12, cnt
|
|
srli state13, cnt, 32
|
|
|
|
/* state[14,15] = 0 */
|
|
mv state14, zero
|
|
mv state15, zero
|
|
|
|
li i, 10
|
|
.Lpermute:
|
|
/* odd round */
|
|
OP_4REG addw line0, line1
|
|
OP_4REG xor line3, line0
|
|
OP_4REG ROTRI line3, _16
|
|
|
|
OP_4REG addw line2, line3
|
|
OP_4REG xor line1, line2
|
|
OP_4REG ROTRI line1, _20
|
|
|
|
OP_4REG addw line0, line1
|
|
OP_4REG xor line3, line0
|
|
OP_4REG ROTRI line3, _24
|
|
|
|
OP_4REG addw line2, line3
|
|
OP_4REG xor line1, line2
|
|
OP_4REG ROTRI line1, _25
|
|
|
|
/* even round */
|
|
OP_4REG addw line0, line1_perm
|
|
OP_4REG xor line3_perm, line0
|
|
OP_4REG ROTRI line3_perm, _16
|
|
|
|
OP_4REG addw line2_perm, line3_perm
|
|
OP_4REG xor line1_perm, line2_perm
|
|
OP_4REG ROTRI line1_perm, _20
|
|
|
|
OP_4REG addw line0, line1_perm
|
|
OP_4REG xor line3_perm, line0
|
|
OP_4REG ROTRI line3_perm, _24
|
|
|
|
OP_4REG addw line2_perm, line3_perm
|
|
OP_4REG xor line1_perm, line2_perm
|
|
OP_4REG ROTRI line1_perm, _25
|
|
|
|
addi i, i, -1
|
|
bnez i, .Lpermute
|
|
|
|
/* output[0,1,2,3] = copy[0,1,2,3] + state[0,1,2,3] */
|
|
OP_4REG addw line0, copy
|
|
sw state0, (output)
|
|
sw state1, 4(output)
|
|
sw state2, 8(output)
|
|
sw state3, 12(output)
|
|
|
|
/* from now on state[0,1,2,3] are scratch registers */
|
|
|
|
/* state[0,1,2,3] = lo(key) */
|
|
lw state0, (key)
|
|
lw state1, 4(key)
|
|
lw state2, 8(key)
|
|
lw state3, 12(key)
|
|
|
|
/* output[4,5,6,7] = state[0,1,2,3] + state[4,5,6,7] */
|
|
OP_4REG addw line1, line0
|
|
sw state4, 16(output)
|
|
sw state5, 20(output)
|
|
sw state6, 24(output)
|
|
sw state7, 28(output)
|
|
|
|
/* state[0,1,2,3] = hi(key) */
|
|
lw state0, 16(key)
|
|
lw state1, 20(key)
|
|
lw state2, 24(key)
|
|
lw state3, 28(key)
|
|
|
|
/* output[8,9,10,11] = tmp[0,1,2,3] + state[8,9,10,11] */
|
|
OP_4REG addw line2, line0
|
|
sw state8, 32(output)
|
|
sw state9, 36(output)
|
|
sw state10, 40(output)
|
|
sw state11, 44(output)
|
|
|
|
/* output[12,13,14,15] = state[12,13,14,15] + [cnt_lo, cnt_hi, 0, 0] */
|
|
addw state12, state12, cnt
|
|
srli state0, cnt, 32
|
|
addw state13, state13, state0
|
|
sw state12, 48(output)
|
|
sw state13, 52(output)
|
|
sw state14, 56(output)
|
|
sw state15, 60(output)
|
|
|
|
/* ++counter */
|
|
addi cnt, cnt, 1
|
|
|
|
/* output += 64 */
|
|
addi output, output, 64
|
|
/* --nblocks */
|
|
addi nblocks, nblocks, -1
|
|
bnez nblocks, .Lblock
|
|
|
|
/* counter = [cnt_lo, cnt_hi] */
|
|
sd cnt, (counter)
|
|
|
|
/* Zero out the potentially sensitive regs, in case nothing uses these
|
|
* again. As at now copy[0,1,2,3] just contains "expand 32-byte k" and
|
|
* state[0,...,11] are s0-s11 those we'll restore in the epilogue, we
|
|
* only need to zero state[12,...,15].
|
|
*/
|
|
mv state12, zero
|
|
mv state13, zero
|
|
mv state14, zero
|
|
mv state15, zero
|
|
|
|
REG_L s0, (sp)
|
|
REG_L s1, SZREG(sp)
|
|
REG_L s2, 2*SZREG(sp)
|
|
REG_L s3, 3*SZREG(sp)
|
|
REG_L s4, 4*SZREG(sp)
|
|
REG_L s5, 5*SZREG(sp)
|
|
REG_L s6, 6*SZREG(sp)
|
|
REG_L s7, 7*SZREG(sp)
|
|
REG_L s8, 8*SZREG(sp)
|
|
REG_L s9, 9*SZREG(sp)
|
|
REG_L s10, 10*SZREG(sp)
|
|
REG_L s11, 11*SZREG(sp)
|
|
addi sp, sp, 12*SZREG
|
|
|
|
ret
|
|
SYM_FUNC_END(__arch_chacha20_blocks_nostack)
|
|
|
|
emit_riscv_feature_1_and
|