From 2a2367d46d7a4ee4122b7a86e57125542dbbe963 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Mon, 17 Aug 2026 22:07:05 +0800 Subject: [PATCH 01/18] LoongArch: Fix acpi_package_ids[] array overflow With LoongArch virt machine, a typical setting is one core per socket, there will max 256 sockets (packages) on one VM. With PPTT acpi table, array acpi_package_ids[] will be overflowed. Here change the array size of acpi_package_ids[] with the max value of MAX_PACKAGES and KVM_MAX_VCPUS. Cc: stable@vger.kernel.org # 6.7+ Fixes: 4e8f58620f67 ("LoongArch: Retrieve CPU package ID from PPTT when available") Reviewed-by: Tao Cui Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/kernel/acpi.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 873e90990771..cb454ee92b20 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -202,7 +203,7 @@ static void __init acpi_process_madt(void) int pptt_enabled; static int acpi_nr_packages; -static int acpi_package_ids[MAX_PACKAGES]; +static int acpi_package_ids[MAX(MAX_PACKAGES, KVM_MAX_VCPUS)]; int __init parse_acpi_topology(void) { From 2677f97a67fdbc62a82ce1faa67791f54451d36f Mon Sep 17 00:00:00 2001 From: Han Gao Date: Mon, 17 Aug 2026 22:07:05 +0800 Subject: [PATCH 02/18] LoongArch: Add DIRECT_MAP_PHYSMEM_END definition get_free_mem_region() and mhp_get_pluggable_range() bound their search to DIRECT_MAP_PHYSMEM_END. LoongArch does not define it, so the fallback in include/linux/mm.h applies: under CONFIG_SPARSEMEM_VMEMMAP it is (1ULL << MAX_PHYSMEM_BITS) - 1, a compile-time constant that does not adapt to the CPU's physical address space bits (cpu_pabits, probed from CPUCFG1). The vmemmap window only covers physical space below 2^(cpu_pabits+1) (i.e. VMEMMAP_SIZE), so on CPUs with fewer physical address bits than MAX_PHYSMEM_BITS the fallback allows get_free_mem_region() to return a ZONE_DEVICE region outside the vmemmap window; vmemmap_populate() then wraps the memmap range around and maps it into low memory, silently corrupting the page tables. The same search also picked the top-of- address-space region that crashed memmap_init_zone_device() with amdkfd on Loongson-3C6000 in 6.16 [1]; the commit 2969b42c8f99 ("LoongArch/mm: align vmemmap to maximal folio size") keeps that region in bounds on current Loongson-3C6000 configs, but CPUs with smaller cpu_pabits (e.g. the Loongson-2K series) are still affected. Define DIRECT_MAP_PHYSMEM_END as the vmemmap-covered physical range, (1ULL << (cpu_pabits + 1)) - 1, capped at (1ULL << MAX_PHYSMEM_BITS) - 1 under CONFIG_SPARSEMEM, similar to the commit f3336b48cf9d ("riscv: mm: Define DIRECT_MAP_PHYSMEM_END"). [1] https://lore.kernel.org/amd-gfx/20250814032153.227285-1-jeffbai@aosc.io/ Cc: stable@vger.kernel.org # v6.13+ Signed-off-by: Han Gao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/pgtable.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 223528c04d73..59b37a824bab 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -125,6 +125,13 @@ struct vm_area_struct; #endif +/* Needed to limit get_free_mem_region() */ +#ifndef CONFIG_SPARSEMEM +#define DIRECT_MAP_PHYSMEM_END ((1ULL << (cpu_pabits + 1)) - 1) +#else +#define DIRECT_MAP_PHYSMEM_END min((1ULL << (cpu_pabits + 1)) - 1, (1ULL << MAX_PHYSMEM_BITS) - 1) +#endif + #define ptep_get(ptep) READ_ONCE(*(ptep)) #define pmdp_get(pmdp) READ_ONCE(*(pmdp)) From 18210a104bb97e28c26dc31fd9fc7b5c381fec62 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:14 +0800 Subject: [PATCH 03/18] LoongArch: Expand module virtual address space to 2GB The current 256MB module virtual address space is easily exhausted when loading massive graphics drivers such as amdgpu along with the large unstripped symbol tables, resulting in allocation failures of "execmem: unable to allocate memory". Thus, expand the module virtual address space to 2GB while keeping the current normal code model '-mcmodel=normal', rather than using the medium code model '-mcmodel=medium'. This approach avoids the extra performance overhead and larger binary size of forcing every function call into a 2-instruction sequence of 'pcaddu18i + jirl'. Given that individual module code segments rarely exceed 128MB, most jumps remain fast direct calls by using the bl instruction. For the long-distance jumps exceeding the +/-128MB limit, apply_r_larch_b26() emits PLT entries, while signed_imm_check() guarantees the run-time safety by rejecting any out-of-bound instruction offsets. There is still a risk that the distance between .init.text and .text of the same module exceeds 128MB. So we divide the 2GB virtual space to be two sub-regions: the first 256MB is for module text, and the rest is for module data. Cc: stable@vger.kernel.org Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/pgtable.h | 2 +- arch/loongarch/mm/init.c | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 59b37a824bab..a2191044435d 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -96,7 +96,7 @@ struct vm_area_struct; #ifdef CONFIG_64BIT #define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) -#define MODULES_END (MODULES_VADDR + SZ_256M) +#define MODULES_END (MODULES_VADDR + SZ_2G) /* 256MB for text, rest for data */ #ifdef CONFIG_KFENCE #define KFENCE_AREA_SIZE (((CONFIG_KFENCE_NUM_OBJECTS + 1) * 2 + 2) * PAGE_SIZE) diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c index 3407030f3e7a..4b46c5d30708 100644 --- a/arch/loongarch/mm/init.c +++ b/arch/loongarch/mm/init.c @@ -237,15 +237,26 @@ pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; EXPORT_SYMBOL(invalid_pte_table); #if defined(CONFIG_EXECMEM) && defined(MODULES_VADDR) +#define MODULES_TEXT_START (MODULES_VADDR) +#define MODULES_TEXT_END (MODULES_VADDR + SZ_256M) +#define MODULES_DATA_START (MODULES_VADDR + SZ_256M) +#define MODULES_DATA_END (MODULES_END) + static struct execmem_info execmem_info __ro_after_init; struct execmem_info __init *execmem_arch_setup(void) { execmem_info = (struct execmem_info){ .ranges = { - [EXECMEM_DEFAULT] = { - .start = MODULES_VADDR, - .end = MODULES_END, + [EXECMEM_MODULE_TEXT] = { + .start = MODULES_TEXT_START, + .end = MODULES_TEXT_END, + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + [EXECMEM_MODULE_DATA] = { + .start = MODULES_DATA_START, + .end = MODULES_DATA_END, .pgprot = PAGE_KERNEL, .alignment = 1, }, From 4b5b0b79d1f8ef9a683b6572267867d5d9755338 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:14 +0800 Subject: [PATCH 04/18] LoongArch: Use generic cmp_int() instead of custom cmp_3way() Currently, the module-sections.c file defines a custom cmp_3way() macro to perform a three-way comparison. There is already a generic cmp_int() macro to do the same thing in linux/sort.h, thus remove the custom macro and use the generic interface. This is similar with commit 3e17a4b443bb ("riscv: module: Use generic cmp_int() instead of custom cmp_3way()"). Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/kernel/module-sections.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/loongarch/kernel/module-sections.c b/arch/loongarch/kernel/module-sections.c index 9fa1c9814fcc..0259d9c7ea30 100644 --- a/arch/loongarch/kernel/module-sections.c +++ b/arch/loongarch/kernel/module-sections.c @@ -62,16 +62,14 @@ Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr v return (Elf_Addr)&plt[nr]; } -#define cmp_3way(a, b) ((a) < (b) ? -1 : (a) > (b)) - static int compare_rela(const void *x, const void *y) { int ret; const Elf_Rela *rela_x = x, *rela_y = y; - ret = cmp_3way(rela_x->r_info, rela_y->r_info); + ret = cmp_int(rela_x->r_info, rela_y->r_info); if (ret == 0) - ret = cmp_3way(rela_x->r_addend, rela_y->r_addend); + ret = cmp_int(rela_x->r_addend, rela_y->r_addend); return ret; } From 29479b14315de24616cc1fad86cbd392c36f557a Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:14 +0800 Subject: [PATCH 05/18] LoongArch: Use current_stack_pointer in current_pt_regs() The current implementation of current_pt_regs() relies on the compiler __builtin_frame_address(0). This introduces an unnecessary dependency on the frame pointer register, which forces the compiler to generate redundant prologue and epilogue code, create a larger stack frame, and perform redundant memory operations to preserve the frame pointer even in functions where it is otherwise unnecessary. Optimize this by switching to current_stack_pointer, which explicitly maps to the hardware stack pointer register. This allows the compiler to compute the stack alignment directly from the natively maintained "$sp" register, completely eliminating the overhead of preserving and restoring the frame pointer on the stack memory. As a prominent example, this optimization improves the hot-path function copy_thread(). A disassembly comparison of copy_thread() illustrates the elimination of the frame pointer, the reduction of stack frame size from 48 bytes down to 32 bytes, and a more compact epilogue path: Before: 00000000000004f0 : 4f0: 02ff4063 addi.d $sp, $sp, -48 4f4: 29c08076 st.d $fp, $sp, 32 4f8: 29c06077 st.d $s0, $sp, 24 4fc: 29c0a061 st.d $ra, $sp, 40 500: 02c0c076 addi.d $fp, $sp, 48 ... 54c: 1400006e lu12i.w $t2, 3 ... 55c: 03bffdce ori $t2, $t2, 0xfff 560: 00153ace or $t2, $fp, $t2 564: 02fb05cd addi.d $t1, $t2, -319 ... 628: 28c0a061 ld.d $ra, $sp, 40 62c: 28c08076 ld.d $fp, $sp, 32 630: 28c06077 ld.d $s0, $sp, 24 634: 00150004 move $a0, $zero 638: 02c0c063 addi.d $sp, $sp, 48 63c: 4c000020 ret After: 00000000000004f0 : 4f0: 02ff8063 addi.d $sp, $sp, -32 4f4: 29c04077 st.d $s0, $sp, 16 4f8: 29c06061 st.d $ra, $sp, 24 [ prologue st.d and addi.d for $fp are completely eliminated ] ... 544: 1400006e lu12i.w $t2, 3 ... 554: 03bffdce ori $t2, $t2, 0xfff 558: 0015386e or $t2, $sp, $t2 55c: 02fb05cd addi.d $t1, $t2, -319 ... 620: 28c06061 ld.d $ra, $sp, 24 624: 28c04077 ld.d $s0, $sp, 16 628: 00150004 move $a0, $zero [ epilogue ld.d for $fp is eliminated; exit path is shortened ] 62c: 02c08063 addi.d $sp, $sp, 32 630: 4c000020 ret Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/ptrace.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/loongarch/include/asm/ptrace.h b/arch/loongarch/include/asm/ptrace.h index e5d21e836d99..2a7ed442d905 100644 --- a/arch/loongarch/include/asm/ptrace.h +++ b/arch/loongarch/include/asm/ptrace.h @@ -170,11 +170,7 @@ static inline void die_if_kernel(const char *str, struct pt_regs *regs) die(str, regs); } -#define current_pt_regs() \ -({ \ - unsigned long sp = (unsigned long)__builtin_frame_address(0); \ - (struct pt_regs *)((sp | (THREAD_SIZE - 1)) + 1) - 1; \ -}) +#define current_pt_regs() ((struct pt_regs *)((current_stack_pointer | (THREAD_SIZE - 1)) + 1) - 1) /* Helpers for working with the user stack pointer */ From fd3cb1bfeb9d98618bd709bfee9c1133e9f189e6 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:23 +0800 Subject: [PATCH 06/18] LoongArch: BPF: Optimize redundant TCC loads in epilogue The legacy epilogue implementation pops the tail call counter (TCC) context via a redundant double-load pattern. It first decrements the load_offset by 2 slots to fetch 'tcc_ptr', and then immediately bumps it back up by 1 slot to load the original 'tcc' value into REG_TCC, unnecessarily overwriting the register. Optimize this sequence by adjusting the load_offset by only 1 slot. This aligns the offset directly with the higher stack slot containing the entry TCC counter (or caller state), allowing us to restore the REG_TCC register safely with a single load. This removes one redundant instruction from the epilogue hot path, improves code readability, and ensures the correct TCC register context is handed back cleanly upon normal return. Cc: stable@vger.kernel.org Fixes: c0fcc955ff82 ("LoongArch: BPF: Fix the tailcall hierarchy") Fixes: ef54c517a937 ("LoongArch: BPF: Implement PROBE_MEM32 pseudo instructions") Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 29c281bef28e..b0c39d45d6a6 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -246,14 +246,8 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset); } - /* - * When push into the stack, follow the order of tcc then tcc_ptr. - * When pop from the stack, first pop tcc_ptr then followed by tcc. - */ - load_offset -= 2 * sizeof(long); - emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); - - load_offset += sizeof(long); + /* Only restore the TCC state into REG_TCC from the higher slot */ + load_offset -= sizeof(long); emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); From cd7e356b07a27e91394838cf3fb655862b519294 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:23 +0800 Subject: [PATCH 07/18] LoongArch: BPF: Move arena register slot below TCC context Currently, the stack layout places the optional arena register slot above the tail call counter context. When arena_vm_start is dynamically enabled, it shifts the relative offset of the tcc_ptr slot within the stack frame, causing hardcoded tracking macros to mismatch and leading to memory misalignment or corruption potentially. To fix this, move the arena register save and restore sequences below the tail call counter context slots in both build_prologue() and the epilogue. Update __build_epilogue() to insert a proper offset decrement to safely skip the unneeded tcc_ptr reading block while accurately aligning with the relocated arena slot at the very bottom. With this patch, the tcc_ptr slot is always positioned at a fixed distance directly underneath the base callee-saved registers that is independent of whether the arena features are on. Cc: stable@vger.kernel.org Fixes: ef54c517a937 ("LoongArch: BPF: Implement PROBE_MEM32 pseudo instructions") Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index b0c39d45d6a6..fd5f8c4e5360 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -124,6 +124,9 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) * | tcc | * +-------------------------+ * | tcc_ptr | + * +-------------------------+ + * | arena | + * | (optional) | * +-------------------------+ <--BPF_REG_FP * | prog->aux->stack_depth | * | (optional) | @@ -145,7 +148,7 @@ static void build_prologue(struct jit_ctx *ctx) stack_adjust += sizeof(long) * 2; if (ctx->arena_vm_start) - stack_adjust += 8; + stack_adjust += sizeof(long); stack_adjust = round_up(stack_adjust, 16); stack_adjust += bpf_stack_adjust; @@ -194,13 +197,13 @@ static void build_prologue(struct jit_ctx *ctx) store_offset -= sizeof(long); emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); + prepare_bpf_tail_call_cnt(ctx, &store_offset); + if (ctx->arena_vm_start) { store_offset -= sizeof(long); emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset); } - prepare_bpf_tail_call_cnt(ctx, &store_offset); - emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); if (bpf_stack_adjust) @@ -241,15 +244,18 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) load_offset -= sizeof(long); emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); + /* Only restore the TCC state into REG_TCC from the higher slot */ + load_offset -= sizeof(long); + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); + + /* Skip the unused local 'tcc_ptr' slot to align with arena */ + load_offset -= sizeof(long); + if (ctx->arena_vm_start) { load_offset -= sizeof(long); emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset); } - /* Only restore the TCC state into REG_TCC from the higher slot */ - load_offset -= sizeof(long); - emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); - emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); if (!is_tail_call) { From 37d545d12f21c4d50612ecaebd7ae1e5bf91b2d8 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:23 +0800 Subject: [PATCH 08/18] LoongArch: BPF: Refactor jump offset calculation in tail call The old macro-based jmp_offset calculation derives the jump distance from a stale prior-pass code stride, which can lead to wrong branch offsets and soft lockups under extra JIT passes. Fix this by calculating the offset directly on the absolute target: "ctx->offset[insn + 1] - ctx->idx". To avoid a false 16-bit range check abort during size estimation, add a "ctx->image == NULL" guard to inject a safe dummy offset. Cc: stable@vger.kernel.org Fixes: cd39d9e6b7e4 ("LoongArch: BPF: Fix jump offset calculation in tailcall") Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index fd5f8c4e5360..846d652b0d22 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -290,17 +290,13 @@ bool bpf_jit_supports_far_kfunc_call(void) static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) { - int off, tc_ninsn = 0; + int off, jmp_offset; int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); u8 a1 = LOONGARCH_GPR_A1; u8 a2 = LOONGARCH_GPR_A2; u8 t1 = LOONGARCH_GPR_T1; u8 t2 = LOONGARCH_GPR_T2; u8 t3 = LOONGARCH_GPR_T3; - const int idx0 = ctx->idx; - -#define cur_offset (ctx->idx - idx0) -#define jmp_offset (tc_ninsn - (cur_offset)) /* * a0: &ctx @@ -310,12 +306,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) * if (index >= array->map.max_entries) * goto out; */ - tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; emit_zext_32(ctx, a2, true); off = offsetof(struct bpf_array, map.max_entries); emit_insn(ctx, ldwu, t1, a1, off); /* bgeu $a2, $t1, jmp_offset */ + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) goto toofar; @@ -326,6 +322,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); emit_insn(ctx, ldd, t3, REG_TCC, 0); emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0) goto toofar; @@ -340,6 +337,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) off = offsetof(struct bpf_array, ptrs); emit_insn(ctx, ldd, t2, t2, off); /* beq $t2, $zero, jmp_offset */ + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) goto toofar; @@ -355,8 +353,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) toofar: pr_info_once("tail_call: jump too far\n"); return -1; -#undef cur_offset -#undef jmp_offset } static void emit_store_stack_imm64(struct jit_ctx *ctx, int reg, int stack_off, u64 imm64) From 434308885db42e7cc929e8466ac9eb7f69a999e0 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:23 +0800 Subject: [PATCH 09/18] LoongArch: BPF: Implement branchless conditional move for TCC The current implementation handles combined bpf2bpf and tail calls by checking at runtime whether REG_TCC holds a scalar count or a pointer address via a conditional jump. This adds branch prediction overhead in the hot path of tail call execution. To implement branchless conditional move, use an unsigned comparison (sltui) combined with mask instructions (maskeqz/masknez) to achieve branchless classification and blending of incoming scalar counts and kernel pointers in REG_TCC. This optimization refactors the inner logic of the helper function, unifies the offset decrement at the function entry, and removes all runtime branching from the prologue hot path completely. Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/inst.h | 6 ++++ arch/loongarch/net/bpf_jit.c | 47 +++++++++---------------------- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 76b723590023..19feae166e1c 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -97,6 +97,7 @@ enum reg2i6_op { }; enum reg2i12_op { + sltui_op = 0x09, addiw_op = 0x0a, addid_op = 0x0b, lu52id_op = 0x0c, @@ -153,6 +154,8 @@ enum reg3_op { addd_op = 0x21, subw_op = 0x22, subd_op = 0x23, + maskeqz_op = 0x26, + masknez_op = 0x27, nor_op = 0x28, and_op = 0x29, or_op = 0x2a, @@ -644,6 +647,7 @@ static inline void emit_##NAME(union loongarch_instruction *insn, \ insn->reg2i12_format.rj = rj; \ } +DEF_EMIT_REG2I12_FORMAT(sltui, sltui_op) DEF_EMIT_REG2I12_FORMAT(addiw, addiw_op) DEF_EMIT_REG2I12_FORMAT(addid, addid_op) DEF_EMIT_REG2I12_FORMAT(lu52id, lu52id_op) @@ -749,6 +753,8 @@ DEF_EMIT_REG3_FORMAT(divd, divd_op) DEF_EMIT_REG3_FORMAT(modd, modd_op) DEF_EMIT_REG3_FORMAT(divdu, divdu_op) DEF_EMIT_REG3_FORMAT(moddu, moddu_op) +DEF_EMIT_REG3_FORMAT(maskeqz, maskeqz_op) +DEF_EMIT_REG3_FORMAT(masknez, masknez_op) DEF_EMIT_REG3_FORMAT(and, and_op) DEF_EMIT_REG3_FORMAT(or, or_op) DEF_EMIT_REG3_FORMAT(xor, xor_op) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 846d652b0d22..f9ba7b961b04 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -52,50 +52,29 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) const struct bpf_prog *prog = ctx->prog; const bool is_main_prog = !bpf_is_subprog(prog); + *store_offset -= sizeof(long); if (is_main_prog) { - /* - * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT - * if (REG_TCC > T3 ) - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * else - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * REG_TCC = LOONGARCH_GPR_SP + store_offset - * - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * - * The purpose of this code is to first push the TCC into stack, - * and then push the address of TCC into stack. - * In cases where bpf2bpf and tailcall are used in combination, - * the value in REG_TCC may be a count or an address, - * these two cases need to be judged and handled separately. - */ - emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); - *store_offset -= sizeof(long); - - emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4); - - /* - * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count, - * push tcc into stack - */ + /* Save entrance TCC state (scalar count or kernel pointer) to local 'tcc' slot */ emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); - /* Push the address of TCC into the REG_TCC */ - emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset); - - emit_uncond_jmp(ctx, 2); + /* Compute the absolute pointer to the local 'tcc' slot */ + emit_insn(ctx, addid, LOONGARCH_GPR_T7, LOONGARCH_GPR_SP, *store_offset); /* - * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address, - * push tcc_ptr into stack + * Branchless classification and blending: + * Combine interleaved inputs between a scalar count (0 to 33) + * and a kernel pointer address without runtime branching. */ - emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); + emit_insn(ctx, sltui, LOONGARCH_GPR_T8, REG_TCC, MAX_TAIL_CALL_CNT + 1); + emit_insn(ctx, maskeqz, LOONGARCH_GPR_T7, LOONGARCH_GPR_T7, LOONGARCH_GPR_T8); + emit_insn(ctx, masknez, REG_TCC, REG_TCC, LOONGARCH_GPR_T8); + emit_insn(ctx, or, REG_TCC, REG_TCC, LOONGARCH_GPR_T7); } else { - *store_offset -= sizeof(long); + /* Subprograms: backup the verified TCC pointer inherited via REG_TCC */ emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); } - /* Push tcc_ptr into stack */ + /* Store the finalized TCC pointer value securely into the local 'tcc_ptr' slot */ *store_offset -= sizeof(long); emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); } From 32a5e3eaa086c64c2ca9a4611769faae60716ae7 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:24 +0800 Subject: [PATCH 10/18] LoongArch: BPF: Remove redundant zext jumping in move_imm() In move_imm(), when an immediate hits the 12-bit unsigned range, an `ori rd, $zero, imm` instruction is emitted. According to the manual, the `ori` instruction inherently performs a logical or with zero-extended immediate operands against $zero, so the upper 32 bits of the destination register `rd` are already 0. However, the existing JIT code unconditionally executes `goto zext;` after `ori`, forcing it to fallthrough into `emit_zext_32()` to clear the upper 32 bits for 32-bit ALU operations. Fix this redundancy by directly returning from the function inside the `is_unsigned_imm12()` block. Acked-by: Hengqi Chen Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h index a8e29be35fa8..bb58c42c2f2a 100644 --- a/arch/loongarch/net/bpf_jit.h +++ b/arch/loongarch/net/bpf_jit.h @@ -156,7 +156,7 @@ static inline void move_imm(struct jit_ctx *ctx, enum loongarch_gpr rd, long imm /* ori rd, $zero, imm_11_0 */ if (is_unsigned_imm12(imm)) { emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm); - goto zext; + return; } /* lu52id rd, $zero, imm_63_52 */ From 252ae40ba91c5da3a792bba7e3dc46b106dec86c Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:24 +0800 Subject: [PATCH 11/18] LoongArch: BPF: Remove dead move_imm() call in BPF_NEG path The BPF_NEG operation is a unary operator that performs `dst = -dst`. The current code unconditionally executes a move_imm() call before the subtraction, generating useless JITted instructions to load data into the temporary register `t1`. This `t1` register is never used anywhere else in the entire BPF_NEG path. Remove this dead `move_imm()` call to avoid useless instructions. Acked-by: Hengqi Chen Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index f9ba7b961b04..a5e902eb646c 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -869,7 +869,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext /* dst = -dst */ case BPF_ALU | BPF_NEG: case BPF_ALU64 | BPF_NEG: - move_imm(ctx, t1, imm, is32); emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); emit_zext_32(ctx, dst, is32); break; From 9fcdba34230a4bbc5d907b5c48f23f95061ceed1 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:39 +0800 Subject: [PATCH 12/18] LoongArch: BPF: Split unconditional branch JA paths statically In build_insn(), both 32-bit and 64-bit unconditional branch JA paths currently share a single case block. It relies on a runtime condition check to multiplex between the 'off' and 'imm' fields. Since the instruction classes are already resolved at compile-time via distinct switch-case labels, this runtime check is redundant. Split the two paths into individual case blocks to remove the redundant runtime check. Acked-by: Hengqi Chen Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index a5e902eb646c..48d368ba0529 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1125,11 +1125,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext /* PC += off */ case BPF_JMP | BPF_JA: + jmp_offset = bpf2la_offset(i, off, ctx); + if (emit_uncond_jmp(ctx, jmp_offset) < 0) + goto toofar; + break; case BPF_JMP32 | BPF_JA: - if (BPF_CLASS(code) == BPF_JMP) - jmp_offset = bpf2la_offset(i, off, ctx); - else - jmp_offset = bpf2la_offset(i, imm, ctx); + jmp_offset = bpf2la_offset(i, imm, ctx); if (emit_uncond_jmp(ctx, jmp_offset) < 0) goto toofar; break; From 576412d98297bd0b3b837c20643ab2e1ba54766f Mon Sep 17 00:00:00 2001 From: Chenguang Zhao Date: Mon, 17 Aug 2026 22:07:39 +0800 Subject: [PATCH 13/18] LoongArch: BPF: Align value-returning atomics with LKMM Per the Linux Kernel Memory Model, value-returning atomic RMW operations must provide sequentially consistent ordering (a full memory barrier). On LoongArch, plain AMO instructions and bare ll/sc loops do not satisfy this requirement by themselves. Update emit_atomic_rmw() to emit barrier-carrying instructions for all value-returning BPF atomics: - BPF_FETCH (ADD/AND/OR/XOR): use am*_db.{b,h,w,d} - BPF_XCHG: use amswap_db.{b,h,w,d} - BPF_CMPXCHG: emit dbar 0x700 after the ll/sc loop, matching __WEAK_LLSC_MB in cmpxchg.h Add the corresponding instruction encodings and emit helpers to inst.h. Non-value-returning RMW ops (plain BPF_ADD, BPF_AND, etc.) are left as weakly ordered, consistent with LKMM. Acked-by: Hengqi Chen Acked-by: Tiezhu Yang Tested-by: Tiezhu Yang Signed-off-by: Chenguang Zhao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/inst.h | 18 +++++++++++++++++ arch/loongarch/net/bpf_jit.c | 32 +++++++++++++++++-------------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 19feae166e1c..54f35bcd0f66 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -202,6 +202,10 @@ enum reg3_op { amswaph_op = 0x70b9, amaddb_op = 0x70ba, amaddh_op = 0x70bb, + amswapdbb_op = 0x70bc, + amswapdbh_op = 0x70bd, + amadddbb_op = 0x70be, + amadddbh_op = 0x70bf, amswapw_op = 0x70c0, amswapd_op = 0x70c1, amaddw_op = 0x70c2, @@ -789,6 +793,20 @@ DEF_EMIT_REG3_FORMAT(amswapb, amswapb_op) DEF_EMIT_REG3_FORMAT(amswaph, amswaph_op) DEF_EMIT_REG3_FORMAT(amswapw, amswapw_op) DEF_EMIT_REG3_FORMAT(amswapd, amswapd_op) +DEF_EMIT_REG3_FORMAT(amswapdbb, amswapdbb_op) +DEF_EMIT_REG3_FORMAT(amswapdbh, amswapdbh_op) +DEF_EMIT_REG3_FORMAT(amadddbb, amadddbb_op) +DEF_EMIT_REG3_FORMAT(amadddbh, amadddbh_op) +DEF_EMIT_REG3_FORMAT(amadddbw, amadddbw_op) +DEF_EMIT_REG3_FORMAT(amadddbd, amadddbd_op) +DEF_EMIT_REG3_FORMAT(amanddbw, amanddbw_op) +DEF_EMIT_REG3_FORMAT(amanddbd, amanddbd_op) +DEF_EMIT_REG3_FORMAT(amordbw, amordbw_op) +DEF_EMIT_REG3_FORMAT(amordbd, amordbd_op) +DEF_EMIT_REG3_FORMAT(amxordbw, amxordbw_op) +DEF_EMIT_REG3_FORMAT(amxordbd, amxordbd_op) +DEF_EMIT_REG3_FORMAT(amswapdbw, amswapdbw_op) +DEF_EMIT_REG3_FORMAT(amswapdbd, amswapdbd_op) #define DEF_EMIT_REG3SA2_FORMAT(NAME, OP) \ static inline void emit_##NAME(union loongarch_instruction *insn, \ diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 48d368ba0529..e11f9c9dbe1a 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -8,6 +8,9 @@ #include #include "bpf_jit.h" +/* DBAR hint for LL/SC completion ordering, see __WEAK_LLSC_MB */ +#define DBAR_LLSC_MB 0x700 + #define LOONGARCH_MAX_REG_ARGS 8 #define LOONGARCH_SAVE_RA_NINSNS 1 @@ -408,7 +411,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) pr_err_once("bpf-jit: amadd.b instruction is not supported\n"); return -EINVAL; } - emit_insn(ctx, amaddb, src, t1, t3); + emit_insn(ctx, amadddbb, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_H: @@ -416,39 +419,39 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) pr_err_once("bpf-jit: amadd.h instruction is not supported\n"); return -EINVAL; } - emit_insn(ctx, amaddh, src, t1, t3); + emit_insn(ctx, amadddbh, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_W: - emit_insn(ctx, amaddw, src, t1, t3); + emit_insn(ctx, amadddbw, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_DW: - emit_insn(ctx, amaddd, src, t1, t3); + emit_insn(ctx, amadddbd, src, t1, t3); break; } break; case BPF_AND | BPF_FETCH: if (isdw) { - emit_insn(ctx, amandd, src, t1, t3); + emit_insn(ctx, amanddbd, src, t1, t3); } else { - emit_insn(ctx, amandw, src, t1, t3); + emit_insn(ctx, amanddbw, src, t1, t3); emit_zext_32(ctx, src, true); } break; case BPF_OR | BPF_FETCH: if (isdw) { - emit_insn(ctx, amord, src, t1, t3); + emit_insn(ctx, amordbd, src, t1, t3); } else { - emit_insn(ctx, amorw, src, t1, t3); + emit_insn(ctx, amordbw, src, t1, t3); emit_zext_32(ctx, src, true); } break; case BPF_XOR | BPF_FETCH: if (isdw) { - emit_insn(ctx, amxord, src, t1, t3); + emit_insn(ctx, amxordbd, src, t1, t3); } else { - emit_insn(ctx, amxorw, src, t1, t3); + emit_insn(ctx, amxordbw, src, t1, t3); emit_zext_32(ctx, src, true); } break; @@ -460,7 +463,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) pr_err_once("bpf-jit: amswap.b instruction is not supported\n"); return -EINVAL; } - emit_insn(ctx, amswapb, src, t1, t3); + emit_insn(ctx, amswapdbb, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_H: @@ -468,15 +471,15 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) pr_err_once("bpf-jit: amswap.h instruction is not supported\n"); return -EINVAL; } - emit_insn(ctx, amswaph, src, t1, t3); + emit_insn(ctx, amswapdbh, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_W: - emit_insn(ctx, amswapw, src, t1, t3); + emit_insn(ctx, amswapdbw, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_DW: - emit_insn(ctx, amswapd, src, t1, t3); + emit_insn(ctx, amswapdbd, src, t1, t3); break; } break; @@ -499,6 +502,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); emit_zext_32(ctx, r0, true); } + emit_insn(ctx, dbar, DBAR_LLSC_MB); break; default: pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm); From 1fec6bbc09e3066a0f39d85741df2a7877c97945 Mon Sep 17 00:00:00 2001 From: Chenguang Zhao Date: Mon, 17 Aug 2026 22:07:39 +0800 Subject: [PATCH 14/18] LoongArch: BPF: Advertise JIT support for kptr xchg inline The BPF verifier can lower bpf_kptr_xchg() to BPF_XCHG when the JIT advertises ptr xchg support. With ordered amswap_db.* emission from the previous patch, declare that LoongArch bpf JIT supports this inlining. Acked-by: Hengqi Chen Acked-by: Tiezhu Yang Tested-by: Tiezhu Yang Signed-off-by: Chenguang Zhao Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index e11f9c9dbe1a..d2ce187d1e23 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -2371,6 +2371,11 @@ bool bpf_jit_supports_fsession(void) return true; } +bool bpf_jit_supports_ptr_xchg(void) +{ + return true; +} + /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ bool bpf_jit_supports_subprog_tailcalls(void) { From 1db6d003e93d033fca25bcfbef73e2780a7e46d7 Mon Sep 17 00:00:00 2001 From: George Guo Date: Mon, 17 Aug 2026 22:07:40 +0800 Subject: [PATCH 15/18] LoongArch: BPF: Resolve per-CPU addrs for internal-only MOV Add support for the internal-only BPF_MOV instruction that resolves the absolute addresses of the per-CPU data from their per-CPU offsets. This instruction is used only for internal inlining optimizations between the BPF verifier and the JITs (e.g. inlining bpf_get_smp_processor_id() and per-CPU map lookups). LoongArch keeps the per-CPU offset of the current CPU in $r21 register (a.k.a. __my_cpu_offset), so resolving a per-CPU address only requires adding $r21 to the source register holding the per-CPU offset. Advertise the capability via bpf_jit_supports_percpu_insn(). Acked-by: Tiezhu Yang Tested-by: Tiezhu Yang Signed-off-by: George Guo Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/inst.h | 1 + arch/loongarch/net/bpf_jit.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 54f35bcd0f66..585667e361c2 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -411,6 +411,7 @@ enum loongarch_gpr { LOONGARCH_GPR_T6, LOONGARCH_GPR_T7, LOONGARCH_GPR_T8, + LOONGARCH_GPR_U0 = 21, /* Kernel per-CPU base register ($r21) */ LOONGARCH_GPR_FP = 22, LOONGARCH_GPR_S0 = 23, LOONGARCH_GPR_S1, diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index d2ce187d1e23..4e56da413752 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -722,6 +722,15 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext move_reg(ctx, dst, t1); break; } + if (insn_is_mov_percpu_addr(insn)) { + if (dst != src) + move_reg(ctx, dst, src); +#ifdef CONFIG_SMP + /* dst += __my_cpu_offset, held in $r21 */ + emit_insn(ctx, addd, dst, dst, LOONGARCH_GPR_U0); +#endif + break; + } switch (off) { case 0: move_reg(ctx, dst, src); @@ -2371,6 +2380,11 @@ bool bpf_jit_supports_fsession(void) return true; } +bool bpf_jit_supports_percpu_insn(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; From 3fbf3534c2ba6d9018dca04f2f96efa9f4b35fe4 Mon Sep 17 00:00:00 2001 From: George Guo Date: Mon, 17 Aug 2026 22:07:40 +0800 Subject: [PATCH 16/18] LoongArch: BPF: Add timed may_goto implementation Implement arch_bpf_timed_may_goto() support and advertise it through bpf_jit_supports_timed_may_goto() so the verifier lowers may_goto into the timed variant: instead of a fixed iteration counter, the loop is bounded by a wall-clock timeout maintained in a per-loop stack slot. arch_bpf_timed_may_goto() uses a custom calling convention: the verifier passes the count/timestamp stack offset in BPF_REG_AX and expects the updated count back in the same register. The JIT call path therefore can skip the usual 'BPF_REG_0 = C return value' move for this helper. Acked-by: Tiezhu Yang Tested-by: Tiezhu Yang Signed-off-by: George Guo Signed-off-by: Huacai Chen --- arch/loongarch/net/Makefile | 2 +- arch/loongarch/net/bpf_jit.c | 13 ++++++- arch/loongarch/net/bpf_timed_may_goto.S | 47 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 arch/loongarch/net/bpf_timed_may_goto.S diff --git a/arch/loongarch/net/Makefile b/arch/loongarch/net/Makefile index 1ec12a0c324a..8d9ddb48f9ea 100644 --- a/arch/loongarch/net/Makefile +++ b/arch/loongarch/net/Makefile @@ -4,4 +4,4 @@ # # Copyright (C) 2022 Loongson Technology Corporation Limited # -obj-$(CONFIG_BPF_JIT) += bpf_jit.o +obj-$(CONFIG_BPF_JIT) += bpf_jit.o bpf_timed_may_goto.o diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 4e56da413752..7533917c6d47 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1192,7 +1192,13 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext move_addr(ctx, t1, func_addr); emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); - if (insn->src_reg != BPF_PSEUDO_CALL) + /* + * Call to arch_bpf_timed_may_goto() uses a custom calling + * convention with the argument and return value in BPF_REG_AX, + * so skip moving the C return value into BPF_REG_0. + */ + if (insn->src_reg != BPF_PSEUDO_CALL && + func_addr != (u64)arch_bpf_timed_may_goto) move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); break; @@ -2396,6 +2402,11 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +bool bpf_jit_supports_timed_may_goto(void) +{ + return true; +} + bool bpf_jit_inlines_helper_call(s32 imm) { switch (imm) { diff --git a/arch/loongarch/net/bpf_timed_may_goto.S b/arch/loongarch/net/bpf_timed_may_goto.S new file mode 100644 index 000000000000..fa128acdb108 --- /dev/null +++ b/arch/loongarch/net/bpf_timed_may_goto.S @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Author: George Guo + * Copyright (C) 2026 KylinSoft Corporation. + */ + +#include +#include +#include +#include + +SYM_FUNC_START(arch_bpf_timed_may_goto) + addi.d sp, sp, -64 + st.d ra, sp, 56 + + /* Save BPF registers R0 - R5 (a5, a0 - a4) */ + st.d a5, sp, 8 + st.d a0, sp, 16 + st.d a1, sp, 24 + st.d a2, sp, 32 + st.d a3, sp, 40 + st.d a4, sp, 48 + + /* + * BPF_REG_AX (t0) holds the offset passed in by the verifier; + * add it to BPF_REG_FP (s4) to get the pointer to the count and + * timestamp, then pass it as the first argument in a0. + * + * The verifier emits a load using FP right before this call, + * so BPF_REG_FP (s4) is always set up by the JIT in this case. + */ + add.d a0, t0, s4 + bl bpf_check_timed_may_goto + /* BPF_REG_AX (t0) will be stored into count, so move the return value to it. */ + move t0, a0 + + ld.d ra, sp, 56 + ld.d a5, sp, 8 + ld.d a0, sp, 16 + ld.d a1, sp, 24 + ld.d a2, sp, 32 + ld.d a3, sp, 40 + ld.d a4, sp, 48 + addi.d sp, sp, 64 + + jr ra +SYM_FUNC_END(arch_bpf_timed_may_goto) From e7103e5e3bde91e573f19bd70aed359fd7e17d5a Mon Sep 17 00:00:00 2001 From: George Guo Date: Mon, 17 Aug 2026 22:07:40 +0800 Subject: [PATCH 17/18] LoongArch: BPF: Add arch_bpf_stack_walk() implementation Implement arch_bpf_stack_walk() on top of the ORC unwinder within the LoongArch BPF JIT backend to provide generic BPF stack walking capabilities. This function is required by advanced BPF features, including timed may_goto timeout tracing and BPF exceptions. It will be invoked in the BPF core subsystem unwinding paths: bpf_prog_find_from_stack(), bpf_stream_stage_dump_stack(), and bpf_throw(). Co-developed-by: Tiezhu Yang Signed-off-by: Tiezhu Yang Signed-off-by: George Guo Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.c | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 7533917c6d47..1eb588e443c9 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -2366,6 +2366,44 @@ void bpf_jit_free(struct bpf_prog *prog) bpf_prog_unlock_free(prog); } +#if defined(CONFIG_UNWINDER_ORC) +#include + +static noinline void walk_bpf_stackframe(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), + void *cookie, unsigned long fp) +{ + unsigned long addr; + struct unwind_state state; + struct pt_regs dummyregs; + struct pt_regs *regs = &dummyregs; + + regs->regs[1] = 0; + regs->regs[22] = fp; + regs->regs[3] = (unsigned long)__builtin_frame_address(0); + regs->csr_era = (unsigned long)__builtin_return_address(0); + + for (unwind_start(&state, current, regs); + !unwind_done(&state); unwind_next_frame(&state)) { + addr = unwind_get_return_address(&state); + if (!addr || !consume_fn(cookie, (u64)addr, (u64)state.sp, (u64)state.fp)) + break; + } +} + +void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) +{ + unsigned long fp; + + /* + * Capture the live frame pointer ($r22) at the very front-line before + * any kernel C code clobbers it. This must be a thin wrapper with no + * large stack locals to prevent the compiler from reusing $r22 early. + */ + asm volatile("move %0, $r22" : "=r"(fp)); + walk_bpf_stackframe(consume_fn, cookie, fp); +} +#endif /* CONFIG_UNWINDER_ORC */ + bool bpf_jit_bypass_spec_v1(void) { return true; From 6afb03599653619e661121628611f808de1f7c7a Mon Sep 17 00:00:00 2001 From: Chenguang Zhao Date: Mon, 17 Aug 2026 22:07:49 +0800 Subject: [PATCH 18/18] selftests/bpf: Enable kptr_xchg_inline test on LoongArch Enable the kptr_xchg_inline functional test on LoongArch64 now that the BPF JIT can inline bpf_kptr_xchg() with correct memory ordering. Acked-by: Hengqi Chen Acked-by: Tiezhu Yang Tested-by: Tiezhu Yang Signed-off-by: Chenguang Zhao Signed-off-by: Huacai Chen --- tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c b/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c index 7def158da9eb..1215d6edd590 100644 --- a/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c +++ b/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c @@ -14,7 +14,8 @@ void test_kptr_xchg_inline(void) int err; #if !(defined(__x86_64__) || defined(__aarch64__) || \ - (defined(__riscv) && __riscv_xlen == 64)) + (defined(__riscv) && __riscv_xlen == 64) || \ + (defined(__loongarch__) && __loongarch_grlen == 64)) test__skip(); return; #endif