mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 09:20:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'loongarch-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson

Browse Source 
Pull LoongArch updates from Huacai Chen:

 - Add DIRECT_MAP_PHYSMEM_END definition

 - Expand module virtual address space to 2GB

 - Use current_stack_pointer in current_pt_regs()

 - BPF JIT enhancements such as kptr_xchg and timed may_goto

 - Some bug fixes and other small changes

* tag 'loongarch-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson:
  selftests/bpf: Enable kptr_xchg_inline test on LoongArch
  LoongArch: BPF: Add arch_bpf_stack_walk() implementation
  LoongArch: BPF: Add timed may_goto implementation
  LoongArch: BPF: Resolve per-CPU addrs for internal-only MOV
  LoongArch: BPF: Advertise JIT support for kptr xchg inline
  LoongArch: BPF: Align value-returning atomics with LKMM
  LoongArch: BPF: Split unconditional branch JA paths statically
  LoongArch: BPF: Remove dead move_imm() call in BPF_NEG path
  LoongArch: BPF: Remove redundant zext jumping in move_imm()
  LoongArch: BPF: Implement branchless conditional move for TCC
  LoongArch: BPF: Refactor jump offset calculation in tail call
  LoongArch: BPF: Move arena register slot below TCC context
  LoongArch: BPF: Optimize redundant TCC loads in epilogue
  LoongArch: Use current_stack_pointer in current_pt_regs()
  LoongArch: Use generic cmp_int() instead of custom cmp_3way()
  LoongArch: Expand module virtual address space to 2GB
  LoongArch: Add DIRECT_MAP_PHYSMEM_END definition
  LoongArch: Fix acpi_package_ids[] array overflow
This commit is contained in:
Linus Torvalds
2026-08-20 14:46:50 -07:00
parent 55ab7e1422 6afb035996
commit caf325ab81
11 changed files with 225 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
arch/loongarch/include/asm/inst.h
View File

@@ -97,6 +97,7 @@ enum reg2i6_op {
};
enum reg2i12_op {
sltui_op = 0x09,
addiw_op = 0x0a,
addid_op = 0x0b,
lu52id_op = 0x0c,
@@ -153,6 +154,8 @@ enum reg3_op {
addd_op = 0x21,
subw_op = 0x22,
subd_op = 0x23,
maskeqz_op = 0x26,
masknez_op = 0x27,
nor_op = 0x28,
and_op = 0x29,
or_op = 0x2a,
@@ -199,6 +202,10 @@ enum reg3_op {
amswaph_op = 0x70b9,
amaddb_op = 0x70ba,
amaddh_op = 0x70bb,
amswapdbb_op = 0x70bc,
amswapdbh_op = 0x70bd,
amadddbb_op = 0x70be,
amadddbh_op = 0x70bf,
amswapw_op = 0x70c0,
amswapd_op = 0x70c1,
amaddw_op = 0x70c2,
@@ -404,6 +411,7 @@ enum loongarch_gpr {
LOONGARCH_GPR_T6,
LOONGARCH_GPR_T7,
LOONGARCH_GPR_T8,
LOONGARCH_GPR_U0 = 21, /* Kernel per-CPU base register ($r21) */
LOONGARCH_GPR_FP = 22,
LOONGARCH_GPR_S0 = 23,
LOONGARCH_GPR_S1,
@@ -644,6 +652,7 @@ static inline void emit_##NAME(union loongarch_instruction *insn, \
insn->reg2i12_format.rj = rj; \
}
DEF_EMIT_REG2I12_FORMAT(sltui, sltui_op)
DEF_EMIT_REG2I12_FORMAT(addiw, addiw_op)
DEF_EMIT_REG2I12_FORMAT(addid, addid_op)
DEF_EMIT_REG2I12_FORMAT(lu52id, lu52id_op)
@@ -749,6 +758,8 @@ DEF_EMIT_REG3_FORMAT(divd, divd_op)
DEF_EMIT_REG3_FORMAT(modd, modd_op)
DEF_EMIT_REG3_FORMAT(divdu, divdu_op)
DEF_EMIT_REG3_FORMAT(moddu, moddu_op)
DEF_EMIT_REG3_FORMAT(maskeqz, maskeqz_op)
DEF_EMIT_REG3_FORMAT(masknez, masknez_op)
DEF_EMIT_REG3_FORMAT(and, and_op)
DEF_EMIT_REG3_FORMAT(or, or_op)
DEF_EMIT_REG3_FORMAT(xor, xor_op)
@@ -783,6 +794,20 @@ DEF_EMIT_REG3_FORMAT(amswapb, amswapb_op)
DEF_EMIT_REG3_FORMAT(amswaph, amswaph_op)
DEF_EMIT_REG3_FORMAT(amswapw, amswapw_op)
DEF_EMIT_REG3_FORMAT(amswapd, amswapd_op)
DEF_EMIT_REG3_FORMAT(amswapdbb, amswapdbb_op)
DEF_EMIT_REG3_FORMAT(amswapdbh, amswapdbh_op)
DEF_EMIT_REG3_FORMAT(amadddbb, amadddbb_op)
DEF_EMIT_REG3_FORMAT(amadddbh, amadddbh_op)
DEF_EMIT_REG3_FORMAT(amadddbw, amadddbw_op)
DEF_EMIT_REG3_FORMAT(amadddbd, amadddbd_op)
DEF_EMIT_REG3_FORMAT(amanddbw, amanddbw_op)
DEF_EMIT_REG3_FORMAT(amanddbd, amanddbd_op)
DEF_EMIT_REG3_FORMAT(amordbw, amordbw_op)
DEF_EMIT_REG3_FORMAT(amordbd, amordbd_op)
DEF_EMIT_REG3_FORMAT(amxordbw, amxordbw_op)
DEF_EMIT_REG3_FORMAT(amxordbd, amxordbd_op)
DEF_EMIT_REG3_FORMAT(amswapdbw, amswapdbw_op)
DEF_EMIT_REG3_FORMAT(amswapdbd, amswapdbd_op)
#define DEF_EMIT_REG3SA2_FORMAT(NAME, OP) \
static inline void emit_##NAME(union loongarch_instruction *insn, \

9
arch/loongarch/include/asm/pgtable.h
View File

@@ -96,7 +96,7 @@ struct vm_area_struct;
#ifdef CONFIG_64BIT
#define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE))
#define MODULES_END (MODULES_VADDR + SZ_256M)
#define MODULES_END (MODULES_VADDR + SZ_2G) /* 256MB for text, rest for data */
#ifdef CONFIG_KFENCE
#define KFENCE_AREA_SIZE (((CONFIG_KFENCE_NUM_OBJECTS + 1) * 2 + 2) * PAGE_SIZE)
@@ -125,6 +125,13 @@ struct vm_area_struct;
#endif
/* Needed to limit get_free_mem_region() */
#ifndef CONFIG_SPARSEMEM
#define DIRECT_MAP_PHYSMEM_END ((1ULL << (cpu_pabits + 1)) - 1)
#else
#define DIRECT_MAP_PHYSMEM_END min((1ULL << (cpu_pabits + 1)) - 1, (1ULL << MAX_PHYSMEM_BITS) - 1)
#endif
#define ptep_get(ptep) READ_ONCE(*(ptep))
#define pmdp_get(pmdp) READ_ONCE(*(pmdp))

6
arch/loongarch/include/asm/ptrace.h
View File

@@ -170,11 +170,7 @@ static inline void die_if_kernel(const char *str, struct pt_regs *regs)
die(str, regs);
}
#define current_pt_regs() \
({ \
unsigned long sp = (unsigned long)__builtin_frame_address(0); \
(struct pt_regs *)((sp | (THREAD_SIZE - 1)) + 1) - 1; \
})
#define current_pt_regs() ((struct pt_regs *)((current_stack_pointer | (THREAD_SIZE - 1)) + 1) - 1)
/* Helpers for working with the user stack pointer */

3
arch/loongarch/kernel/acpi.c
View File

@@ -13,6 +13,7 @@
#include <linux/export.h>
#include <linux/irq.h>
#include <linux/irqdomain.h>
#include <linux/kvm_host.h>
#include <linux/memblock.h>
#include <linux/of_fdt.h>
#include <linux/serial_core.h>
@@ -202,7 +203,7 @@ static void __init acpi_process_madt(void)
int pptt_enabled;
static int acpi_nr_packages;
static int acpi_package_ids[MAX_PACKAGES];
static int acpi_package_ids[MAX(MAX_PACKAGES, KVM_MAX_VCPUS)];
int __init parse_acpi_topology(void)
{

6
arch/loongarch/kernel/module-sections.c
View File

@@ -62,16 +62,14 @@ Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Shdr *sechdrs, Elf_Addr v
return (Elf_Addr)&plt[nr];
}
#define cmp_3way(a, b) ((a) < (b) ? -1 : (a) > (b))
static int compare_rela(const void *x, const void *y)
{
int ret;
const Elf_Rela *rela_x = x, *rela_y = y;
ret = cmp_3way(rela_x->r_info, rela_y->r_info);
ret = cmp_int(rela_x->r_info, rela_y->r_info);
if (ret == 0)
ret = cmp_3way(rela_x->r_addend, rela_y->r_addend);
ret = cmp_int(rela_x->r_addend, rela_y->r_addend);
return ret;
}

17
arch/loongarch/mm/init.c
View File

@@ -237,15 +237,26 @@ pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
EXPORT_SYMBOL(invalid_pte_table);
#if defined(CONFIG_EXECMEM) && defined(MODULES_VADDR)
#define MODULES_TEXT_START (MODULES_VADDR)
#define MODULES_TEXT_END (MODULES_VADDR + SZ_256M)
#define MODULES_DATA_START (MODULES_VADDR + SZ_256M)
#define MODULES_DATA_END (MODULES_END)
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
[EXECMEM_MODULE_TEXT] = {
.start = MODULES_TEXT_START,
.end = MODULES_TEXT_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
[EXECMEM_MODULE_DATA] = {
.start = MODULES_DATA_START,
.end = MODULES_DATA_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},

2
arch/loongarch/net/Makefile
View File

@@ -4,4 +4,4 @@
#
# Copyright (C) 2022 Loongson Technology Corporation Limited
#
obj-$(CONFIG_BPF_JIT) += bpf_jit.o
obj-$(CONFIG_BPF_JIT) += bpf_jit.o bpf_timed_may_goto.o

197
arch/loongarch/net/bpf_jit.c
View File

@@ -8,6 +8,9 @@
#include <asm/asm-offsets.h>
#include "bpf_jit.h"
/* DBAR hint for LL/SC completion ordering, see __WEAK_LLSC_MB */
#define DBAR_LLSC_MB 0x700
#define LOONGARCH_MAX_REG_ARGS 8
#define LOONGARCH_SAVE_RA_NINSNS 1
@@ -52,50 +55,29 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
const struct bpf_prog *prog = ctx->prog;
const bool is_main_prog = !bpf_is_subprog(prog);
*store_offset -= sizeof(long);
if (is_main_prog) {
/*
* LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT
* if (REG_TCC > T3 )
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
* else
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
* REG_TCC = LOONGARCH_GPR_SP + store_offset
*
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
*
* The purpose of this code is to first push the TCC into stack,
* and then push the address of TCC into stack.
* In cases where bpf2bpf and tailcall are used in combination,
* the value in REG_TCC may be a count or an address,
* these two cases need to be judged and handled separately.
*/
emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
*store_offset -= sizeof(long);
emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4);
/*
* If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count,
* push tcc into stack
*/
/* Save entrance TCC state (scalar count or kernel pointer) to local 'tcc' slot */
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
/* Push the address of TCC into the REG_TCC */
emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
emit_uncond_jmp(ctx, 2);
/* Compute the absolute pointer to the local 'tcc' slot */
emit_insn(ctx, addid, LOONGARCH_GPR_T7, LOONGARCH_GPR_SP, *store_offset);
/*
* If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address,
* push tcc_ptr into stack
* Branchless classification and blending:
* Combine interleaved inputs between a scalar count (0 to 33)
* and a kernel pointer address without runtime branching.
*/
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
emit_insn(ctx, sltui, LOONGARCH_GPR_T8, REG_TCC, MAX_TAIL_CALL_CNT + 1);
emit_insn(ctx, maskeqz, LOONGARCH_GPR_T7, LOONGARCH_GPR_T7, LOONGARCH_GPR_T8);
emit_insn(ctx, masknez, REG_TCC, REG_TCC, LOONGARCH_GPR_T8);
emit_insn(ctx, or, REG_TCC, REG_TCC, LOONGARCH_GPR_T7);
} else {
*store_offset -= sizeof(long);
/* Subprograms: backup the verified TCC pointer inherited via REG_TCC */
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
}
/* Push tcc_ptr into stack */
/* Store the finalized TCC pointer value securely into the local 'tcc_ptr' slot */
*store_offset -= sizeof(long);
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
}
@@ -124,6 +106,9 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
* | tcc |
* +-------------------------+
* | tcc_ptr |
* +-------------------------+
* | arena |
* | (optional) |
* +-------------------------+ <--BPF_REG_FP
* | prog->aux->stack_depth |
* | (optional) |
@@ -145,7 +130,7 @@ static void build_prologue(struct jit_ctx *ctx)
stack_adjust += sizeof(long) * 2;
if (ctx->arena_vm_start)
stack_adjust += 8;
stack_adjust += sizeof(long);
stack_adjust = round_up(stack_adjust, 16);
stack_adjust += bpf_stack_adjust;
@@ -194,13 +179,13 @@ static void build_prologue(struct jit_ctx *ctx)
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
prepare_bpf_tail_call_cnt(ctx, &store_offset);
if (ctx->arena_vm_start) {
store_offset -= sizeof(long);
emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset);
}
prepare_bpf_tail_call_cnt(ctx, &store_offset);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
if (bpf_stack_adjust)
@@ -241,21 +226,18 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
/* Only restore the TCC state into REG_TCC from the higher slot */
load_offset -= sizeof(long);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
/* Skip the unused local 'tcc_ptr' slot to align with arena */
load_offset -= sizeof(long);
if (ctx->arena_vm_start) {
load_offset -= sizeof(long);
emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset);
}
/*
* When push into the stack, follow the order of tcc then tcc_ptr.
* When pop from the stack, first pop tcc_ptr then followed by tcc.
*/
load_offset -= 2 * sizeof(long);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
load_offset += sizeof(long);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
if (!is_tail_call) {
@@ -290,17 +272,13 @@ bool bpf_jit_supports_far_kfunc_call(void)
static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
{
int off, tc_ninsn = 0;
int off, jmp_offset;
int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
u8 a1 = LOONGARCH_GPR_A1;
u8 a2 = LOONGARCH_GPR_A2;
u8 t1 = LOONGARCH_GPR_T1;
u8 t2 = LOONGARCH_GPR_T2;
u8 t3 = LOONGARCH_GPR_T3;
const int idx0 = ctx->idx;
#define cur_offset (ctx->idx - idx0)
#define jmp_offset (tc_ninsn - (cur_offset))
/*
* a0: &ctx
@@ -310,12 +288,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
* if (index >= array->map.max_entries)
* goto out;
*/
tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0];
emit_zext_32(ctx, a2, true);
off = offsetof(struct bpf_array, map.max_entries);
emit_insn(ctx, ldwu, t1, a1, off);
/* bgeu $a2, $t1, jmp_offset */
jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0;
if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
goto toofar;
@@ -326,6 +304,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
emit_insn(ctx, ldd, t3, REG_TCC, 0);
emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0;
if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0)
goto toofar;
@@ -340,6 +319,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
off = offsetof(struct bpf_array, ptrs);
emit_insn(ctx, ldd, t2, t2, off);
/* beq $t2, $zero, jmp_offset */
jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0;
if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
goto toofar;
@@ -355,8 +335,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
toofar:
pr_info_once("tail_call: jump too far\n");
return -1;
#undef cur_offset
#undef jmp_offset
}
static void emit_store_stack_imm64(struct jit_ctx *ctx, int reg, int stack_off, u64 imm64)
@@ -433,7 +411,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amadd.b instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amaddb, src, t1, t3);
emit_insn(ctx, amadddbb, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_H:
@@ -441,39 +419,39 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amadd.h instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amaddh, src, t1, t3);
emit_insn(ctx, amadddbh, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_W:
emit_insn(ctx, amaddw, src, t1, t3);
emit_insn(ctx, amadddbw, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_DW:
emit_insn(ctx, amaddd, src, t1, t3);
emit_insn(ctx, amadddbd, src, t1, t3);
break;
}
break;
case BPF_AND | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amandd, src, t1, t3);
emit_insn(ctx, amanddbd, src, t1, t3);
} else {
emit_insn(ctx, amandw, src, t1, t3);
emit_insn(ctx, amanddbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
case BPF_OR | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amord, src, t1, t3);
emit_insn(ctx, amordbd, src, t1, t3);
} else {
emit_insn(ctx, amorw, src, t1, t3);
emit_insn(ctx, amordbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
case BPF_XOR | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amxord, src, t1, t3);
emit_insn(ctx, amxordbd, src, t1, t3);
} else {
emit_insn(ctx, amxorw, src, t1, t3);
emit_insn(ctx, amxordbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
@@ -485,7 +463,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amswap.b instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amswapb, src, t1, t3);
emit_insn(ctx, amswapdbb, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_H:
@@ -493,15 +471,15 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amswap.h instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amswaph, src, t1, t3);
emit_insn(ctx, amswapdbh, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_W:
emit_insn(ctx, amswapw, src, t1, t3);
emit_insn(ctx, amswapdbw, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_DW:
emit_insn(ctx, amswapd, src, t1, t3);
emit_insn(ctx, amswapdbd, src, t1, t3);
break;
}
break;
@@ -524,6 +502,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
emit_zext_32(ctx, r0, true);
}
emit_insn(ctx, dbar, DBAR_LLSC_MB);
break;
default:
pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm);
@@ -743,6 +722,15 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
move_reg(ctx, dst, t1);
break;
}
if (insn_is_mov_percpu_addr(insn)) {
if (dst != src)
move_reg(ctx, dst, src);
#ifdef CONFIG_SMP
/* dst += __my_cpu_offset, held in $r21 */
emit_insn(ctx, addd, dst, dst, LOONGARCH_GPR_U0);
#endif
break;
}
switch (off) {
case 0:
move_reg(ctx, dst, src);
@@ -894,7 +882,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
/* dst = -dst */
case BPF_ALU | BPF_NEG:
case BPF_ALU64 | BPF_NEG:
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
emit_zext_32(ctx, dst, is32);
break;
@@ -1151,11 +1138,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
/* PC += off */
case BPF_JMP | BPF_JA:
jmp_offset = bpf2la_offset(i, off, ctx);
if (emit_uncond_jmp(ctx, jmp_offset) < 0)
goto toofar;
break;
case BPF_JMP32 | BPF_JA:
if (BPF_CLASS(code) == BPF_JMP)
jmp_offset = bpf2la_offset(i, off, ctx);
else
jmp_offset = bpf2la_offset(i, imm, ctx);
jmp_offset = bpf2la_offset(i, imm, ctx);
if (emit_uncond_jmp(ctx, jmp_offset) < 0)
goto toofar;
break;
@@ -1204,7 +1192,13 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
move_addr(ctx, t1, func_addr);
emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0);
if (insn->src_reg != BPF_PSEUDO_CALL)
/*
* Call to arch_bpf_timed_may_goto() uses a custom calling
* convention with the argument and return value in BPF_REG_AX,
* so skip moving the C return value into BPF_REG_0.
*/
if (insn->src_reg != BPF_PSEUDO_CALL &&
func_addr != (u64)arch_bpf_timed_may_goto)
move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
break;
@@ -2372,6 +2366,44 @@ void bpf_jit_free(struct bpf_prog *prog)
bpf_prog_unlock_free(prog);
}
#if defined(CONFIG_UNWINDER_ORC)
#include <asm/unwind.h>
static noinline void walk_bpf_stackframe(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp),
void *cookie, unsigned long fp)
{
unsigned long addr;
struct unwind_state state;
struct pt_regs dummyregs;
struct pt_regs *regs = &dummyregs;
regs->regs[1] = 0;
regs->regs[22] = fp;
regs->regs[3] = (unsigned long)__builtin_frame_address(0);
regs->csr_era = (unsigned long)__builtin_return_address(0);
for (unwind_start(&state, current, regs);
!unwind_done(&state); unwind_next_frame(&state)) {
addr = unwind_get_return_address(&state);
if (!addr || !consume_fn(cookie, (u64)addr, (u64)state.sp, (u64)state.fp))
break;
}
}
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)
{
unsigned long fp;
/*
* Capture the live frame pointer ($r22) at the very front-line before
* any kernel C code clobbers it. This must be a thin wrapper with no
* large stack locals to prevent the compiler from reusing $r22 early.
*/
asm volatile("move %0, $r22" : "=r"(fp));
walk_bpf_stackframe(consume_fn, cookie, fp);
}
#endif /* CONFIG_UNWINDER_ORC */
bool bpf_jit_bypass_spec_v1(void)
{
return true;
@@ -2392,12 +2424,27 @@ bool bpf_jit_supports_fsession(void)
return true;
}
bool bpf_jit_supports_percpu_insn(void)
{
return true;
}
bool bpf_jit_supports_ptr_xchg(void)
{
return true;
}
/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
bool bpf_jit_supports_subprog_tailcalls(void)
{
return true;
}
bool bpf_jit_supports_timed_may_goto(void)
{
return true;
}
bool bpf_jit_inlines_helper_call(s32 imm)
{
switch (imm) {

2
arch/loongarch/net/bpf_jit.h
View File

@@ -156,7 +156,7 @@ static inline void move_imm(struct jit_ctx *ctx, enum loongarch_gpr rd, long imm
/* ori rd, $zero, imm_11_0 */
if (is_unsigned_imm12(imm)) {
emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm);
goto zext;
return;
}
/* lu52id rd, $zero, imm_63_52 */

47
arch/loongarch/net/bpf_timed_may_goto.S Normal file
View File

@@ -0,0 +1,47 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Author: George Guo <guodongtai@kylinos.cn>
* Copyright (C) 2026 KylinSoft Corporation.
*/
#include <asm/asmmacro.h>
#include <asm/regdef.h>
#include <linux/export.h>
#include <linux/linkage.h>
SYM_FUNC_START(arch_bpf_timed_may_goto)
addi.d sp, sp, -64
st.d ra, sp, 56
/* Save BPF registers R0 - R5 (a5, a0 - a4) */
st.d a5, sp, 8
st.d a0, sp, 16
st.d a1, sp, 24
st.d a2, sp, 32
st.d a3, sp, 40
st.d a4, sp, 48
/*
* BPF_REG_AX (t0) holds the offset passed in by the verifier;
* add it to BPF_REG_FP (s4) to get the pointer to the count and
* timestamp, then pass it as the first argument in a0.
*
* The verifier emits a load using FP right before this call,
* so BPF_REG_FP (s4) is always set up by the JIT in this case.
*/
add.d a0, t0, s4
bl bpf_check_timed_may_goto
/* BPF_REG_AX (t0) will be stored into count, so move the return value to it. */
move t0, a0
ld.d ra, sp, 56
ld.d a5, sp, 8
ld.d a0, sp, 16
ld.d a1, sp, 24
ld.d a2, sp, 32
ld.d a3, sp, 40
ld.d a4, sp, 48
addi.d sp, sp, 64
jr ra
SYM_FUNC_END(arch_bpf_timed_may_goto)

3
tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c
View File

@@ -14,7 +14,8 @@ void test_kptr_xchg_inline(void)
int err;
#if !(defined(__x86_64__) || defined(__aarch64__) || \
(defined(__riscv) && __riscv_xlen == 64))
(defined(__riscv) && __riscv_xlen == 64) || \
(defined(__loongarch__) && __loongarch_grlen == 64))
test__skip();
return;
#endif