mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-05-09 22:14:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

x86/mm: Use INVLPGB for kernel TLB flushes

Browse Source 
Use broadcast TLB invalidation for kernel addresses when available.
Remove the need to send IPIs for kernel TLB flushes.

   [ bp: Integrate dhansen's comments additions, merge the
     flush_tlb_all() change into this one too. ]

Signed-off-by: Rik van Riel <riel@surriel.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20250226030129.530345-5-riel@surriel.com
This commit is contained in:
Rik van Riel
2025-02-25 22:00:39 -05:00
committed by Ingo Molnar
parent b7aa05cbdc
commit 82378c6c2f
1 changed files with 44 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
arch/x86/mm/tlb.c
View File

@@ -1064,7 +1064,6 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
}
static void do_flush_tlb_all(void *info)
{
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
@@ -1074,7 +1073,32 @@ static void do_flush_tlb_all(void *info)
void flush_tlb_all(void)
{
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
on_each_cpu(do_flush_tlb_all, NULL, 1);
/* First try (faster) hardware-assisted TLB invalidation. */
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_flush_all();
else
/* Fall back to the IPI-based invalidation. */
on_each_cpu(do_flush_tlb_all, NULL, 1);
}
/* Flush an arbitrarily large range of memory with INVLPGB. */
static void invlpgb_kernel_range_flush(struct flush_tlb_info *info)
{
unsigned long addr, nr;
for (addr = info->start; addr < info->end; addr += nr << PAGE_SHIFT) {
nr = (info->end - addr) >> PAGE_SHIFT;
/*
* INVLPGB has a limit on the size of ranges it can
* flush. Break up large flushes.
*/
nr = clamp_val(nr, 1, invlpgb_count_max);
invlpgb_flush_addr_nosync(addr, nr);
}
__tlbsync();
}
static void do_kernel_range_flush(void *info)
@@ -1087,6 +1111,22 @@ static void do_kernel_range_flush(void *info)
flush_tlb_one_kernel(addr);
}
static void kernel_tlb_flush_all(struct flush_tlb_info *info)
{
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_flush_all();
else
on_each_cpu(do_flush_tlb_all, NULL, 1);
}
static void kernel_tlb_flush_range(struct flush_tlb_info *info)
{
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_kernel_range_flush(info);
else
on_each_cpu(do_kernel_range_flush, info, 1);
}
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
struct flush_tlb_info *info;
@@ -1097,9 +1137,9 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
TLB_GENERATION_INVALID);
if (info->end == TLB_FLUSH_ALL)
on_each_cpu(do_flush_tlb_all, NULL, 1);
kernel_tlb_flush_all(info);
else
on_each_cpu(do_kernel_range_flush, info, 1);
kernel_tlb_flush_range(info);
put_flush_tlb_info();
}