diff --git a/MAINTAINERS b/MAINTAINERS index c1509e331ff5..8ee9446da9df 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16887,16 +16887,6 @@ F: mm/mm_init.c F: mm/rodata_test.c F: tools/testing/memblock/ -MEMORY ALLOCATION PROFILING -M: Suren Baghdasaryan -R: Hao Ge -L: linux-mm@kvack.org -S: Maintained -F: Documentation/mm/allocation-profiling.rst -F: include/linux/alloc_tag.h -F: include/linux/pgalloc_tag.h -F: lib/alloc_tag.c - MEMORY CONTROLLER DRIVERS M: Krzysztof Kozlowski L: linux-kernel@vger.kernel.org @@ -16941,6 +16931,16 @@ T: quilt git://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new F: mm/ F: tools/mm/ +MEMORY MANAGEMENT - ALLOCATION PROFILING (ALLOC TAG) +M: Suren Baghdasaryan +R: Hao Ge +L: linux-mm@kvack.org +S: Maintained +F: Documentation/mm/allocation-profiling.rst +F: include/linux/alloc_tag.h +F: include/linux/pgalloc_tag.h +F: mm/alloc_tag.c + MEMORY MANAGEMENT - BALLOON M: Andrew Morton M: David Hildenbrand diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1244dcac2294..b82515cde538 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1048,34 +1048,6 @@ config CODE_TAGGING bool select KALLSYMS -config MEM_ALLOC_PROFILING - bool "Enable memory allocation profiling" - default n - depends on MMU - depends on PROC_FS - depends on !DEBUG_FORCE_WEAK_PER_CPU - select CODE_TAGGING - select PAGE_EXTENSION - select SLAB_OBJ_EXT - help - Track allocation source code and record total allocation size - initiated at that code location. The mechanism can be used to track - memory leaks with a low performance and memory impact. - -config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - bool "Enable memory allocation profiling by default" - default y - depends on MEM_ALLOC_PROFILING - -config MEM_ALLOC_PROFILING_DEBUG - bool "Memory allocation profiler debugging" - default n - depends on MEM_ALLOC_PROFILING - select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - help - Adds warnings with helpful error messages for memory allocation - profiling. - source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" source "lib/Kconfig.kmsan" diff --git a/lib/Makefile b/lib/Makefile index 7f75cc6edf94..531a0be88062 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -196,7 +196,6 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CODE_TAGGING) += codetag.o -obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o lib-$(CONFIG_GENERIC_BUG) += bug.o diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 91b3e027b753..5737a504efbb 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -320,3 +320,31 @@ config PER_VMA_LOCK_STATS overhead in the page fault path. If in doubt, say N. + +config MEM_ALLOC_PROFILING + bool "Enable memory allocation profiling" + default n + depends on MMU + depends on PROC_FS + depends on !DEBUG_FORCE_WEAK_PER_CPU + select CODE_TAGGING + select PAGE_EXTENSION + select SLAB_OBJ_EXT + help + Track allocation source code and record total allocation size + initiated at that code location. The mechanism can be used to track + memory leaks with a low performance and memory impact. + +config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + bool "Enable memory allocation profiling by default" + default y + depends on MEM_ALLOC_PROFILING + +config MEM_ALLOC_PROFILING_DEBUG + bool "Memory allocation profiler debugging" + default n + depends on MEM_ALLOC_PROFILING + select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + help + Adds warnings with helpful error messages for memory allocation + profiling. diff --git a/mm/Makefile b/mm/Makefile index eff9f9e7e061..4fc713867b9b 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c new file mode 100644 index 000000000000..e5b218176c5a --- /dev/null +++ b/mm/alloc_tag.c @@ -0,0 +1,1029 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ALLOCINFO_FILE_NAME "allocinfo" +#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag)) +#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME) +#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME) + +#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT +static bool mem_profiling_support = true; +#else +static bool mem_profiling_support; +#endif + +/* + * Memory allocation profiling is permanently disabled and cannot be enabled. + * Must be called after setup_early_mem_profiling(). + */ +bool mem_alloc_profiling_permanently_disabled(void) +{ + return !mem_profiling_support; +} + +static struct codetag_type *alloc_tag_cttype; + +#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU +DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); +EXPORT_SYMBOL(_shared_alloc_tag); +#endif + +DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, + mem_alloc_profiling_key); +EXPORT_SYMBOL(mem_alloc_profiling_key); + +DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed); + +struct alloc_tag_kernel_section kernel_tags = { NULL, 0 }; +unsigned long alloc_tag_ref_mask; +int alloc_tag_ref_offs; + +struct allocinfo_private { + struct codetag_iterator iter; + struct codetag_iterator reported_iter; + bool print_header; +}; + +static void *allocinfo_start(struct seq_file *m, loff_t *pos) +{ + struct allocinfo_private *priv; + loff_t node = *pos; + + priv = (struct allocinfo_private *)m->private; + codetag_lock_module_list(alloc_tag_cttype); + if (node == 0) { + priv->print_header = true; + priv->iter = codetag_get_ct_iter(alloc_tag_cttype); + } else { + priv->iter = priv->reported_iter; + } + codetag_next_ct(&priv->iter); + return priv->iter.ct ? priv : NULL; +} + +static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) +{ + struct allocinfo_private *priv = (struct allocinfo_private *)arg; + struct codetag *ct; + + priv->reported_iter = priv->iter; + ct = codetag_next_ct(&priv->iter); + (*pos)++; + if (!ct) + return NULL; + + return priv; +} + +static void allocinfo_stop(struct seq_file *m, void *arg) +{ + codetag_unlock_module_list(alloc_tag_cttype); +} + +static void print_allocinfo_header(struct seq_buf *buf) +{ + /* Output format version, so we can change it. */ + seq_buf_printf(buf, "allocinfo - version: 2.0\n"); + seq_buf_printf(buf, "# \n"); +} + +static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct) +{ + struct alloc_tag *tag = ct_to_alloc_tag(ct); + struct alloc_tag_counters counter = alloc_tag_read(tag); + s64 bytes = counter.bytes; + + seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls); + codetag_to_text(out, ct); + if (unlikely(alloc_tag_is_inaccurate(tag))) + seq_buf_printf(out, " accurate:no"); + seq_buf_putc(out, ' '); + seq_buf_putc(out, '\n'); +} + +static int allocinfo_show(struct seq_file *m, void *arg) +{ + struct allocinfo_private *priv = (struct allocinfo_private *)arg; + char *bufp; + size_t n = seq_get_buf(m, &bufp); + struct seq_buf buf; + + seq_buf_init(&buf, bufp, n); + if (priv->print_header) { + print_allocinfo_header(&buf); + priv->print_header = false; + } + alloc_tag_to_text(&buf, priv->iter.ct); + seq_commit(m, seq_buf_used(&buf)); + return 0; +} + +static const struct seq_operations allocinfo_seq_op = { + .start = allocinfo_start, + .next = allocinfo_next, + .stop = allocinfo_stop, + .show = allocinfo_show, +}; + +size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) +{ + struct codetag_iterator iter; + struct codetag *ct; + struct codetag_bytes n; + unsigned int i, nr = 0; + + if (IS_ERR_OR_NULL(alloc_tag_cttype)) + return 0; + + if (can_sleep) + codetag_lock_module_list(alloc_tag_cttype); + else if (!codetag_trylock_module_list(alloc_tag_cttype)) + return 0; + + iter = codetag_get_ct_iter(alloc_tag_cttype); + while ((ct = codetag_next_ct(&iter))) { + struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct)); + + n.ct = ct; + n.bytes = counter.bytes; + + for (i = 0; i < nr; i++) + if (n.bytes > tags[i].bytes) + break; + + if (i < count) { + nr -= nr == count; + memmove(&tags[i + 1], + &tags[i], + sizeof(tags[0]) * (nr - i)); + nr++; + tags[i] = n; + } + } + + codetag_unlock_module_list(alloc_tag_cttype); + + return nr; +} + +void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) +{ + int i; + struct alloc_tag *tag; + unsigned int nr_pages = 1 << new_order; + + if (!mem_alloc_profiling_enabled()) + return; + + tag = __pgalloc_tag_get(&folio->page); + if (!tag) + return; + + for (i = nr_pages; i < (1 << old_order); i += nr_pages) { + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) { + /* Set new reference to point to the original tag */ + alloc_tag_ref_set(&ref, tag); + update_page_tag_ref(handle, &ref); + put_page_tag_ref(handle); + } + } +} + +void pgalloc_tag_swap(struct folio *new, struct folio *old) +{ + union pgtag_ref_handle handle_old, handle_new; + union codetag_ref ref_old, ref_new; + struct alloc_tag *tag_old, *tag_new; + + if (!mem_alloc_profiling_enabled()) + return; + + tag_old = __pgalloc_tag_get(&old->page); + if (!tag_old) + return; + tag_new = __pgalloc_tag_get(&new->page); + if (!tag_new) + return; + + if (!get_page_tag_ref(&old->page, &ref_old, &handle_old)) + return; + if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) { + put_page_tag_ref(handle_old); + return; + } + + /* + * Clear tag references to avoid debug warning when using + * __alloc_tag_ref_set() with non-empty reference. + */ + set_codetag_empty(&ref_old); + set_codetag_empty(&ref_new); + + /* swap tags */ + __alloc_tag_ref_set(&ref_old, tag_new); + update_page_tag_ref(handle_old, &ref_old); + __alloc_tag_ref_set(&ref_new, tag_old); + update_page_tag_ref(handle_new, &ref_new); + + put_page_tag_ref(handle_old); + put_page_tag_ref(handle_new); +} + +static void shutdown_mem_profiling(bool remove_file) +{ + if (mem_alloc_profiling_enabled()) + static_branch_disable(&mem_alloc_profiling_key); + + if (!mem_profiling_support) + return; + + if (remove_file) + remove_proc_entry(ALLOCINFO_FILE_NAME, NULL); + mem_profiling_support = false; +} + +void __init alloc_tag_sec_init(void) +{ + struct alloc_tag *last_codetag; + + if (!mem_profiling_support) + return; + + if (!static_key_enabled(&mem_profiling_compressed)) + return; + + kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name( + SECTION_START(ALLOC_TAG_SECTION_NAME)); + last_codetag = (struct alloc_tag *)kallsyms_lookup_name( + SECTION_STOP(ALLOC_TAG_SECTION_NAME)); + kernel_tags.count = last_codetag - kernel_tags.first_tag; + + /* Check if kernel tags fit into page flags */ + if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { + shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ + pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", + kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); + return; + } + + alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS); + alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1); + pr_debug("Memory allocation profiling compression is using %d page flag bits!\n", + NR_UNUSED_PAGEFLAG_BITS); +} + +#ifdef CONFIG_MODULES + +static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE); +static struct vm_struct *vm_module_tags; +/* A dummy object used to indicate an unloaded module */ +static struct module unloaded_mod; +/* A dummy object used to indicate a module prepended area */ +static struct module prepend_mod; + +struct alloc_tag_module_section module_tags; + +static inline unsigned long alloc_tag_align(unsigned long val) +{ + if (!static_key_enabled(&mem_profiling_compressed)) { + /* No alignment requirements when we are not indexing the tags */ + return val; + } + + if (val % sizeof(struct alloc_tag) == 0) + return val; + return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag); +} + +static bool ensure_alignment(unsigned long align, unsigned int *prepend) +{ + if (!static_key_enabled(&mem_profiling_compressed)) { + /* No alignment requirements when we are not indexing the tags */ + return true; + } + + /* + * If alloc_tag size is not a multiple of required alignment, tag + * indexing does not work. + */ + if (!IS_ALIGNED(sizeof(struct alloc_tag), align)) + return false; + + /* Ensure prepend consumes multiple of alloc_tag-sized blocks */ + if (*prepend) + *prepend = alloc_tag_align(*prepend); + + return true; +} + +static inline bool tags_addressable(void) +{ + unsigned long tag_idx_count; + + if (!static_key_enabled(&mem_profiling_compressed)) + return true; /* with page_ext tags are always addressable */ + + tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count + + module_tags.size / sizeof(struct alloc_tag); + + return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS); +} + +static bool needs_section_mem(struct module *mod, unsigned long size) +{ + if (!mem_profiling_support) + return false; + + return size >= sizeof(struct alloc_tag); +} + +static bool clean_unused_counters(struct alloc_tag *start_tag, + struct alloc_tag *end_tag) +{ + struct alloc_tag *tag; + bool ret = true; + + for (tag = start_tag; tag <= end_tag; tag++) { + struct alloc_tag_counters counter; + + if (!tag->counters) + continue; + + counter = alloc_tag_read(tag); + if (!counter.bytes) { + free_percpu(tag->counters); + tag->counters = NULL; + } else { + ret = false; + } + } + + return ret; +} + +/* Called with mod_area_mt locked */ +static void clean_unused_module_areas_locked(void) +{ + MA_STATE(mas, &mod_area_mt, 0, module_tags.size); + struct module *val; + + mas_for_each(&mas, val, module_tags.size) { + struct alloc_tag *start_tag; + struct alloc_tag *end_tag; + + if (val != &unloaded_mod) + continue; + + /* Release area if all tags are unused */ + start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); + end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); + if (clean_unused_counters(start_tag, end_tag)) + mas_erase(&mas); + } +} + +/* Called with mod_area_mt locked */ +static bool find_aligned_area(struct ma_state *mas, unsigned long section_size, + unsigned long size, unsigned int prepend, unsigned long align) +{ + bool cleanup_done = false; + +repeat: + /* Try finding exact size and hope the start is aligned */ + if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) { + if (IS_ALIGNED(mas->index + prepend, align)) + return true; + + /* Try finding larger area to align later */ + mas_reset(mas); + if (!mas_empty_area(mas, 0, section_size - 1, + size + prepend + align - 1)) + return true; + } + + /* No free area, try cleanup stale data and repeat the search once */ + if (!cleanup_done) { + clean_unused_module_areas_locked(); + cleanup_done = true; + mas_reset(mas); + goto repeat; + } + + return false; +} + +static int vm_module_tags_populate(void) +{ + unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) + + (vm_module_tags->nr_pages << PAGE_SHIFT); + unsigned long new_end = module_tags.start_addr + module_tags.size; + + if (phys_end < new_end) { + struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages; + unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN); + unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN); + unsigned long more_pages; + unsigned long nr = 0; + + more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT; + while (nr < more_pages) { + unsigned long allocated; + + allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN, + NUMA_NO_NODE, more_pages - nr, next_page + nr); + + if (!allocated) + break; + nr += allocated; + } + + if (nr < more_pages || + vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL, + next_page, PAGE_SHIFT) < 0) { + release_pages_arg arg = { .pages = next_page }; + + /* Clean up and error out */ + release_pages(arg, nr); + return -ENOMEM; + } + + vm_module_tags->nr_pages += nr; + + /* + * Kasan allocates 1 byte of shadow for every 8 bytes of data. + * When kasan_alloc_module_shadow allocates shadow memory, + * its unit of allocation is a page. + * Therefore, here we need to align to MODULE_ALIGN. + */ + if (old_shadow_end < new_shadow_end) + kasan_alloc_module_shadow((void *)old_shadow_end, + new_shadow_end - old_shadow_end, + GFP_KERNEL); + } + + /* + * Mark the pages as accessible, now that they are mapped. + * With hardware tag-based KASAN, marking is skipped for + * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). + */ + kasan_unpoison_vmalloc((void *)module_tags.start_addr, + new_end - module_tags.start_addr, + KASAN_VMALLOC_PROT_NORMAL); + + return 0; +} + +static void *reserve_module_tags(struct module *mod, unsigned long size, + unsigned int prepend, unsigned long align) +{ + unsigned long section_size = module_tags.end_addr - module_tags.start_addr; + MA_STATE(mas, &mod_area_mt, 0, section_size - 1); + unsigned long offset; + void *ret = NULL; + + /* If no tags return error */ + if (size < sizeof(struct alloc_tag)) + return ERR_PTR(-EINVAL); + + /* + * align is always power of 2, so we can use IS_ALIGNED and ALIGN. + * align 0 or 1 means no alignment, to simplify set to 1. + */ + if (!align) + align = 1; + + if (!ensure_alignment(align, &prepend)) { + shutdown_mem_profiling(true); + pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n", + mod->name, align); + return ERR_PTR(-EINVAL); + } + + mas_lock(&mas); + if (!find_aligned_area(&mas, section_size, size, prepend, align)) { + ret = ERR_PTR(-ENOMEM); + goto unlock; + } + + /* Mark found area as reserved */ + offset = mas.index; + offset += prepend; + offset = ALIGN(offset, align); + if (offset != mas.index) { + unsigned long pad_start = mas.index; + + mas.last = offset - 1; + mas_store(&mas, &prepend_mod); + if (mas_is_err(&mas)) { + ret = ERR_PTR(xa_err(mas.node)); + goto unlock; + } + mas.index = offset; + mas.last = offset + size - 1; + mas_store(&mas, mod); + if (mas_is_err(&mas)) { + mas.index = pad_start; + mas_erase(&mas); + ret = ERR_PTR(xa_err(mas.node)); + } + } else { + mas.last = offset + size - 1; + mas_store(&mas, mod); + if (mas_is_err(&mas)) + ret = ERR_PTR(xa_err(mas.node)); + } +unlock: + mas_unlock(&mas); + + if (IS_ERR(ret)) + return ret; + + if (module_tags.size < offset + size) { + int grow_res; + + module_tags.size = offset + size; + if (mem_alloc_profiling_enabled() && !tags_addressable()) { + shutdown_mem_profiling(true); + pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n", + mod->name, NR_UNUSED_PAGEFLAG_BITS); + } + + grow_res = vm_module_tags_populate(); + if (grow_res) { + shutdown_mem_profiling(true); + pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n", + mod->name); + return ERR_PTR(grow_res); + } + } + + return (struct alloc_tag *)(module_tags.start_addr + offset); +} + +static void release_module_tags(struct module *mod, bool used) +{ + MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size); + struct alloc_tag *start_tag; + struct alloc_tag *end_tag; + struct module *val; + + mas_lock(&mas); + mas_for_each_rev(&mas, val, 0) + if (val == mod) + break; + + if (!val) /* module not found */ + goto out; + + if (!used) + goto release_area; + + start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); + end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); + if (!clean_unused_counters(start_tag, end_tag)) { + struct alloc_tag *tag; + + for (tag = start_tag; tag <= end_tag; tag++) { + struct alloc_tag_counters counter; + + if (!tag->counters) + continue; + + counter = alloc_tag_read(tag); + pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n", + tag->ct.filename, tag->ct.lineno, tag->ct.modname, + tag->ct.function, counter.bytes); + } + } else { + used = false; + } +release_area: + mas_store(&mas, used ? &unloaded_mod : NULL); + val = mas_prev_range(&mas, 0); + if (val == &prepend_mod) + mas_store(&mas, NULL); +out: + mas_unlock(&mas); +} + +static int load_module(struct module *mod, struct codetag *start, struct codetag *stop) +{ + /* Allocate module alloc_tag percpu counters */ + struct alloc_tag *start_tag; + struct alloc_tag *stop_tag; + struct alloc_tag *tag; + + /* percpu counters for core allocations are already statically allocated */ + if (!mod) + return 0; + + start_tag = ct_to_alloc_tag(start); + stop_tag = ct_to_alloc_tag(stop); + for (tag = start_tag; tag < stop_tag; tag++) { + WARN_ON(tag->counters); + tag->counters = alloc_percpu(struct alloc_tag_counters); + if (!tag->counters) { + while (--tag >= start_tag) { + free_percpu(tag->counters); + tag->counters = NULL; + } + pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n", + mod->name); + return -ENOMEM; + } + + /* + * Avoid a kmemleak false positive. The pointer to the counters is stored + * in the alloc_tag section of the module and cannot be directly accessed. + */ + kmemleak_ignore_percpu(tag->counters); + } + return 0; +} + +static void replace_module(struct module *mod, struct module *new_mod) +{ + MA_STATE(mas, &mod_area_mt, 0, module_tags.size); + struct module *val; + + mas_lock(&mas); + mas_for_each(&mas, val, module_tags.size) { + if (val != mod) + continue; + + mas_store_gfp(&mas, new_mod, GFP_KERNEL); + break; + } + mas_unlock(&mas); +} + +static int __init alloc_mod_tags_mem(void) +{ + /* Map space to copy allocation tags */ + vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE); + if (!vm_module_tags) { + pr_err("Failed to map %lu bytes for module allocation tags\n", + MODULE_ALLOC_TAG_VMAP_SIZE); + module_tags.start_addr = 0; + return -ENOMEM; + } + + vm_module_tags->pages = kmalloc_objs(struct page *, + get_vm_area_size(vm_module_tags) >> PAGE_SHIFT, + GFP_KERNEL | __GFP_ZERO); + if (!vm_module_tags->pages) { + free_vm_area(vm_module_tags); + return -ENOMEM; + } + + module_tags.start_addr = (unsigned long)vm_module_tags->addr; + module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE; + /* Ensure the base is alloc_tag aligned when required for indexing */ + module_tags.start_addr = alloc_tag_align(module_tags.start_addr); + + return 0; +} + +static void __init free_mod_tags_mem(void) +{ + release_pages_arg arg = { .pages = vm_module_tags->pages }; + + module_tags.start_addr = 0; + release_pages(arg, vm_module_tags->nr_pages); + kfree(vm_module_tags->pages); + free_vm_area(vm_module_tags); +} + +#else /* CONFIG_MODULES */ + +static inline int alloc_mod_tags_mem(void) { return 0; } +static inline void free_mod_tags_mem(void) {} + +#endif /* CONFIG_MODULES */ + +/* See: Documentation/mm/allocation-profiling.rst */ +static int __init setup_early_mem_profiling(char *str) +{ + bool compressed = false; + bool enable; + + if (!str || !str[0]) + return -EINVAL; + + if (!strncmp(str, "never", 5)) { + enable = false; + mem_profiling_support = false; + pr_info("Memory allocation profiling is disabled!\n"); + } else { + char *token = strsep(&str, ","); + + if (kstrtobool(token, &enable)) + return -EINVAL; + + if (str) { + + if (strcmp(str, "compressed")) + return -EINVAL; + + compressed = true; + } + mem_profiling_support = true; + pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n", + compressed ? "with" : "without", str_on_off(enable)); + } + + if (enable != mem_alloc_profiling_enabled()) { + if (enable) + static_branch_enable(&mem_alloc_profiling_key); + else + static_branch_disable(&mem_alloc_profiling_key); + } + if (compressed != static_key_enabled(&mem_profiling_compressed)) { + if (compressed) + static_branch_enable(&mem_profiling_compressed); + else + static_branch_disable(&mem_profiling_compressed); + } + + return 0; +} +early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling); + +static __init bool need_page_alloc_tagging(void) +{ + if (static_key_enabled(&mem_profiling_compressed)) + return false; + + return mem_profiling_support; +} + +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* + * Track page allocations before page_ext is initialized. + * Some pages are allocated before page_ext becomes available, leaving + * their codetag uninitialized. Track these early PFNs so we can clear + * their codetag refs later to avoid warnings when they are freed. + * + * Each page is cast to a pfn_pool: the first few bytes hold metadata + * (next pointer and slot count), the remainder stores PFNs. + */ +struct pfn_pool { + struct pfn_pool *next; + atomic_t count; + unsigned long pfns[]; +}; + +#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ + sizeof(unsigned long)) + +/* + * Skip early PFN recording for a page allocation. Reuses the + * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid + * recursion when allocating pages for the early PFN tracking list + * itself. + * + * Codetags of the pages allocated with __GFP_NO_CODETAG should be + * cleared (via clear_page_tag_ref()) before freeing the pages to prevent + * alloc_tag_sub_check() from triggering a warning. + */ +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT + +static struct pfn_pool *current_pfn_pool __initdata; + +static void __init __alloc_tag_add_early_pfn(unsigned long pfn) +{ + struct pfn_pool *pool; + int idx; + + do { + pool = READ_ONCE(current_pfn_pool); + if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { + struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG); + struct pfn_pool *new; + + if (!new_page) { + pr_warn_once("early PFN tracking page allocation failed\n"); + return; + } + new = page_address(new_page); + new->next = pool; + atomic_set(&new->count, 0); + if (cmpxchg(¤t_pfn_pool, pool, new) != pool) { + clear_page_tag_ref(new_page); + __free_page(new_page); + continue; + } + pool = new; + } + idx = atomic_read(&pool->count); + if (idx >= PFN_POOL_SIZE) + continue; + if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx) + break; + } while (1); + + pool->pfns[idx] = pfn; +} + +typedef void alloc_tag_add_func(unsigned long pfn); +static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = + RCU_INITIALIZER(__alloc_tag_add_early_pfn); + +void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) +{ + alloc_tag_add_func *alloc_tag_add; + + if (static_key_enabled(&mem_profiling_compressed)) + return; + + /* Skip allocations for the tracking list itself to avoid recursion. */ + if (gfp_flags & __GFP_NO_CODETAG) + return; + + rcu_read_lock(); + alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr); + if (alloc_tag_add) + alloc_tag_add(pfn); + rcu_read_unlock(); +} + +static void __init clear_early_alloc_pfn_tag_refs(void) +{ + struct pfn_pool *pool, *next; + struct page *page; + int i; + + if (static_key_enabled(&mem_profiling_compressed)) + return; + + rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL); + /* Make sure we are not racing with __alloc_tag_add_early_pfn() */ + synchronize_rcu(); + + for (pool = current_pfn_pool; pool; pool = next) { + int nr_pfns = atomic_read(&pool->count); + + for (i = 0; i < nr_pfns; i++) { + unsigned long pfn = pool->pfns[i]; + + if (pfn_valid(pfn)) { + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) { + /* + * An early-allocated page could be freed and reallocated + * after its page_ext is initialized but before we clear it. + * In that case, it already has a valid tag set. + * We should not overwrite that valid tag + * with CODETAG_EMPTY. + * + * Note: there is still a small race window between checking + * ref.ct and calling set_codetag_empty(). We accept this + * race as it's unlikely and the extra complexity of atomic + * cmpxchg is not worth it for this debug-only code path. + */ + if (ref.ct) { + put_page_tag_ref(handle); + continue; + } + + set_codetag_empty(&ref); + update_page_tag_ref(handle, &ref); + put_page_tag_ref(handle); + } + } + } + + next = pool->next; + page = virt_to_page(pool); + clear_page_tag_ref(page); + __free_page(page); + } +} +#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ +static inline void __init clear_early_alloc_pfn_tag_refs(void) {} +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + +static __init void init_page_alloc_tagging(void) +{ + clear_early_alloc_pfn_tag_refs(); +} + +struct page_ext_operations page_alloc_tagging_ops = { + .size = sizeof(union codetag_ref), + .need = need_page_alloc_tagging, + .init = init_page_alloc_tagging, +}; +EXPORT_SYMBOL(page_alloc_tagging_ops); + +#ifdef CONFIG_SYSCTL +/* + * Not using proc_do_static_key() directly to prevent enabling profiling + * after it was shut down. + */ +static int proc_mem_profiling_handler(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + if (write) { + /* + * Call from do_sysctl_args() which is a no-op since the same + * value was already set by setup_early_mem_profiling. + * Return success to avoid warnings from do_sysctl_args(). + */ + if (!current->mm) + return 0; + +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG + /* User can't toggle profiling while debugging */ + return -EACCES; +#endif + if (!mem_profiling_support) + return -EINVAL; + } + + return proc_do_static_key(table, write, buffer, lenp, ppos); +} + + +static const struct ctl_table memory_allocation_profiling_sysctls[] = { + { + .procname = "mem_profiling", + .data = &mem_alloc_profiling_key, + .mode = 0644, + .proc_handler = proc_mem_profiling_handler, + }, +}; + +static void __init sysctl_init(void) +{ + register_sysctl_init("vm", memory_allocation_profiling_sysctls); +} +#else /* CONFIG_SYSCTL */ +static inline void sysctl_init(void) {} +#endif /* CONFIG_SYSCTL */ + +static int __init alloc_tag_init(void) +{ + const struct codetag_type_desc desc = { + .section = ALLOC_TAG_SECTION_NAME, + .tag_size = sizeof(struct alloc_tag), +#ifdef CONFIG_MODULES + .needs_section_mem = needs_section_mem, + .alloc_section_mem = reserve_module_tags, + .free_section_mem = release_module_tags, + .module_load = load_module, + .module_replaced = replace_module, +#endif + }; + int res; + + sysctl_init(); + + if (!mem_profiling_support) { + pr_info("Memory allocation profiling is not supported!\n"); + return 0; + } + + if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op, + sizeof(struct allocinfo_private), NULL)) { + pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); + shutdown_mem_profiling(false); + return -ENOMEM; + } + + res = alloc_mod_tags_mem(); + if (res) { + pr_err("Failed to reserve address space for module tags, errno = %d\n", res); + shutdown_mem_profiling(true); + return res; + } + + alloc_tag_cttype = codetag_register_type(&desc); + if (IS_ERR(alloc_tag_cttype)) { + pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype); + free_mod_tags_mem(); + shutdown_mem_profiling(true); + return PTR_ERR(alloc_tag_cttype); + } + + return 0; +} +module_init(alloc_tag_init);