From d52341da4db0cd993d3549aa20cbdf063b412c3b Mon Sep 17 00:00:00 2001 From: Pierre-Henry Moussay Date: Mon, 17 Nov 2025 14:24:37 +0000 Subject: [PATCH 1/9] dt-bindings: cache: sifive,ccache0: add a pic64gx compatible The pic64gx use the same IP than mpfs, therefore add compatibility with mpfs as fallback. Signed-off-by: Pierre-Henry Moussay Acked-by: Rob Herring (Arm) Signed-off-by: Conor Dooley --- Documentation/devicetree/bindings/cache/sifive,ccache0.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/devicetree/bindings/cache/sifive,ccache0.yaml b/Documentation/devicetree/bindings/cache/sifive,ccache0.yaml index 579bacb66f34..c0e5ebb1fa4c 100644 --- a/Documentation/devicetree/bindings/cache/sifive,ccache0.yaml +++ b/Documentation/devicetree/bindings/cache/sifive,ccache0.yaml @@ -48,6 +48,11 @@ properties: - const: microchip,mpfs-ccache - const: sifive,fu540-c000-ccache - const: cache + - items: + - const: microchip,pic64gx-ccache + - const: microchip,mpfs-ccache + - const: sifive,fu540-c000-ccache + - const: cache cache-block-size: const: 64 From f49ae86483c494ddc793d889f6df5ea68d138569 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 17 Nov 2025 10:47:54 +0000 Subject: [PATCH 2/9] memregion: Drop unused IORES_DESC_* parameter from cpu_cache_invalidate_memregion() The res_desc parameter was originally introduced for documentation purposes and with the idea that with HDM-DB CXL invalidation could be triggered from the device. That has not come to pass and the continued existence of the option is confusing when we add a range in the following patch which might not be a strict subset of the res_desc. So avoid that confusion by dropping the parameter. Link: https://lore.kernel.org/linux-mm/686eedb25ed02_24471002e@dwillia2-xfh.jf.intel.com.notmuch/ Reviewed-by: Dan Williams Suggested-by: Dan Williams Signed-off-by: Jonathan Cameron Signed-off-by: Conor Dooley --- arch/x86/mm/pat/set_memory.c | 2 +- drivers/cxl/core/region.c | 2 +- drivers/nvdimm/region.c | 2 +- drivers/nvdimm/region_devs.c | 2 +- include/linux/memregion.h | 7 +++---- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 8834c76f91c9..4019b17fb65e 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -368,7 +368,7 @@ bool cpu_cache_has_invalidate_memregion(void) } EXPORT_SYMBOL_NS_GPL(cpu_cache_has_invalidate_memregion, "DEVMEM"); -int cpu_cache_invalidate_memregion(int res_desc) +int cpu_cache_invalidate_memregion(void) { if (WARN_ON_ONCE(!cpu_cache_has_invalidate_memregion())) return -ENXIO; diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 71cc42d05248..d7fa76810f82 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -228,7 +228,7 @@ static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) return -ENXIO; } - cpu_cache_invalidate_memregion(IORES_DESC_CXL); + cpu_cache_invalidate_memregion(); return 0; } diff --git a/drivers/nvdimm/region.c b/drivers/nvdimm/region.c index 88dc062af5f8..c43506448edf 100644 --- a/drivers/nvdimm/region.c +++ b/drivers/nvdimm/region.c @@ -110,7 +110,7 @@ static void nd_region_remove(struct device *dev) * here is ok. */ if (cpu_cache_has_invalidate_memregion()) - cpu_cache_invalidate_memregion(IORES_DESC_PERSISTENT_MEMORY); + cpu_cache_invalidate_memregion(); } static int child_notify(struct device *dev, void *data) diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index de1ee5ebc851..3cdd93d40997 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -90,7 +90,7 @@ static int nd_region_invalidate_memregion(struct nd_region *nd_region) } } - cpu_cache_invalidate_memregion(IORES_DESC_PERSISTENT_MEMORY); + cpu_cache_invalidate_memregion(); out: for (i = 0; i < nd_region->ndr_mappings; i++) { struct nd_mapping *nd_mapping = &nd_region->mapping[i]; diff --git a/include/linux/memregion.h b/include/linux/memregion.h index c01321467789..945646bde825 100644 --- a/include/linux/memregion.h +++ b/include/linux/memregion.h @@ -26,8 +26,7 @@ static inline void memregion_free(int id) /** * cpu_cache_invalidate_memregion - drop any CPU cached data for - * memregions described by @res_desc - * @res_desc: one of the IORES_DESC_* types + * memregion * * Perform cache maintenance after a memory event / operation that * changes the contents of physical memory in a cache-incoherent manner. @@ -46,7 +45,7 @@ static inline void memregion_free(int id) * the cache maintenance. */ #ifdef CONFIG_ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION -int cpu_cache_invalidate_memregion(int res_desc); +int cpu_cache_invalidate_memregion(void); bool cpu_cache_has_invalidate_memregion(void); #else static inline bool cpu_cache_has_invalidate_memregion(void) @@ -54,7 +53,7 @@ static inline bool cpu_cache_has_invalidate_memregion(void) return false; } -static inline int cpu_cache_invalidate_memregion(int res_desc) +static inline int cpu_cache_invalidate_memregion(void) { WARN_ON_ONCE("CPU cache invalidation required"); return -ENXIO; From b43652d867cf2a5f31b14e3d9a320ad01fca0992 Mon Sep 17 00:00:00 2001 From: Yicong Yang Date: Mon, 17 Nov 2025 10:47:55 +0000 Subject: [PATCH 3/9] memregion: Support fine grained invalidate by cpu_cache_invalidate_memregion() Extend cpu_cache_invalidate_memregion() to support invalidating a particular range of memory by introducing start and length parameters. Control of types of invalidation is left for when use cases turn up. For now everything is Clean and Invalidate. Where the range is unknown, use the provided cpu_cache_invalidate_all() helper to act as documentation of intent in a fashion that is clearer than passing (0, -1) to cpu_cache_invalidate_memregion(). Signed-off-by: Yicong Yang Reviewed-by: Dan Williams Acked-by: Davidlohr Bueso Signed-off-by: Jonathan Cameron Signed-off-by: Conor Dooley --- arch/x86/mm/pat/set_memory.c | 2 +- drivers/cxl/core/region.c | 5 ++++- drivers/nvdimm/region.c | 2 +- drivers/nvdimm/region_devs.c | 2 +- include/linux/memregion.h | 13 +++++++++++-- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 4019b17fb65e..292c7202faed 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -368,7 +368,7 @@ bool cpu_cache_has_invalidate_memregion(void) } EXPORT_SYMBOL_NS_GPL(cpu_cache_has_invalidate_memregion, "DEVMEM"); -int cpu_cache_invalidate_memregion(void) +int cpu_cache_invalidate_memregion(phys_addr_t start, size_t len) { if (WARN_ON_ONCE(!cpu_cache_has_invalidate_memregion())) return -ENXIO; diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index d7fa76810f82..410e41cef5d3 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -228,7 +228,10 @@ static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) return -ENXIO; } - cpu_cache_invalidate_memregion(); + if (!cxlr->params.res) + return -ENXIO; + cpu_cache_invalidate_memregion(cxlr->params.res->start, + resource_size(cxlr->params.res)); return 0; } diff --git a/drivers/nvdimm/region.c b/drivers/nvdimm/region.c index c43506448edf..42e982db5b04 100644 --- a/drivers/nvdimm/region.c +++ b/drivers/nvdimm/region.c @@ -110,7 +110,7 @@ static void nd_region_remove(struct device *dev) * here is ok. */ if (cpu_cache_has_invalidate_memregion()) - cpu_cache_invalidate_memregion(); + cpu_cache_invalidate_all(); } static int child_notify(struct device *dev, void *data) diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 3cdd93d40997..e27fc380f6c0 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -90,7 +90,7 @@ static int nd_region_invalidate_memregion(struct nd_region *nd_region) } } - cpu_cache_invalidate_memregion(); + cpu_cache_invalidate_all(); out: for (i = 0; i < nd_region->ndr_mappings; i++) { struct nd_mapping *nd_mapping = &nd_region->mapping[i]; diff --git a/include/linux/memregion.h b/include/linux/memregion.h index 945646bde825..a55f62cc5266 100644 --- a/include/linux/memregion.h +++ b/include/linux/memregion.h @@ -27,6 +27,9 @@ static inline void memregion_free(int id) /** * cpu_cache_invalidate_memregion - drop any CPU cached data for * memregion + * @start: start physical address of the target memory region. + * @len: length of the target memory region. -1 for all the regions of + * the target type. * * Perform cache maintenance after a memory event / operation that * changes the contents of physical memory in a cache-incoherent manner. @@ -45,7 +48,7 @@ static inline void memregion_free(int id) * the cache maintenance. */ #ifdef CONFIG_ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION -int cpu_cache_invalidate_memregion(void); +int cpu_cache_invalidate_memregion(phys_addr_t start, size_t len); bool cpu_cache_has_invalidate_memregion(void); #else static inline bool cpu_cache_has_invalidate_memregion(void) @@ -53,10 +56,16 @@ static inline bool cpu_cache_has_invalidate_memregion(void) return false; } -static inline int cpu_cache_invalidate_memregion(void) +static inline int cpu_cache_invalidate_memregion(phys_addr_t start, size_t len) { WARN_ON_ONCE("CPU cache invalidation required"); return -ENXIO; } #endif + +static inline int cpu_cache_invalidate_all(void) +{ + return cpu_cache_invalidate_memregion(0, -1); +} + #endif /* _MEMREGION_H_ */ From c460697d3472d4252917fba9bbc1d1a23eafc124 Mon Sep 17 00:00:00 2001 From: Yicong Yang Date: Mon, 17 Nov 2025 10:47:56 +0000 Subject: [PATCH 4/9] lib: Support ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION provides the mechanism for invalidating certain memory regions in a cache-incoherent manner. Currently this is used by NVDIMM and CXL memory drivers in cases where it is necessary to flush all data from caches by physical address range. The operations in question are effectively memory hotplug, where stale data might otherwise remain in the caches. This is separate from the invalidates done to enable use of non-coherent DMA masters, primarily in terms of when it is needed (not related to DMA mappings) and how deep the flush must push data. The flushes done for non-coherent DMA only need to reach the Point of Coherence of a single host (which is often nearer CPUs and DMA masters than the physical storage). This operation must push the data out of non architectural caches (memory-side caches, write buffers etc) and typically all the way to the memory device. In some architectures these operations are supported by system components that may become available only later in boot as they are either present on a discoverable bus, or via a firmware description of an MMIO interface (e.g. ACPI DSDT). Provide a framework to handle this case. Architectures can opt in for this support via CONFIG_GENERIC_CPU_CACHE_MAINTENANCE Add a registration framework. Each driver provides an ops structure and the first op is Write Back and Invalidate by PA Range. The driver may over invalidate. For systems that can perform this operation asynchronously an optional completion check operation is also provided. If present that must be called to ensure that the action has finished. This provides a considerable performance advantage if multiple agents are involved in the maintenance operation. When multiple agents are present in the system each should register with this framework and the core code will issue the invalidate to all of them before checking for completion on each. This is done to avoid need for filtering in the core code which can become complex when interleave, potentially across different cache coherency hardware is going on, so it is easier to tell everyone and let those who don't care do nothing. Signed-off-by: Yicong Yang Co-developed-by: Jonathan Cameron Signed-off-by: Jonathan Cameron Acked-by: Conor Dooley Signed-off-by: Conor Dooley --- include/linux/cache_coherency.h | 61 ++++++++++++++ lib/Kconfig | 3 + lib/Makefile | 2 + lib/cache_maint.c | 138 ++++++++++++++++++++++++++++++++ 4 files changed, 204 insertions(+) create mode 100644 include/linux/cache_coherency.h create mode 100644 lib/cache_maint.c diff --git a/include/linux/cache_coherency.h b/include/linux/cache_coherency.h new file mode 100644 index 000000000000..cc81c5733e31 --- /dev/null +++ b/include/linux/cache_coherency.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Cache coherency maintenance operation device drivers + * + * Copyright Huawei 2025 + */ +#ifndef _LINUX_CACHE_COHERENCY_H_ +#define _LINUX_CACHE_COHERENCY_H_ + +#include +#include +#include + +struct cc_inval_params { + phys_addr_t addr; + size_t size; +}; + +struct cache_coherency_ops_inst; + +struct cache_coherency_ops { + int (*wbinv)(struct cache_coherency_ops_inst *cci, + struct cc_inval_params *invp); + int (*done)(struct cache_coherency_ops_inst *cci); +}; + +struct cache_coherency_ops_inst { + struct kref kref; + struct list_head node; + const struct cache_coherency_ops *ops; +}; + +int cache_coherency_ops_instance_register(struct cache_coherency_ops_inst *cci); +void cache_coherency_ops_instance_unregister(struct cache_coherency_ops_inst *cci); + +struct cache_coherency_ops_inst * +_cache_coherency_ops_instance_alloc(const struct cache_coherency_ops *ops, + size_t size); +/** + * cache_coherency_ops_instance_alloc - Allocate cache coherency ops instance + * @ops: Cache maintenance operations + * @drv_struct: structure that contains the struct cache_coherency_ops_inst + * @member: Name of the struct cache_coherency_ops_inst member in @drv_struct. + * + * This allocates a driver specific structure and initializes the + * cache_coherency_ops_inst embedded in the drv_struct. Upon success the + * pointer must be freed via cache_coherency_ops_instance_put(). + * + * Returns a &drv_struct * on success, %NULL on error. + */ +#define cache_coherency_ops_instance_alloc(ops, drv_struct, member) \ + ({ \ + static_assert(__same_type(struct cache_coherency_ops_inst, \ + ((drv_struct *)NULL)->member)); \ + static_assert(offsetof(drv_struct, member) == 0); \ + (drv_struct *)_cache_coherency_ops_instance_alloc(ops, \ + sizeof(drv_struct)); \ + }) +void cache_coherency_ops_instance_put(struct cache_coherency_ops_inst *cci); + +#endif diff --git a/lib/Kconfig b/lib/Kconfig index c483951b624f..cd8e5844f9bb 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -543,6 +543,9 @@ config MEMREGION config ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION bool +config GENERIC_CPU_CACHE_MAINTENANCE + bool + config ARCH_HAS_MEMREMAP_COMPAT_ALIGN bool diff --git a/lib/Makefile b/lib/Makefile index 392ff808c9b9..eed20c50f358 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -130,6 +130,8 @@ obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o +obj-$(CONFIG_GENERIC_CPU_CACHE_MAINTENANCE) += cache_maint.o + lib-y += logic_pio.o lib-$(CONFIG_INDIRECT_IOMEM) += logic_iomem.o diff --git a/lib/cache_maint.c b/lib/cache_maint.c new file mode 100644 index 000000000000..9256a9ffc34c --- /dev/null +++ b/lib/cache_maint.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Generic support for Memory System Cache Maintenance operations. + * + * Coherency maintenance drivers register with this simple framework that will + * iterate over each registered instance to first kick off invalidation and + * then to wait until it is complete. + * + * If no implementations are registered yet cpu_cache_has_invalidate_memregion() + * will return false. If this runs concurrently with unregistration then a + * race exists but this is no worse than the case where the operations instance + * responsible for a given memory region has not yet registered. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static LIST_HEAD(cache_ops_instance_list); +static DECLARE_RWSEM(cache_ops_instance_list_lock); + +static void __cache_coherency_ops_instance_free(struct kref *kref) +{ + struct cache_coherency_ops_inst *cci = + container_of(kref, struct cache_coherency_ops_inst, kref); + kfree(cci); +} + +void cache_coherency_ops_instance_put(struct cache_coherency_ops_inst *cci) +{ + kref_put(&cci->kref, __cache_coherency_ops_instance_free); +} +EXPORT_SYMBOL_GPL(cache_coherency_ops_instance_put); + +static int cache_inval_one(struct cache_coherency_ops_inst *cci, void *data) +{ + if (!cci->ops) + return -EINVAL; + + return cci->ops->wbinv(cci, data); +} + +static int cache_inval_done_one(struct cache_coherency_ops_inst *cci) +{ + if (!cci->ops) + return -EINVAL; + + if (!cci->ops->done) + return 0; + + return cci->ops->done(cci); +} + +static int cache_invalidate_memregion(phys_addr_t addr, size_t size) +{ + int ret; + struct cache_coherency_ops_inst *cci; + struct cc_inval_params params = { + .addr = addr, + .size = size, + }; + + guard(rwsem_read)(&cache_ops_instance_list_lock); + list_for_each_entry(cci, &cache_ops_instance_list, node) { + ret = cache_inval_one(cci, ¶ms); + if (ret) + return ret; + } + list_for_each_entry(cci, &cache_ops_instance_list, node) { + ret = cache_inval_done_one(cci); + if (ret) + return ret; + } + + return 0; +} + +struct cache_coherency_ops_inst * +_cache_coherency_ops_instance_alloc(const struct cache_coherency_ops *ops, + size_t size) +{ + struct cache_coherency_ops_inst *cci; + + if (!ops || !ops->wbinv) + return NULL; + + cci = kzalloc(size, GFP_KERNEL); + if (!cci) + return NULL; + + cci->ops = ops; + INIT_LIST_HEAD(&cci->node); + kref_init(&cci->kref); + + return cci; +} +EXPORT_SYMBOL_NS_GPL(_cache_coherency_ops_instance_alloc, "CACHE_COHERENCY"); + +int cache_coherency_ops_instance_register(struct cache_coherency_ops_inst *cci) +{ + guard(rwsem_write)(&cache_ops_instance_list_lock); + list_add(&cci->node, &cache_ops_instance_list); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_register, "CACHE_COHERENCY"); + +void cache_coherency_ops_instance_unregister(struct cache_coherency_ops_inst *cci) +{ + guard(rwsem_write)(&cache_ops_instance_list_lock); + list_del(&cci->node); +} +EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_unregister, "CACHE_COHERENCY"); + +int cpu_cache_invalidate_memregion(phys_addr_t start, size_t len) +{ + return cache_invalidate_memregion(start, len); +} +EXPORT_SYMBOL_NS_GPL(cpu_cache_invalidate_memregion, "DEVMEM"); + +/* + * Used for optimization / debug purposes only as removal can race + * + * Machines that do not support invalidation, e.g. VMs, will not have any + * operations instance to register and so this will always return false. + */ +bool cpu_cache_has_invalidate_memregion(void) +{ + guard(rwsem_read)(&cache_ops_instance_list_lock); + return !list_empty(&cache_ops_instance_list); +} +EXPORT_SYMBOL_NS_GPL(cpu_cache_has_invalidate_memregion, "DEVMEM"); From 4d873c5dc3ed5a189a39fcbddad8bcd2bd2a1785 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 17 Nov 2025 10:47:57 +0000 Subject: [PATCH 5/9] arm64: Select GENERIC_CPU_CACHE_MAINTENANCE The generic CPU cache maintenance framework provides a way to register drivers for devices implementing the underlying support for cpu_cache_has_invalidate_memregion(). Enable it for arm64 by selecting GENERIC_CPU_CACHE_MAINTENANCE which provides the implementation for, and in turn selects, ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION. Signed-off-by: Jonathan Cameron Acked-by: Catalin Marinas Signed-off-by: Conor Dooley --- arch/arm64/Kconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index e9bbfacc35a6..15bf429b3f59 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -21,6 +21,7 @@ config ARM64 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_CACHE_LINE_SIZE select ARCH_HAS_CC_PLATFORM + select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEBUG_VM_PGTABLE @@ -146,6 +147,7 @@ config ARM64 select GENERIC_ARCH_TOPOLOGY select GENERIC_CLOCKEVENTS_BROADCAST select GENERIC_CPU_AUTOPROBE + select GENERIC_CPU_CACHE_MAINTENANCE select GENERIC_CPU_DEVICES select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP From 9b9de5a56a62c86472848ec7d48ca939411511e6 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 17 Nov 2025 10:47:58 +0000 Subject: [PATCH 6/9] MAINTAINERS: Add Jonathan Cameron to drivers/cache and add lib/cache_maint.c + header Seems unfair to inflict the cache-coherency drivers on Conor with out also stepping up as a second maintainer for drivers/cache. Include the library support for cache-coherency maintenance drivers to the existing entry. Signed-off-by: Jonathan Cameron Acked-by: Conor Dooley Signed-off-by: Conor Dooley --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index fe168477caa4..039d10ded9e9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23967,10 +23967,13 @@ F: drivers/staging/ STANDALONE CACHE CONTROLLER DRIVERS M: Conor Dooley +M: Jonathan Cameron S: Maintained T: git https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git/ F: Documentation/devicetree/bindings/cache/ F: drivers/cache +F: include/cache_coherency.h +F: lib/cache_maint.c STARFIRE/DURALAN NETWORK DRIVER M: Ion Badulescu From 4d1608d0ab3365d1ef9447bdbc0cb4c0962f1774 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 17 Nov 2025 10:47:59 +0000 Subject: [PATCH 7/9] cache: Make top level Kconfig menu a boolean dependent on RISCV The next patch will add a new type of cache maintenance driver responsible for flushing deeper than is necessary for non coherent DMA (current use case of drivers/cache drivers), as needed when performing operations such as memory hotplug and security unlocking of persistent memory. The two types of operation are similar enough to share a drivers/cache directory and MAINTAINERS but are otherwise currently unrelated. To avoid confusion have two separate menus. Each has dependencies that are implemented by making them boolean symbols, here CACHEMAINT_FOR_DMA which is dependent on RISCV as all driver are currently for platforms of that architecture. Set new symbol default to y to avoid breaking existing configs. This has no affect on actual code built, just visibility of the menu. Suggested-by: Arnd Bergmann Signed-off-by: Jonathan Cameron Signed-off-by: Conor Dooley --- drivers/cache/Kconfig | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/cache/Kconfig b/drivers/cache/Kconfig index db51386c663a..59a79df4c0ce 100644 --- a/drivers/cache/Kconfig +++ b/drivers/cache/Kconfig @@ -1,9 +1,17 @@ # SPDX-License-Identifier: GPL-2.0 -menu "Cache Drivers" + +menuconfig CACHEMAINT_FOR_DMA + bool "Cache management for noncoherent DMA" + depends on RISCV + default y + help + These drivers implement support for noncoherent DMA master devices + on platforms that lack the standard CPU interfaces for this. + +if CACHEMAINT_FOR_DMA config AX45MP_L2_CACHE bool "Andes Technology AX45MP L2 Cache controller" - depends on RISCV select RISCV_NONSTANDARD_CACHE_OPS help Support for the L2 cache controller on Andes Technology AX45MP platforms. @@ -16,7 +24,6 @@ config SIFIVE_CCACHE config STARFIVE_STARLINK_CACHE bool "StarFive StarLink Cache controller" - depends on RISCV depends on ARCH_STARFIVE depends on 64BIT select RISCV_DMA_NONCOHERENT @@ -24,4 +31,4 @@ config STARFIVE_STARLINK_CACHE help Support for the StarLink cache controller IP from StarFive. -endmenu +endif #CACHEMAINT_FOR_DMA From 2ec3b54a6ff04046c07b7050d02321e406c4dcd1 Mon Sep 17 00:00:00 2001 From: Yushan Wang Date: Mon, 17 Nov 2025 10:48:00 +0000 Subject: [PATCH 8/9] cache: Support cache maintenance for HiSilicon SoC Hydra Home Agent Hydra Home Agent is a device used to maintain cache coherency. Add support for explicit cache maintenance operations using it. A system has multiple of these agents. Whilst only one agent is responsible for a given cache line, interleave means that for a range operation, responsibility for the cache lines making up the range will typically be spread across multiple instances. Put this driver on a new Kconfig menu under drivers/cache. The short description as memory hotplug like operations is intended to cover the somewhat complex set of cases where this unit applies and differentiate it clearly from typical non coherent DMA flows. Co-developed-by: Yicong Yang Signed-off-by: Yicong Yang Signed-off-by: Yushan Wang Signed-off-by: Jonathan Cameron Signed-off-by: Conor Dooley --- drivers/cache/Kconfig | 22 ++++ drivers/cache/Makefile | 2 + drivers/cache/hisi_soc_hha.c | 194 +++++++++++++++++++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 drivers/cache/hisi_soc_hha.c diff --git a/drivers/cache/Kconfig b/drivers/cache/Kconfig index 59a79df4c0ce..1518449d47b5 100644 --- a/drivers/cache/Kconfig +++ b/drivers/cache/Kconfig @@ -32,3 +32,25 @@ config STARFIVE_STARLINK_CACHE Support for the StarLink cache controller IP from StarFive. endif #CACHEMAINT_FOR_DMA + +menuconfig CACHEMAINT_FOR_HOTPLUG + bool "Cache management for memory hot plug like operations" + depends on GENERIC_CPU_CACHE_MAINTENANCE + help + These drivers implement cache management for flows where it is necessary + to flush data from all host caches. + +if CACHEMAINT_FOR_HOTPLUG + +config HISI_SOC_HHA + tristate "HiSilicon Hydra Home Agent (HHA) device driver" + depends on (ARM64 && ACPI) || COMPILE_TEST + help + The Hydra Home Agent (HHA) is responsible for cache coherency + on the SoC. This drivers enables the cache maintenance functions of + the HHA. + + This driver can be built as a module. If so, the module will be + called hisi_soc_hha. + +endif #CACHEMAINT_FOR_HOTPLUG diff --git a/drivers/cache/Makefile b/drivers/cache/Makefile index 55c5e851034d..b3362b15d6c1 100644 --- a/drivers/cache/Makefile +++ b/drivers/cache/Makefile @@ -3,3 +3,5 @@ obj-$(CONFIG_AX45MP_L2_CACHE) += ax45mp_cache.o obj-$(CONFIG_SIFIVE_CCACHE) += sifive_ccache.o obj-$(CONFIG_STARFIVE_STARLINK_CACHE) += starfive_starlink_cache.o + +obj-$(CONFIG_HISI_SOC_HHA) += hisi_soc_hha.o diff --git a/drivers/cache/hisi_soc_hha.c b/drivers/cache/hisi_soc_hha.c new file mode 100644 index 000000000000..25ff0f5ae79b --- /dev/null +++ b/drivers/cache/hisi_soc_hha.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for HiSilicon Hydra Home Agent (HHA). + * + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. + * Author: Yicong Yang + * Yushan Wang + * + * A system typically contains multiple HHAs. Each is responsible for a subset + * of the physical addresses in the system, but interleave can make the mapping + * from a particular cache line to a responsible HHA complex. As such no + * filtering is done in the driver, with the hardware being responsible for + * responding with success for even if it was not responsible for any addresses + * in the range on which the operation was requested. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_HHA_CTRL 0x5004 +#define HISI_HHA_CTRL_EN BIT(0) +#define HISI_HHA_CTRL_RANGE BIT(1) +#define HISI_HHA_CTRL_TYPE GENMASK(3, 2) +#define HISI_HHA_START_L 0x5008 +#define HISI_HHA_START_H 0x500c +#define HISI_HHA_LEN_L 0x5010 +#define HISI_HHA_LEN_H 0x5014 + +/* The maintain operation performs in a 128 Byte granularity */ +#define HISI_HHA_MAINT_ALIGN 128 + +#define HISI_HHA_POLL_GAP_US 10 +#define HISI_HHA_POLL_TIMEOUT_US 50000 + +struct hisi_soc_hha { + /* Must be first element */ + struct cache_coherency_ops_inst cci; + /* Locks HHA instance to forbid overlapping access. */ + struct mutex lock; + void __iomem *base; +}; + +static bool hisi_hha_cache_maintain_wait_finished(struct hisi_soc_hha *soc_hha) +{ + u32 val; + + return !readl_poll_timeout_atomic(soc_hha->base + HISI_HHA_CTRL, val, + !(val & HISI_HHA_CTRL_EN), + HISI_HHA_POLL_GAP_US, + HISI_HHA_POLL_TIMEOUT_US); +} + +static int hisi_soc_hha_wbinv(struct cache_coherency_ops_inst *cci, + struct cc_inval_params *invp) +{ + struct hisi_soc_hha *soc_hha = + container_of(cci, struct hisi_soc_hha, cci); + phys_addr_t top, addr = invp->addr; + size_t size = invp->size; + u32 reg; + + if (!size) + return -EINVAL; + + addr = ALIGN_DOWN(addr, HISI_HHA_MAINT_ALIGN); + top = ALIGN(addr + size, HISI_HHA_MAINT_ALIGN); + size = top - addr; + + guard(mutex)(&soc_hha->lock); + + if (!hisi_hha_cache_maintain_wait_finished(soc_hha)) + return -EBUSY; + + /* + * Hardware will search for addresses ranging [addr, addr + size - 1], + * last byte included, and perform maintenance in 128 byte granules + * on those cachelines which contain the addresses. If a given instance + * is either not responsible for a cacheline or that cacheline is not + * currently present then the search will fail, no operation will be + * necessary and the device will report success. + */ + size -= 1; + + writel(lower_32_bits(addr), soc_hha->base + HISI_HHA_START_L); + writel(upper_32_bits(addr), soc_hha->base + HISI_HHA_START_H); + writel(lower_32_bits(size), soc_hha->base + HISI_HHA_LEN_L); + writel(upper_32_bits(size), soc_hha->base + HISI_HHA_LEN_H); + + reg = FIELD_PREP(HISI_HHA_CTRL_TYPE, 1); /* Clean Invalid */ + reg |= HISI_HHA_CTRL_RANGE | HISI_HHA_CTRL_EN; + writel(reg, soc_hha->base + HISI_HHA_CTRL); + + return 0; +} + +static int hisi_soc_hha_done(struct cache_coherency_ops_inst *cci) +{ + struct hisi_soc_hha *soc_hha = + container_of(cci, struct hisi_soc_hha, cci); + + guard(mutex)(&soc_hha->lock); + if (!hisi_hha_cache_maintain_wait_finished(soc_hha)) + return -ETIMEDOUT; + + return 0; +} + +static const struct cache_coherency_ops hha_ops = { + .wbinv = hisi_soc_hha_wbinv, + .done = hisi_soc_hha_done, +}; + +static int hisi_soc_hha_probe(struct platform_device *pdev) +{ + struct hisi_soc_hha *soc_hha; + struct resource *mem; + int ret; + + soc_hha = cache_coherency_ops_instance_alloc(&hha_ops, + struct hisi_soc_hha, cci); + if (!soc_hha) + return -ENOMEM; + + platform_set_drvdata(pdev, soc_hha); + + mutex_init(&soc_hha->lock); + + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!mem) { + ret = -ENOMEM; + goto err_free_cci; + } + + soc_hha->base = ioremap(mem->start, resource_size(mem)); + if (!soc_hha->base) { + ret = dev_err_probe(&pdev->dev, -ENOMEM, + "failed to remap io memory"); + goto err_free_cci; + } + + ret = cache_coherency_ops_instance_register(&soc_hha->cci); + if (ret) + goto err_iounmap; + + return 0; + +err_iounmap: + iounmap(soc_hha->base); +err_free_cci: + cache_coherency_ops_instance_put(&soc_hha->cci); + return ret; +} + +static void hisi_soc_hha_remove(struct platform_device *pdev) +{ + struct hisi_soc_hha *soc_hha = platform_get_drvdata(pdev); + + cache_coherency_ops_instance_unregister(&soc_hha->cci); + iounmap(soc_hha->base); + cache_coherency_ops_instance_put(&soc_hha->cci); +} + +static const struct acpi_device_id hisi_soc_hha_ids[] = { + { "HISI0511", }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_soc_hha_ids); + +static struct platform_driver hisi_soc_hha_driver = { + .driver = { + .name = "hisi_soc_hha", + .acpi_match_table = hisi_soc_hha_ids, + }, + .probe = hisi_soc_hha_probe, + .remove = hisi_soc_hha_remove, +}; + +module_platform_driver(hisi_soc_hha_driver); + +MODULE_IMPORT_NS("CACHE_COHERENCY"); +MODULE_DESCRIPTION("HiSilicon Hydra Home Agent driver supporting cache maintenance"); +MODULE_AUTHOR("Yicong Yang "); +MODULE_AUTHOR("Yushan Wang "); +MODULE_LICENSE("GPL"); From 055bcc552b5181da208038c1de9437e9cca69380 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 17 Nov 2025 11:53:11 +0100 Subject: [PATCH 9/9] MAINTAINERS: refer to intended file in STANDALONE CACHE CONTROLLER DRIVERS Commit 23db6eed72bd ("MAINTAINERS: Add Jonathan Cameron to drivers/cache and add lib/cache_maint.c + header") intends to add a file entry pointing to the cache_coherency.h file, but messes up to name the right path. Update the entry to the intended file. Signed-off-by: Lukas Bulwahn Acked-by: Jonathan Cameron Signed-off-by: Conor Dooley --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 039d10ded9e9..f90593f063ad 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23972,7 +23972,7 @@ S: Maintained T: git https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git/ F: Documentation/devicetree/bindings/cache/ F: drivers/cache -F: include/cache_coherency.h +F: include/linux/cache_coherency.h F: lib/cache_maint.c STARFIRE/DURALAN NETWORK DRIVER