mirrors/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-12-27 11:06:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'for-6.17/cxl-acquire' into cxl-for-next

Browse Source 
Introduce ACQUIRE() and ACQUIRE_ERR() for conditional locks.
Convert CXL subsystem to use the new macros.
This commit is contained in:
Dave Jiang
2025-07-16 13:30:17 -07:00
parent 12b3d697c8 d03fcf50ba
commit b873adfdde
13 changed files with 482 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
drivers/cxl/core/cdat.c
View File

@@ -336,7 +336,7 @@ static int match_cxlrd_hb(struct device *dev, void *data)
cxlrd = to_cxl_root_decoder(dev);
cxlsd = &cxlrd->cxlsd;
guard(rwsem_read)(&cxl_region_rwsem);
guard(rwsem_read)(&cxl_rwsem.region);
for (int i = 0; i < cxlsd->nr_targets; i++) {
if (host_bridge == cxlsd->target[i]->dport_dev)
return 1;
@@ -987,7 +987,7 @@ void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr)
bool is_root;
int rc;
lockdep_assert_held(&cxl_dpa_rwsem);
lockdep_assert_held(&cxl_rwsem.dpa);
struct xarray *usp_xa __free(free_perf_xa) =
kzalloc(sizeof(*usp_xa), GFP_KERNEL);
@@ -1057,7 +1057,7 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
{
struct cxl_dpa_perf *perf;
lockdep_assert_held(&cxl_dpa_rwsem);
lockdep_assert_held(&cxl_rwsem.dpa);
perf = cxled_get_dpa_perf(cxled);
if (IS_ERR(perf))

32
drivers/cxl/core/core.h
View File

@@ -5,6 +5,7 @@
#define __CXL_CORE_H__
#include <cxl/mailbox.h>
#include <linux/rwsem.h>
extern const struct device_type cxl_nvdimm_bridge_type;
extern const struct device_type cxl_nvdimm_type;
@@ -12,6 +13,11 @@ extern const struct device_type cxl_pmu_type;
extern struct attribute_group cxl_base_attribute_group;
enum cxl_detach_mode {
DETACH_ONLY,
DETACH_INVALIDATE,
};
#ifdef CONFIG_CXL_REGION
extern struct device_attribute dev_attr_create_pmem_region;
extern struct device_attribute dev_attr_create_ram_region;
@@ -20,7 +26,11 @@ extern struct device_attribute dev_attr_region;
extern const struct device_type cxl_pmem_region_type;
extern const struct device_type cxl_dax_region_type;
extern const struct device_type cxl_region_type;
void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled);
int cxl_decoder_detach(struct cxl_region *cxlr,
struct cxl_endpoint_decoder *cxled, int pos,
enum cxl_detach_mode mode);
#define CXL_REGION_ATTR(x) (&dev_attr_##x.attr)
#define CXL_REGION_TYPE(x) (&cxl_region_type)
#define SET_CXL_REGION_ATTR(x) (&dev_attr_##x.attr),
@@ -48,7 +58,9 @@ static inline int cxl_get_poison_by_endpoint(struct cxl_port *port)
{
return 0;
}
static inline void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
static inline int cxl_decoder_detach(struct cxl_region *cxlr,
struct cxl_endpoint_decoder *cxled,
int pos, enum cxl_detach_mode mode)
{
}
static inline int cxl_region_init(void)
@@ -97,8 +109,20 @@ u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb);
#define PCI_RCRB_CAP_HDR_NEXT_MASK GENMASK(15, 8)
#define PCI_CAP_EXP_SIZEOF 0x3c
extern struct rw_semaphore cxl_dpa_rwsem;
extern struct rw_semaphore cxl_region_rwsem;
struct cxl_rwsem {
/*
* All changes to HPA (interleave configuration) occur with this
* lock held for write.
*/
struct rw_semaphore region;
/*
* All changes to a device DPA space occur with this lock held
* for write.
*/
struct rw_semaphore dpa;
};
extern struct cxl_rwsem cxl_rwsem;
int cxl_memdev_init(void);
void cxl_memdev_exit(void);

44
drivers/cxl/core/edac.c
View File

@@ -115,10 +115,9 @@ static int cxl_scrub_get_attrbs(struct cxl_patrol_scrub_context *cxl_ps_ctx,
flags, min_cycle);
}
struct rw_semaphore *region_lock __free(rwsem_read_release) =
rwsem_read_intr_acquire(&cxl_region_rwsem);
if (!region_lock)
return -EINTR;
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return ret;
cxlr = cxl_ps_ctx->cxlr;
p = &cxlr->params;
@@ -158,10 +157,9 @@ static int cxl_scrub_set_attrbs_region(struct device *dev,
struct cxl_region *cxlr;
int ret, i;
struct rw_semaphore *region_lock __free(rwsem_read_release) =
rwsem_read_intr_acquire(&cxl_region_rwsem);
if (!region_lock)
return -EINTR;
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return ret;
cxlr = cxl_ps_ctx->cxlr;
p = &cxlr->params;
@@ -1340,16 +1338,15 @@ cxl_mem_perform_sparing(struct device *dev,
struct cxl_memdev_sparing_in_payload sparing_pi;
struct cxl_event_dram *rec = NULL;
u16 validity_flags = 0;
int ret;
struct rw_semaphore *region_lock __free(rwsem_read_release) =
rwsem_read_intr_acquire(&cxl_region_rwsem);
if (!region_lock)
return -EINTR;
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
return ret;
struct rw_semaphore *dpa_lock __free(rwsem_read_release) =
rwsem_read_intr_acquire(&cxl_dpa_rwsem);
if (!dpa_lock)
return -EINTR;
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
return ret;
if (!cxl_sparing_ctx->cap_safe_when_in_use) {
/* Memory to repair must be offline */
@@ -1787,16 +1784,15 @@ static int cxl_mem_perform_ppr(struct cxl_ppr_context *cxl_ppr_ctx)
struct cxl_memdev_ppr_maintenance_attrbs maintenance_attrbs;
struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
struct cxl_mem_repair_attrbs attrbs = { 0 };
int ret;
struct rw_semaphore *region_lock __free(rwsem_read_release) =
rwsem_read_intr_acquire(&cxl_region_rwsem);
if (!region_lock)
return -EINTR;
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
return ret;
struct rw_semaphore *dpa_lock __free(rwsem_read_release) =
rwsem_read_intr_acquire(&cxl_dpa_rwsem);
if (!dpa_lock)
return -EINTR;
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
if ((ret = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
return ret;
if (!cxl_ppr_ctx->media_accessible || !cxl_ppr_ctx->data_retained) {
/* Memory to repair must be offline */

118
drivers/cxl/core/hdm.c
View File

@@ -16,7 +16,10 @@
* for enumerating these registers and capabilities.
*/
DECLARE_RWSEM(cxl_dpa_rwsem);
struct cxl_rwsem cxl_rwsem = {
.region = __RWSEM_INITIALIZER(cxl_rwsem.region),
.dpa = __RWSEM_INITIALIZER(cxl_rwsem.dpa),
};
static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
int *target_map)
@@ -214,7 +217,7 @@ void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
{
struct resource *p1, *p2;
guard(rwsem_read)(&cxl_dpa_rwsem);
guard(rwsem_read)(&cxl_rwsem.dpa);
for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
__cxl_dpa_debug(file, p1, 0);
for (p2 = p1->child; p2; p2 = p2->sibling)
@@ -266,7 +269,7 @@ static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
struct resource *res = cxled->dpa_res;
resource_size_t skip_start;
lockdep_assert_held_write(&cxl_dpa_rwsem);
lockdep_assert_held_write(&cxl_rwsem.dpa);
/* save @skip_start, before @res is released */
skip_start = res->start - cxled->skip;
@@ -281,7 +284,7 @@ static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
static void cxl_dpa_release(void *cxled)
{
guard(rwsem_write)(&cxl_dpa_rwsem);
guard(rwsem_write)(&cxl_rwsem.dpa);
__cxl_dpa_release(cxled);
}
@@ -293,7 +296,7 @@ static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
{
struct cxl_port *port = cxled_to_port(cxled);
lockdep_assert_held_write(&cxl_dpa_rwsem);
lockdep_assert_held_write(&cxl_rwsem.dpa);
devm_remove_action(&port->dev, cxl_dpa_release, cxled);
__cxl_dpa_release(cxled);
}
@@ -361,7 +364,7 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
struct resource *res;
int rc;
lockdep_assert_held_write(&cxl_dpa_rwsem);
lockdep_assert_held_write(&cxl_rwsem.dpa);
if (!len) {
dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
@@ -470,7 +473,7 @@ int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info)
{
struct device *dev = cxlds->dev;
guard(rwsem_write)(&cxl_dpa_rwsem);
guard(rwsem_write)(&cxl_rwsem.dpa);
if (cxlds->nr_partitions)
return -EBUSY;
@@ -516,9 +519,8 @@ int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
struct cxl_port *port = cxled_to_port(cxled);
int rc;
down_write(&cxl_dpa_rwsem);
rc = __cxl_dpa_reserve(cxled, base, len, skipped);
up_write(&cxl_dpa_rwsem);
scoped_guard(rwsem_write, &cxl_rwsem.dpa)
rc = __cxl_dpa_reserve(cxled, base, len, skipped);
if (rc)
return rc;
@@ -529,7 +531,7 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, "CXL");
resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
{
guard(rwsem_read)(&cxl_dpa_rwsem);
guard(rwsem_read)(&cxl_rwsem.dpa);
if (cxled->dpa_res)
return resource_size(cxled->dpa_res);
@@ -540,7 +542,7 @@ resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
{
resource_size_t base = -1;
lockdep_assert_held(&cxl_dpa_rwsem);
lockdep_assert_held(&cxl_rwsem.dpa);
if (cxled->dpa_res)
base = cxled->dpa_res->start;
@@ -559,7 +561,7 @@ int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
struct cxl_port *port = cxled_to_port(cxled);
struct device *dev = &cxled->cxld.dev;
guard(rwsem_write)(&cxl_dpa_rwsem);
guard(rwsem_write)(&cxl_rwsem.dpa);
if (!cxled->dpa_res)
return 0;
if (cxled->cxld.region) {
@@ -589,7 +591,7 @@ int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled,
struct device *dev = &cxled->cxld.dev;
int part;
guard(rwsem_write)(&cxl_dpa_rwsem);
guard(rwsem_write)(&cxl_rwsem.dpa);
if (cxled->cxld.flags & CXL_DECODER_F_ENABLE)
return -EBUSY;
@@ -621,7 +623,7 @@ static int __cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size)
struct resource *p, *last;
int part;
guard(rwsem_write)(&cxl_dpa_rwsem);
guard(rwsem_write)(&cxl_rwsem.dpa);
if (cxled->cxld.region) {
dev_dbg(dev, "decoder attached to %s\n",
dev_name(&cxled->cxld.region->dev));
@@ -771,46 +773,12 @@ static int cxld_await_commit(void __iomem *hdm, int id)
return -ETIMEDOUT;
}
static int cxl_decoder_commit(struct cxl_decoder *cxld)
static void setup_hw_decoder(struct cxl_decoder *cxld, void __iomem *hdm)
{
struct cxl_port *port = to_cxl_port(cxld->dev.parent);
struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
void __iomem *hdm = cxlhdm->regs.hdm_decoder;
int id = cxld->id, rc;
int id = cxld->id;
u64 base, size;
u32 ctrl;
if (cxld->flags & CXL_DECODER_F_ENABLE)
return 0;
if (cxl_num_decoders_committed(port) != id) {
dev_dbg(&port->dev,
"%s: out of order commit, expected decoder%d.%d\n",
dev_name(&cxld->dev), port->id,
cxl_num_decoders_committed(port));
return -EBUSY;
}
/*
* For endpoint decoders hosted on CXL memory devices that
* support the sanitize operation, make sure sanitize is not in-flight.
*/
if (is_endpoint_decoder(&cxld->dev)) {
struct cxl_endpoint_decoder *cxled =
to_cxl_endpoint_decoder(&cxld->dev);
struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
struct cxl_memdev_state *mds =
to_cxl_memdev_state(cxlmd->cxlds);
if (mds && mds->security.sanitize_active) {
dev_dbg(&cxlmd->dev,
"attempted to commit %s during sanitize\n",
dev_name(&cxld->dev));
return -EBUSY;
}
}
down_read(&cxl_dpa_rwsem);
/* common decoder settings */
ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
cxld_set_interleave(cxld, &ctrl);
@@ -844,7 +812,47 @@ static int cxl_decoder_commit(struct cxl_decoder *cxld)
}
writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
up_read(&cxl_dpa_rwsem);
}
static int cxl_decoder_commit(struct cxl_decoder *cxld)
{
struct cxl_port *port = to_cxl_port(cxld->dev.parent);
struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
void __iomem *hdm = cxlhdm->regs.hdm_decoder;
int id = cxld->id, rc;
if (cxld->flags & CXL_DECODER_F_ENABLE)
return 0;
if (cxl_num_decoders_committed(port) != id) {
dev_dbg(&port->dev,
"%s: out of order commit, expected decoder%d.%d\n",
dev_name(&cxld->dev), port->id,
cxl_num_decoders_committed(port));
return -EBUSY;
}
/*
* For endpoint decoders hosted on CXL memory devices that
* support the sanitize operation, make sure sanitize is not in-flight.
*/
if (is_endpoint_decoder(&cxld->dev)) {
struct cxl_endpoint_decoder *cxled =
to_cxl_endpoint_decoder(&cxld->dev);
struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
struct cxl_memdev_state *mds =
to_cxl_memdev_state(cxlmd->cxlds);
if (mds && mds->security.sanitize_active) {
dev_dbg(&cxlmd->dev,
"attempted to commit %s during sanitize\n",
dev_name(&cxld->dev));
return -EBUSY;
}
}
scoped_guard(rwsem_read, &cxl_rwsem.dpa)
setup_hw_decoder(cxld, hdm);
port->commit_end++;
rc = cxld_await_commit(hdm, cxld->id);
@@ -882,7 +890,7 @@ void cxl_port_commit_reap(struct cxl_decoder *cxld)
{
struct cxl_port *port = to_cxl_port(cxld->dev.parent);
lockdep_assert_held_write(&cxl_region_rwsem);
lockdep_assert_held_write(&cxl_rwsem.region);
/*
* Once the highest committed decoder is disabled, free any other
@@ -914,7 +922,6 @@ static void cxl_decoder_reset(struct cxl_decoder *cxld)
"%s: out of order reset, expected decoder%d.%d\n",
dev_name(&cxld->dev), port->id, port->commit_end);
down_read(&cxl_dpa_rwsem);
ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
@@ -923,7 +930,6 @@ static void cxl_decoder_reset(struct cxl_decoder *cxld)
writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
up_read(&cxl_dpa_rwsem);
cxld->flags &= ~CXL_DECODER_F_ENABLE;
@@ -1032,7 +1038,7 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
else
cxld->target_type = CXL_DECODER_DEVMEM;
guard(rwsem_write)(&cxl_region_rwsem);
guard(rwsem_write)(&cxl_rwsem.region);
if (cxld->id != cxl_num_decoders_committed(port)) {
dev_warn(&port->dev,
"decoder%d.%d: Committed out of order\n",

13
drivers/cxl/core/mbox.c
View File

@@ -909,8 +909,8 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
* translations. Take topology mutation locks and lookup
* { HPA, REGION } from { DPA, MEMDEV } in the event record.
*/
guard(rwsem_read)(&cxl_region_rwsem);
guard(rwsem_read)(&cxl_dpa_rwsem);
guard(rwsem_read)(&cxl_rwsem.region);
guard(rwsem_read)(&cxl_rwsem.dpa);
dpa = le64_to_cpu(evt->media_hdr.phys_addr) & CXL_DPA_MASK;
cxlr = cxl_dpa_to_region(cxlmd, dpa);
@@ -1265,7 +1265,7 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)
/* synchronize with cxl_mem_probe() and decoder write operations */
guard(device)(&cxlmd->dev);
endpoint = cxlmd->endpoint;
guard(rwsem_read)(&cxl_region_rwsem);
guard(rwsem_read)(&cxl_rwsem.region);
/*
* Require an endpoint to be safe otherwise the driver can not
* be sure that the device is unmapped.
@@ -1401,8 +1401,8 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
int nr_records = 0;
int rc;
rc = mutex_lock_interruptible(&mds->poison.lock);
if (rc)
ACQUIRE(mutex_intr, lock)(&mds->poison.mutex);
if ((rc = ACQUIRE_ERR(mutex_intr, &lock)))
return rc;
po = mds->poison.list_out;
@@ -1437,7 +1437,6 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
}
} while (po->flags & CXL_POISON_FLAG_MORE);
mutex_unlock(&mds->poison.lock);
return rc;
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, "CXL");
@@ -1473,7 +1472,7 @@ int cxl_poison_state_init(struct cxl_memdev_state *mds)
return rc;
}
mutex_init(&mds->poison.lock);
mutex_init(&mds->poison.mutex);
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, "CXL");

50
drivers/cxl/core/memdev.c
View File

@@ -232,15 +232,13 @@ int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
if (!port || !is_cxl_endpoint(port))
return -EINVAL;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
return rc;
rc = down_read_interruptible(&cxl_dpa_rwsem);
if (rc) {
up_read(&cxl_region_rwsem);
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
return rc;
}
if (cxl_num_decoders_committed(port) == 0) {
/* No regions mapped to this memdev */
@@ -249,8 +247,6 @@ int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
/* Regions mapped, collect poison by endpoint */
rc = cxl_get_poison_by_endpoint(port);
}
up_read(&cxl_dpa_rwsem);
up_read(&cxl_region_rwsem);
return rc;
}
@@ -292,19 +288,17 @@ int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
if (!IS_ENABLED(CONFIG_DEBUG_FS))
return 0;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
return rc;
rc = down_read_interruptible(&cxl_dpa_rwsem);
if (rc) {
up_read(&cxl_region_rwsem);
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
return rc;
}
rc = cxl_validate_poison_dpa(cxlmd, dpa);
if (rc)
goto out;
return rc;
inject.address = cpu_to_le64(dpa);
mbox_cmd = (struct cxl_mbox_cmd) {
@@ -314,7 +308,7 @@ int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
};
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc)
goto out;
return rc;
cxlr = cxl_dpa_to_region(cxlmd, dpa);
if (cxlr)
@@ -327,11 +321,8 @@ int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
.length = cpu_to_le32(1),
};
trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
out:
up_read(&cxl_dpa_rwsem);
up_read(&cxl_region_rwsem);
return rc;
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL");
@@ -347,19 +338,17 @@ int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
if (!IS_ENABLED(CONFIG_DEBUG_FS))
return 0;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
return rc;
rc = down_read_interruptible(&cxl_dpa_rwsem);
if (rc) {
up_read(&cxl_region_rwsem);
ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
return rc;
}
rc = cxl_validate_poison_dpa(cxlmd, dpa);
if (rc)
goto out;
return rc;
/*
* In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
@@ -378,7 +367,7 @@ int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc)
goto out;
return rc;
cxlr = cxl_dpa_to_region(cxlmd, dpa);
if (cxlr)
@@ -391,11 +380,8 @@ int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
.length = cpu_to_le32(1),
};
trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
out:
up_read(&cxl_dpa_rwsem);
up_read(&cxl_region_rwsem);
return rc;
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL");

27
drivers/cxl/core/port.c
View File

@@ -30,18 +30,12 @@
* instantiated by the core.
*/
/*
* All changes to the interleave configuration occur with this lock held
* for write.
*/
DECLARE_RWSEM(cxl_region_rwsem);
static DEFINE_IDA(cxl_port_ida);
static DEFINE_XARRAY(cxl_root_buses);
int cxl_num_decoders_committed(struct cxl_port *port)
{
lockdep_assert_held(&cxl_region_rwsem);
lockdep_assert_held(&cxl_rwsem.region);
return port->commit_end + 1;
}
@@ -176,7 +170,7 @@ static ssize_t target_list_show(struct device *dev,
ssize_t offset;
int rc;
guard(rwsem_read)(&cxl_region_rwsem);
guard(rwsem_read)(&cxl_rwsem.region);
rc = emit_target_list(cxlsd, buf);
if (rc < 0)
return rc;
@@ -196,7 +190,7 @@ static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
/* without @cxl_dpa_rwsem, make sure @part is not reloaded */
/* without @cxl_rwsem.dpa, make sure @part is not reloaded */
int part = READ_ONCE(cxled->part);
const char *desc;
@@ -235,7 +229,7 @@ static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *at
{
struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
guard(rwsem_read)(&cxl_dpa_rwsem);
guard(rwsem_read)(&cxl_rwsem.dpa);
return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
}
static DEVICE_ATTR_RO(dpa_resource);
@@ -560,7 +554,7 @@ static ssize_t decoders_committed_show(struct device *dev,
{
struct cxl_port *port = to_cxl_port(dev);
guard(rwsem_read)(&cxl_region_rwsem);
guard(rwsem_read)(&cxl_rwsem.region);
return sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port));
}
@@ -1722,7 +1716,7 @@ static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
if (xa_empty(&port->dports))
return -EINVAL;
guard(rwsem_write)(&cxl_region_rwsem);
guard(rwsem_write)(&cxl_rwsem.region);
for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
struct cxl_dport *dport = find_dport(port, target_map[i]);
@@ -2001,12 +1995,9 @@ EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL");
static void cxld_unregister(void *dev)
{
struct cxl_endpoint_decoder *cxled;
if (is_endpoint_decoder(dev)) {
cxled = to_cxl_endpoint_decoder(dev);
cxl_decoder_kill_region(cxled);
}
if (is_endpoint_decoder(dev))
cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1,
DETACH_INVALIDATE);
device_unregister(dev);
}

477
drivers/cxl/core/region.c
View File

@@ -141,16 +141,12 @@ static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
struct cxl_region_params *p = &cxlr->params;
ssize_t rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
return rc;
if (cxlr->mode != CXL_PARTMODE_PMEM)
rc = sysfs_emit(buf, "\n");
else
rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
up_read(&cxl_region_rwsem);
return rc;
return sysfs_emit(buf, "\n");
return sysfs_emit(buf, "%pUb\n", &p->uuid);
}
static int is_dup(struct device *match, void *data)
@@ -162,7 +158,7 @@ static int is_dup(struct device *match, void *data)
if (!is_cxl_region(match))
return 0;
lockdep_assert_held(&cxl_region_rwsem);
lockdep_assert_held(&cxl_rwsem.region);
cxlr = to_cxl_region(match);
p = &cxlr->params;
@@ -192,27 +188,22 @@ static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
if (uuid_is_null(&temp))
return -EINVAL;
rc = down_write_killable(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_write_kill, region_rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_write_kill, &region_rwsem)))
return rc;
if (uuid_equal(&p->uuid, &temp))
goto out;
return len;
rc = -EBUSY;
if (p->state >= CXL_CONFIG_ACTIVE)
goto out;
return -EBUSY;
rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
if (rc < 0)
goto out;
return rc;
uuid_copy(&p->uuid, &temp);
out:
up_write(&cxl_region_rwsem);
if (rc)
return rc;
return len;
}
static DEVICE_ATTR_RW(uuid);
@@ -349,6 +340,58 @@ static int cxl_region_decode_commit(struct cxl_region *cxlr)
return rc;
}
static int queue_reset(struct cxl_region *cxlr)
{
struct cxl_region_params *p = &cxlr->params;
int rc;
ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
return rc;
/* Already in the requested state? */
if (p->state < CXL_CONFIG_COMMIT)
return 0;
p->state = CXL_CONFIG_RESET_PENDING;
return 0;
}
static int __commit(struct cxl_region *cxlr)
{
struct cxl_region_params *p = &cxlr->params;
int rc;
ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
return rc;
/* Already in the requested state? */
if (p->state >= CXL_CONFIG_COMMIT)
return 0;
/* Not ready to commit? */
if (p->state < CXL_CONFIG_ACTIVE)
return -ENXIO;
/*
* Invalidate caches before region setup to drop any speculative
* consumption of this address space
*/
rc = cxl_region_invalidate_memregion(cxlr);
if (rc)
return rc;
rc = cxl_region_decode_commit(cxlr);
if (rc)
return rc;
p->state = CXL_CONFIG_COMMIT;
return 0;
}
static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t len)
{
@@ -361,55 +404,38 @@ static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
if (rc)
return rc;
rc = down_write_killable(&cxl_region_rwsem);
if (commit) {
rc = __commit(cxlr);
if (rc)
return rc;
return len;
}
rc = queue_reset(cxlr);
if (rc)
return rc;
/* Already in the requested state? */
if (commit && p->state >= CXL_CONFIG_COMMIT)
goto out;
if (!commit && p->state < CXL_CONFIG_COMMIT)
goto out;
/* Not ready to commit? */
if (commit && p->state < CXL_CONFIG_ACTIVE) {
rc = -ENXIO;
goto out;
}
/*
* Invalidate caches before region setup to drop any speculative
* consumption of this address space
* Unmap the region and depend the reset-pending state to ensure
* it does not go active again until post reset
*/
rc = cxl_region_invalidate_memregion(cxlr);
if (rc)
goto out;
device_release_driver(&cxlr->dev);
if (commit) {
rc = cxl_region_decode_commit(cxlr);
if (rc == 0)
p->state = CXL_CONFIG_COMMIT;
} else {
p->state = CXL_CONFIG_RESET_PENDING;
up_write(&cxl_region_rwsem);
device_release_driver(&cxlr->dev);
down_write(&cxl_region_rwsem);
/*
* With the reset pending take cxl_rwsem.region unconditionally
* to ensure the reset gets handled before returning.
*/
guard(rwsem_write)(&cxl_rwsem.region);
/*
* The lock was dropped, so need to revalidate that the reset is
* still pending.
*/
if (p->state == CXL_CONFIG_RESET_PENDING) {
cxl_region_decode_reset(cxlr, p->interleave_ways);
p->state = CXL_CONFIG_ACTIVE;
}
/*
* Revalidate that the reset is still pending in case another
* thread already handled this reset.
*/
if (p->state == CXL_CONFIG_RESET_PENDING) {
cxl_region_decode_reset(cxlr, p->interleave_ways);
p->state = CXL_CONFIG_ACTIVE;
}
out:
up_write(&cxl_region_rwsem);
if (rc)
return rc;
return len;
}
@@ -420,13 +446,10 @@ static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
struct cxl_region_params *p = &cxlr->params;
ssize_t rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return rc;
rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
up_read(&cxl_region_rwsem);
return rc;
return sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
}
static DEVICE_ATTR_RW(commit);
@@ -450,15 +473,12 @@ static ssize_t interleave_ways_show(struct device *dev,
{
struct cxl_region *cxlr = to_cxl_region(dev);
struct cxl_region_params *p = &cxlr->params;
ssize_t rc;
int rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return rc;
rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
up_read(&cxl_region_rwsem);
return rc;
return sysfs_emit(buf, "%d\n", p->interleave_ways);
}
static const struct attribute_group *get_cxl_region_target_group(void);
@@ -493,23 +513,21 @@ static ssize_t interleave_ways_store(struct device *dev,
return -EINVAL;
}
rc = down_write_killable(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
return rc;
if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
rc = -EBUSY;
goto out;
}
if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
return -EBUSY;
save = p->interleave_ways;
p->interleave_ways = val;
rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
if (rc)
if (rc) {
p->interleave_ways = save;
out:
up_write(&cxl_region_rwsem);
if (rc)
return rc;
}
return len;
}
static DEVICE_ATTR_RW(interleave_ways);
@@ -520,15 +538,12 @@ static ssize_t interleave_granularity_show(struct device *dev,
{
struct cxl_region *cxlr = to_cxl_region(dev);
struct cxl_region_params *p = &cxlr->params;
ssize_t rc;
int rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return rc;
rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
up_read(&cxl_region_rwsem);
return rc;
return sysfs_emit(buf, "%d\n", p->interleave_granularity);
}
static ssize_t interleave_granularity_store(struct device *dev,
@@ -561,19 +576,15 @@ static ssize_t interleave_granularity_store(struct device *dev,
if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
return -EINVAL;
rc = down_write_killable(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
return rc;
if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
rc = -EBUSY;
goto out;
}
if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
return -EBUSY;
p->interleave_granularity = val;
out:
up_write(&cxl_region_rwsem);
if (rc)
return rc;
return len;
}
static DEVICE_ATTR_RW(interleave_granularity);
@@ -584,17 +595,15 @@ static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
struct cxl_region *cxlr = to_cxl_region(dev);
struct cxl_region_params *p = &cxlr->params;
u64 resource = -1ULL;
ssize_t rc;
int rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return rc;
if (p->res)
resource = p->res->start;
rc = sysfs_emit(buf, "%#llx\n", resource);
up_read(&cxl_region_rwsem);
return rc;
return sysfs_emit(buf, "%#llx\n", resource);
}
static DEVICE_ATTR_RO(resource);
@@ -622,7 +631,7 @@ static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
struct resource *res;
u64 remainder = 0;
lockdep_assert_held_write(&cxl_region_rwsem);
lockdep_assert_held_write(&cxl_rwsem.region);
/* Nothing to do... */
if (p->res && resource_size(p->res) == size)
@@ -664,7 +673,7 @@ static void cxl_region_iomem_release(struct cxl_region *cxlr)
struct cxl_region_params *p = &cxlr->params;
if (device_is_registered(&cxlr->dev))
lockdep_assert_held_write(&cxl_region_rwsem);
lockdep_assert_held_write(&cxl_rwsem.region);
if (p->res) {
/*
* Autodiscovered regions may not have been able to insert their
@@ -681,7 +690,7 @@ static int free_hpa(struct cxl_region *cxlr)
{
struct cxl_region_params *p = &cxlr->params;
lockdep_assert_held_write(&cxl_region_rwsem);
lockdep_assert_held_write(&cxl_rwsem.region);
if (!p->res)
return 0;
@@ -705,15 +714,14 @@ static ssize_t size_store(struct device *dev, struct device_attribute *attr,
if (rc)
return rc;
rc = down_write_killable(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
return rc;
if (val)
rc = alloc_hpa(cxlr, val);
else
rc = free_hpa(cxlr);
up_write(&cxl_region_rwsem);
if (rc)
return rc;
@@ -729,15 +737,12 @@ static ssize_t size_show(struct device *dev, struct device_attribute *attr,
u64 size = 0;
ssize_t rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return rc;
if (p->res)
size = resource_size(p->res);
rc = sysfs_emit(buf, "%#llx\n", size);
up_read(&cxl_region_rwsem);
return rc;
return sysfs_emit(buf, "%#llx\n", size);
}
static DEVICE_ATTR_RW(size);
@@ -763,26 +768,20 @@ static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
struct cxl_endpoint_decoder *cxled;
int rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return rc;
if (pos >= p->interleave_ways) {
dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
p->interleave_ways);
rc = -ENXIO;
goto out;
return -ENXIO;
}
cxled = p->targets[pos];
if (!cxled)
rc = sysfs_emit(buf, "\n");
else
rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
out:
up_read(&cxl_region_rwsem);
return rc;
return sysfs_emit(buf, "\n");
return sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
}
static int check_commit_order(struct device *dev, void *data)
@@ -897,7 +896,7 @@ cxl_port_pick_region_decoder(struct cxl_port *port,
/*
* This decoder is pinned registered as long as the endpoint decoder is
* registered, and endpoint decoder unregistration holds the
* cxl_region_rwsem over unregister events, so no need to hold on to
* cxl_rwsem.region over unregister events, so no need to hold on to
* this extra reference.
*/
put_device(dev);
@@ -1088,7 +1087,7 @@ static int cxl_port_attach_region(struct cxl_port *port,
unsigned long index;
int rc = -EBUSY;
lockdep_assert_held_write(&cxl_region_rwsem);
lockdep_assert_held_write(&cxl_rwsem.region);
cxl_rr = cxl_rr_load(port, cxlr);
if (cxl_rr) {
@@ -1198,7 +1197,7 @@ static void cxl_port_detach_region(struct cxl_port *port,
struct cxl_region_ref *cxl_rr;
struct cxl_ep *ep = NULL;
lockdep_assert_held_write(&cxl_region_rwsem);
lockdep_assert_held_write(&cxl_rwsem.region);
cxl_rr = cxl_rr_load(port, cxlr);
if (!cxl_rr)
@@ -2094,27 +2093,43 @@ static int cxl_region_attach(struct cxl_region *cxlr,
return 0;
}
static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
static struct cxl_region *
__cxl_decoder_detach(struct cxl_region *cxlr,
struct cxl_endpoint_decoder *cxled, int pos,
enum cxl_detach_mode mode)
{
struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
struct cxl_region *cxlr = cxled->cxld.region;
struct cxl_region_params *p;
int rc = 0;
lockdep_assert_held_write(&cxl_region_rwsem);
lockdep_assert_held_write(&cxl_rwsem.region);
if (!cxlr)
return 0;
if (!cxled) {
p = &cxlr->params;
p = &cxlr->params;
get_device(&cxlr->dev);
if (pos >= p->interleave_ways) {
dev_dbg(&cxlr->dev, "position %d out of range %d\n",
pos, p->interleave_ways);
return ERR_PTR(-ENXIO);
}
if (!p->targets[pos])
return NULL;
cxled = p->targets[pos];
} else {
cxlr = cxled->cxld.region;
if (!cxlr)
return NULL;
p = &cxlr->params;
}
if (mode == DETACH_INVALIDATE)
cxled->part = -1;
if (p->state > CXL_CONFIG_ACTIVE) {
cxl_region_decode_reset(cxlr, p->interleave_ways);
p->state = CXL_CONFIG_ACTIVE;
}
for (iter = ep_port; !is_cxl_root(iter);
for (struct cxl_port *iter = cxled_to_port(cxled); !is_cxl_root(iter);
iter = to_cxl_port(iter->dev.parent))
cxl_port_detach_region(iter, cxlr, cxled);
@@ -2125,7 +2140,7 @@ static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
cxled->pos);
goto out;
return NULL;
}
if (p->state == CXL_CONFIG_ACTIVE) {
@@ -2139,74 +2154,79 @@ static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
.end = -1,
};
/* notify the region driver that one of its targets has departed */
up_write(&cxl_region_rwsem);
device_release_driver(&cxlr->dev);
down_write(&cxl_region_rwsem);
out:
put_device(&cxlr->dev);
return rc;
get_device(&cxlr->dev);
return cxlr;
}
void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
/*
* Cleanup a decoder's interest in a region. There are 2 cases to
* handle, removing an unknown @cxled from a known position in a region
* (detach_target()) or removing a known @cxled from an unknown @cxlr
* (cxld_unregister())
*
* When the detachment finds a region release the region driver.
*/
int cxl_decoder_detach(struct cxl_region *cxlr,
struct cxl_endpoint_decoder *cxled, int pos,
enum cxl_detach_mode mode)
{
down_write(&cxl_region_rwsem);
cxled->part = -1;
cxl_region_detach(cxled);
up_write(&cxl_region_rwsem);
struct cxl_region *detach;
/* when the decoder is being destroyed lock unconditionally */
if (mode == DETACH_INVALIDATE) {
guard(rwsem_write)(&cxl_rwsem.region);
detach = __cxl_decoder_detach(cxlr, cxled, pos, mode);
} else {
int rc;
ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
return rc;
detach = __cxl_decoder_detach(cxlr, cxled, pos, mode);
}
if (detach) {
device_release_driver(&detach->dev);
put_device(&detach->dev);
}
return 0;
}
static int __attach_target(struct cxl_region *cxlr,
struct cxl_endpoint_decoder *cxled, int pos,
unsigned int state)
{
int rc;
if (state == TASK_INTERRUPTIBLE) {
ACQUIRE(rwsem_write_kill, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_write_kill, &rwsem)))
return rc;
guard(rwsem_read)(&cxl_rwsem.dpa);
return cxl_region_attach(cxlr, cxled, pos);
}
guard(rwsem_write)(&cxl_rwsem.region);
guard(rwsem_read)(&cxl_rwsem.dpa);
return cxl_region_attach(cxlr, cxled, pos);
}
static int attach_target(struct cxl_region *cxlr,
struct cxl_endpoint_decoder *cxled, int pos,
unsigned int state)
{
int rc = 0;
int rc = __attach_target(cxlr, cxled, pos, state);
if (state == TASK_INTERRUPTIBLE)
rc = down_write_killable(&cxl_region_rwsem);
else
down_write(&cxl_region_rwsem);
if (rc)
return rc;
down_read(&cxl_dpa_rwsem);
rc = cxl_region_attach(cxlr, cxled, pos);
up_read(&cxl_dpa_rwsem);
up_write(&cxl_region_rwsem);
if (rc)
dev_warn(cxled->cxld.dev.parent,
"failed to attach %s to %s: %d\n",
dev_name(&cxled->cxld.dev), dev_name(&cxlr->dev), rc);
if (rc == 0)
return 0;
dev_warn(cxled->cxld.dev.parent, "failed to attach %s to %s: %d\n",
dev_name(&cxled->cxld.dev), dev_name(&cxlr->dev), rc);
return rc;
}
static int detach_target(struct cxl_region *cxlr, int pos)
{
struct cxl_region_params *p = &cxlr->params;
int rc;
rc = down_write_killable(&cxl_region_rwsem);
if (rc)
return rc;
if (pos >= p->interleave_ways) {
dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
p->interleave_ways);
rc = -ENXIO;
goto out;
}
if (!p->targets[pos]) {
rc = 0;
goto out;
}
rc = cxl_region_detach(p->targets[pos]);
out:
up_write(&cxl_region_rwsem);
return rc;
return cxl_decoder_detach(cxlr, NULL, pos, DETACH_ONLY);
}
static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
@@ -2460,7 +2480,7 @@ static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
return NOTIFY_DONE;
/*
* No need to hold cxl_region_rwsem; region parameters are stable
* No need to hold cxl_rwsem.region; region parameters are stable
* within the cxl_region driver.
*/
region_nid = phys_to_target_node(cxlr->params.res->start);
@@ -2483,7 +2503,7 @@ static int cxl_region_calculate_adistance(struct notifier_block *nb,
int region_nid;
/*
* No need to hold cxl_region_rwsem; region parameters are stable
* No need to hold cxl_rwsem.region; region parameters are stable
* within the cxl_region driver.
*/
region_nid = phys_to_target_node(cxlr->params.res->start);
@@ -2632,17 +2652,13 @@ static ssize_t region_show(struct device *dev, struct device_attribute *attr,
struct cxl_decoder *cxld = to_cxl_decoder(dev);
ssize_t rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc)
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))
return rc;
if (cxld->region)
rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
else
rc = sysfs_emit(buf, "\n");
up_read(&cxl_region_rwsem);
return rc;
return sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
return sysfs_emit(buf, "\n");
}
DEVICE_ATTR_RO(region);
@@ -2981,7 +2997,7 @@ static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
struct device *dev;
int i;
guard(rwsem_read)(&cxl_region_rwsem);
guard(rwsem_read)(&cxl_rwsem.region);
if (p->state != CXL_CONFIG_COMMIT)
return -ENXIO;
@@ -2993,7 +3009,7 @@ static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
cxlr_pmem->hpa_range.start = p->res->start;
cxlr_pmem->hpa_range.end = p->res->end;
/* Snapshot the region configuration underneath the cxl_region_rwsem */
/* Snapshot the region configuration underneath the cxl_rwsem.region */
cxlr_pmem->nr_mappings = p->nr_targets;
for (i = 0; i < p->nr_targets; i++) {
struct cxl_endpoint_decoder *cxled = p->targets[i];
@@ -3070,7 +3086,7 @@ static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
struct cxl_dax_region *cxlr_dax;
struct device *dev;
guard(rwsem_read)(&cxl_region_rwsem);
guard(rwsem_read)(&cxl_rwsem.region);
if (p->state != CXL_CONFIG_COMMIT)
return ERR_PTR(-ENXIO);
@@ -3270,7 +3286,7 @@ static int match_region_by_range(struct device *dev, const void *data)
cxlr = to_cxl_region(dev);
p = &cxlr->params;
guard(rwsem_read)(&cxl_region_rwsem);
guard(rwsem_read)(&cxl_rwsem.region);
if (p->res && p->res->start == r->start && p->res->end == r->end)
return 1;
@@ -3325,7 +3341,7 @@ static int __construct_region(struct cxl_region *cxlr,
struct resource *res;
int rc;
guard(rwsem_write)(&cxl_region_rwsem);
guard(rwsem_write)(&cxl_rwsem.region);
p = &cxlr->params;
if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
dev_err(cxlmd->dev.parent,
@@ -3461,10 +3477,10 @@ int cxl_add_to_region(struct cxl_endpoint_decoder *cxled)
attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
down_read(&cxl_region_rwsem);
p = &cxlr->params;
attach = p->state == CXL_CONFIG_COMMIT;
up_read(&cxl_region_rwsem);
scoped_guard(rwsem_read, &cxl_rwsem.region) {
p = &cxlr->params;
attach = p->state == CXL_CONFIG_COMMIT;
}
if (attach) {
/*
@@ -3489,7 +3505,7 @@ u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa)
if (!endpoint)
return ~0ULL;
guard(rwsem_write)(&cxl_region_rwsem);
guard(rwsem_write)(&cxl_rwsem.region);
xa_for_each(&endpoint->regions, index, iter) {
struct cxl_region_params *p = &iter->region->params;
@@ -3526,40 +3542,45 @@ static void shutdown_notifiers(void *_cxlr)
unregister_mt_adistance_algorithm(&cxlr->adist_notifier);
}
static int cxl_region_probe(struct device *dev)
static int cxl_region_can_probe(struct cxl_region *cxlr)
{
struct cxl_region *cxlr = to_cxl_region(dev);
struct cxl_region_params *p = &cxlr->params;
int rc;
rc = down_read_interruptible(&cxl_region_rwsem);
if (rc) {
ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);
if ((rc = ACQUIRE_ERR(rwsem_read_intr, &rwsem))) {
dev_dbg(&cxlr->dev, "probe interrupted\n");
return rc;
}
if (p->state < CXL_CONFIG_COMMIT) {
dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
rc = -ENXIO;
goto out;
return -ENXIO;
}
if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
dev_err(&cxlr->dev,
"failed to activate, re-commit region and retry\n");
rc = -ENXIO;
goto out;
return -ENXIO;
}
return 0;
}
static int cxl_region_probe(struct device *dev)
{
struct cxl_region *cxlr = to_cxl_region(dev);
struct cxl_region_params *p = &cxlr->params;
int rc;
rc = cxl_region_can_probe(cxlr);
if (rc)
return rc;
/*
* From this point on any path that changes the region's state away from
* CXL_CONFIG_COMMIT is also responsible for releasing the driver.
*/
out:
up_read(&cxl_region_rwsem);
if (rc)
return rc;
cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;

13
drivers/cxl/cxl.h
View File

@@ -471,7 +471,7 @@ enum cxl_config_state {
* @nr_targets: number of targets
* @cache_size: extended linear cache size if exists, otherwise zero.
*
* State transitions are protected by the cxl_region_rwsem
* State transitions are protected by cxl_rwsem.region
*/
struct cxl_region_params {
enum cxl_config_state state;
@@ -914,15 +914,4 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);
#endif
u16 cxl_gpf_get_dvsec(struct device *dev);
static inline struct rw_semaphore *rwsem_read_intr_acquire(struct rw_semaphore *rwsem)
{
if (down_read_interruptible(rwsem))
return NULL;
return rwsem;
}
DEFINE_FREE(rwsem_read_release, struct rw_semaphore *, if (_T) up_read(_T))
#endif /* __CXL_H__ */

4
drivers/cxl/cxlmem.h
View File

@@ -254,7 +254,7 @@ enum security_cmd_enabled_bits {
* @max_errors: Maximum media error records held in device cache
* @enabled_cmds: All poison commands enabled in the CEL
* @list_out: The poison list payload returned by device
* @lock: Protect reads of the poison list
* @mutex: Protect reads of the poison list
*
* Reads of the poison list are synchronized to ensure that a reader
* does not get an incomplete list because their request overlapped
@@ -265,7 +265,7 @@ struct cxl_poison_state {
u32 max_errors;
DECLARE_BITMAP(enabled_cmds, CXL_POISON_ENABLED_MAX);
struct cxl_mbox_poison_out *list_out;
struct mutex lock; /* Protect reads of poison list */
struct mutex mutex; /* Protect reads of poison list */
};
/*

95
include/linux/cleanup.h
View File

@@ -3,6 +3,8 @@
#define _LINUX_CLEANUP_H
#include <linux/compiler.h>
#include <linux/err.h>
#include <linux/args.h>
/**
* DOC: scope-based cleanup helpers
@@ -61,9 +63,21 @@
* Observe the lock is held for the remainder of the "if ()" block not
* the remainder of "func()".
*
* Now, when a function uses both __free() and guard(), or multiple
* instances of __free(), the LIFO order of variable definition order
* matters. GCC documentation says:
* The ACQUIRE() macro can be used in all places that guard() can be
* used and additionally support conditional locks
*
*
* DEFINE_GUARD_COND(pci_dev, _try, pci_dev_trylock(_T))
* ...
* ACQUIRE(pci_dev_try, lock)(dev);
* rc = ACQUIRE_ERR(pci_dev_try, &lock);
* if (rc)
* return rc;
* // @lock is held
*
* Now, when a function uses both __free() and guard()/ACQUIRE(), or
* multiple instances of __free(), the LIFO order of variable definition
* order matters. GCC documentation says:
*
* "When multiple variables in the same scope have cleanup attributes,
* at exit from the scope their associated cleanup functions are run in
@@ -305,14 +319,46 @@ static inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \
* acquire fails.
*
* Only for conditional locks.
*
* ACQUIRE(name, var):
* a named instance of the (guard) class, suitable for conditional
* locks when paired with ACQUIRE_ERR().
*
* ACQUIRE_ERR(name, &var):
* a helper that is effectively a PTR_ERR() conversion of the guard
* pointer. Returns 0 when the lock was acquired and a negative
* error code otherwise.
*/
#define __DEFINE_CLASS_IS_CONDITIONAL(_name, _is_cond) \
static __maybe_unused const bool class_##_name##_is_conditional = _is_cond
#define __DEFINE_GUARD_LOCK_PTR(_name, _exp) \
static inline void * class_##_name##_lock_ptr(class_##_name##_t *_T) \
{ return (void *)(__force unsigned long)*(_exp); }
#define __GUARD_IS_ERR(_ptr) \
({ \
unsigned long _rc = (__force unsigned long)(_ptr); \
unlikely((_rc - 1) >= -MAX_ERRNO - 1); \
})
#define __DEFINE_GUARD_LOCK_PTR(_name, _exp) \
static inline void *class_##_name##_lock_ptr(class_##_name##_t *_T) \
{ \
void *_ptr = (void *)(__force unsigned long)*(_exp); \
if (IS_ERR(_ptr)) { \
_ptr = NULL; \
} \
return _ptr; \
} \
static inline int class_##_name##_lock_err(class_##_name##_t *_T) \
{ \
long _rc = (__force unsigned long)*(_exp); \
if (!_rc) { \
_rc = -EBUSY; \
} \
if (!IS_ERR_VALUE(_rc)) { \
_rc = 0; \
} \
return _rc; \
}
#define DEFINE_CLASS_IS_GUARD(_name) \
__DEFINE_CLASS_IS_CONDITIONAL(_name, false); \
@@ -323,23 +369,37 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond
__DEFINE_GUARD_LOCK_PTR(_name, _T)
#define DEFINE_GUARD(_name, _type, _lock, _unlock) \
DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \
DEFINE_CLASS(_name, _type, if (!__GUARD_IS_ERR(_T)) { _unlock; }, ({ _lock; _T; }), _type _T); \
DEFINE_CLASS_IS_GUARD(_name)
#define DEFINE_GUARD_COND(_name, _ext, _condlock) \
#define DEFINE_GUARD_COND_4(_name, _ext, _lock, _cond) \
__DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \
EXTEND_CLASS(_name, _ext, \
({ void *_t = _T; if (_T && !(_condlock)) _t = NULL; _t; }), \
({ void *_t = _T; int _RET = (_lock); if (_T && !(_cond)) _t = ERR_PTR(_RET); _t; }), \
class_##_name##_t _T) \
static inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \
{ return class_##_name##_lock_ptr(_T); }
{ return class_##_name##_lock_ptr(_T); } \
static inline int class_##_name##_ext##_lock_err(class_##_name##_t *_T) \
{ return class_##_name##_lock_err(_T); }
/*
* Default binary condition; success on 'true'.
*/
#define DEFINE_GUARD_COND_3(_name, _ext, _lock) \
DEFINE_GUARD_COND_4(_name, _ext, _lock, _RET)
#define DEFINE_GUARD_COND(X...) CONCATENATE(DEFINE_GUARD_COND_, COUNT_ARGS(X))(X)
#define guard(_name) \
CLASS(_name, __UNIQUE_ID(guard))
#define __guard_ptr(_name) class_##_name##_lock_ptr
#define __guard_err(_name) class_##_name##_lock_err
#define __is_cond_ptr(_name) class_##_name##_is_conditional
#define ACQUIRE(_name, _var) CLASS(_name, _var)
#define ACQUIRE_ERR(_name, _var) __guard_err(_name)(_var)
/*
* Helper macro for scoped_guard().
*
@@ -401,7 +461,7 @@ typedef struct { \
\
static inline void class_##_name##_destructor(class_##_name##_t *_T) \
{ \
if (_T->lock) { _unlock; } \
if (!__GUARD_IS_ERR(_T->lock)) { _unlock; } \
} \
\
__DEFINE_GUARD_LOCK_PTR(_name, &_T->lock)
@@ -433,15 +493,22 @@ __DEFINE_CLASS_IS_CONDITIONAL(_name, false); \
__DEFINE_UNLOCK_GUARD(_name, void, _unlock, __VA_ARGS__) \
__DEFINE_LOCK_GUARD_0(_name, _lock)
#define DEFINE_LOCK_GUARD_1_COND(_name, _ext, _condlock) \
#define DEFINE_LOCK_GUARD_1_COND_4(_name, _ext, _lock, _cond) \
__DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \
EXTEND_CLASS(_name, _ext, \
({ class_##_name##_t _t = { .lock = l }, *_T = &_t;\
if (_T->lock && !(_condlock)) _T->lock = NULL; \
int _RET = (_lock); \
if (_T->lock && !(_cond)) _T->lock = ERR_PTR(_RET);\
_t; }), \
typeof_member(class_##_name##_t, lock) l) \
static inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \
{ return class_##_name##_lock_ptr(_T); }
{ return class_##_name##_lock_ptr(_T); } \
static inline int class_##_name##_ext##_lock_err(class_##_name##_t *_T) \
{ return class_##_name##_lock_err(_T); }
#define DEFINE_LOCK_GUARD_1_COND_3(_name, _ext, _lock) \
DEFINE_LOCK_GUARD_1_COND_4(_name, _ext, _lock, _RET)
#define DEFINE_LOCK_GUARD_1_COND(X...) CONCATENATE(DEFINE_LOCK_GUARD_1_COND_, COUNT_ARGS(X))(X)
#endif /* _LINUX_CLEANUP_H */

2
include/linux/mutex.h
View File

@@ -224,7 +224,7 @@ extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);
DEFINE_GUARD(mutex, struct mutex *, mutex_lock(_T), mutex_unlock(_T))
DEFINE_GUARD_COND(mutex, _try, mutex_trylock(_T))
DEFINE_GUARD_COND(mutex, _intr, mutex_lock_interruptible(_T) == 0)
DEFINE_GUARD_COND(mutex, _intr, mutex_lock_interruptible(_T), _RET == 0)
extern unsigned long mutex_get_owner(struct mutex *lock);

3
include/linux/rwsem.h
View File

@@ -240,10 +240,11 @@ extern void up_write(struct rw_semaphore *sem);
DEFINE_GUARD(rwsem_read, struct rw_semaphore *, down_read(_T), up_read(_T))
DEFINE_GUARD_COND(rwsem_read, _try, down_read_trylock(_T))
DEFINE_GUARD_COND(rwsem_read, _intr, down_read_interruptible(_T) == 0)
DEFINE_GUARD_COND(rwsem_read, _intr, down_read_interruptible(_T), _RET == 0)
DEFINE_GUARD(rwsem_write, struct rw_semaphore *, down_write(_T), up_write(_T))
DEFINE_GUARD_COND(rwsem_write, _try, down_write_trylock(_T))
DEFINE_GUARD_COND(rwsem_write, _kill, down_write_killable(_T), _RET == 0)
/*
* downgrade write lock to read lock