Commit b873adfd authored by Dave Jiang's avatar Dave Jiang
Browse files

Merge branch 'for-6.17/cxl-acquire' into cxl-for-next 

Introduce ACQUIRE() and ACQUIRE_ERR() for conditional locks.
Convert CXL subsystem to use the new macros.
parents 12b3d697 d03fcf50

drivers/cxl/core/cdat.c

+3 −3
Original line number Diff line number Diff line
@@ -336,7 +336,7 @@ static int match_cxlrd_hb(struct device *dev, void *data) 
	cxlrd = to_cxl_root_decoder(dev);

	cxlsd = &cxlrd->cxlsd;



	guard(rwsem_read)(&cxl_region_rwsem);

	guard(rwsem_read)(&cxl_rwsem.region);

	for (int i = 0; i < cxlsd->nr_targets; i++) {

		if (host_bridge == cxlsd->target[i]->dport_dev)

			return 1;
@@ -987,7 +987,7 @@ void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr) 
	bool is_root;

	int rc;



	lockdep_assert_held(&cxl_dpa_rwsem);

	lockdep_assert_held(&cxl_rwsem.dpa);



	struct xarray *usp_xa __free(free_perf_xa) =

		kzalloc(sizeof(*usp_xa), GFP_KERNEL);
@@ -1057,7 +1057,7 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr, 
{

	struct cxl_dpa_perf *perf;



	lockdep_assert_held(&cxl_dpa_rwsem);

	lockdep_assert_held(&cxl_rwsem.dpa);



	perf = cxled_get_dpa_perf(cxled);

	if (IS_ERR(perf))

drivers/cxl/core/core.h

+28 −4
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
#define __CXL_CORE_H__



#include <cxl/mailbox.h>

#include <linux/rwsem.h>



extern const struct device_type cxl_nvdimm_bridge_type;

extern const struct device_type cxl_nvdimm_type;
@@ -12,6 +13,11 @@ extern const struct device_type cxl_pmu_type; 


extern struct attribute_group cxl_base_attribute_group;



enum cxl_detach_mode {

	DETACH_ONLY,

	DETACH_INVALIDATE,

};



#ifdef CONFIG_CXL_REGION

extern struct device_attribute dev_attr_create_pmem_region;

extern struct device_attribute dev_attr_create_ram_region;
@@ -20,7 +26,11 @@ extern struct device_attribute dev_attr_region; 
extern const struct device_type cxl_pmem_region_type;

extern const struct device_type cxl_dax_region_type;

extern const struct device_type cxl_region_type;

void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled);



int cxl_decoder_detach(struct cxl_region *cxlr,

		       struct cxl_endpoint_decoder *cxled, int pos,

		       enum cxl_detach_mode mode);



#define CXL_REGION_ATTR(x) (&dev_attr_##x.attr)

#define CXL_REGION_TYPE(x) (&cxl_region_type)

#define SET_CXL_REGION_ATTR(x) (&dev_attr_##x.attr),
@@ -48,7 +58,9 @@ static inline int cxl_get_poison_by_endpoint(struct cxl_port *port) 
{

	return 0;

}

static inline void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)

static inline int cxl_decoder_detach(struct cxl_region *cxlr,

				     struct cxl_endpoint_decoder *cxled,

				     int pos, enum cxl_detach_mode mode)

{

}

static inline int cxl_region_init(void)
@@ -97,8 +109,20 @@ u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb); 
#define PCI_RCRB_CAP_HDR_NEXT_MASK	GENMASK(15, 8)

#define PCI_CAP_EXP_SIZEOF		0x3c



extern struct rw_semaphore cxl_dpa_rwsem;

extern struct rw_semaphore cxl_region_rwsem;

struct cxl_rwsem {

	/*

	 * All changes to HPA (interleave configuration) occur with this

	 * lock held for write.

	 */

	struct rw_semaphore region;

	/*

	 * All changes to a device DPA space occur with this lock held

	 * for write.

	 */

	struct rw_semaphore dpa;

};



extern struct cxl_rwsem cxl_rwsem;



int cxl_memdev_init(void);

void cxl_memdev_exit(void);

drivers/cxl/core/edac.c

+20 −24
Original line number Diff line number Diff line
@@ -115,10 +115,9 @@ static int cxl_scrub_get_attrbs(struct cxl_patrol_scrub_context *cxl_ps_ctx, 
						flags, min_cycle);

	}



	struct rw_semaphore *region_lock __free(rwsem_read_release) =

		rwsem_read_intr_acquire(&cxl_region_rwsem);

	if (!region_lock)

		return -EINTR;

	ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);

	if ((ret = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))

		return ret;



	cxlr = cxl_ps_ctx->cxlr;

	p = &cxlr->params;
@@ -158,10 +157,9 @@ static int cxl_scrub_set_attrbs_region(struct device *dev, 
	struct cxl_region *cxlr;

	int ret, i;



	struct rw_semaphore *region_lock __free(rwsem_read_release) =

		rwsem_read_intr_acquire(&cxl_region_rwsem);

	if (!region_lock)

		return -EINTR;

	ACQUIRE(rwsem_read_intr, rwsem)(&cxl_rwsem.region);

	if ((ret = ACQUIRE_ERR(rwsem_read_intr, &rwsem)))

		return ret;



	cxlr = cxl_ps_ctx->cxlr;

	p = &cxlr->params;
@@ -1340,16 +1338,15 @@ cxl_mem_perform_sparing(struct device *dev, 
	struct cxl_memdev_sparing_in_payload sparing_pi;

	struct cxl_event_dram *rec = NULL;

	u16 validity_flags = 0;

	int ret;



	struct rw_semaphore *region_lock __free(rwsem_read_release) =

		rwsem_read_intr_acquire(&cxl_region_rwsem);

	if (!region_lock)

		return -EINTR;

	ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);

	if ((ret = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))

		return ret;



	struct rw_semaphore *dpa_lock __free(rwsem_read_release) =

		rwsem_read_intr_acquire(&cxl_dpa_rwsem);

	if (!dpa_lock)

		return -EINTR;

	ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);

	if ((ret = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))

		return ret;



	if (!cxl_sparing_ctx->cap_safe_when_in_use) {

		/* Memory to repair must be offline */
@@ -1787,16 +1784,15 @@ static int cxl_mem_perform_ppr(struct cxl_ppr_context *cxl_ppr_ctx) 
	struct cxl_memdev_ppr_maintenance_attrbs maintenance_attrbs;

	struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;

	struct cxl_mem_repair_attrbs attrbs = { 0 };

	int ret;



	struct rw_semaphore *region_lock __free(rwsem_read_release) =

		rwsem_read_intr_acquire(&cxl_region_rwsem);

	if (!region_lock)

		return -EINTR;

	ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);

	if ((ret = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))

		return ret;



	struct rw_semaphore *dpa_lock __free(rwsem_read_release) =

		rwsem_read_intr_acquire(&cxl_dpa_rwsem);

	if (!dpa_lock)

		return -EINTR;

	ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);

	if ((ret = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))

		return ret;



	if (!cxl_ppr_ctx->media_accessible || !cxl_ppr_ctx->data_retained) {

		/* Memory to repair must be offline */

drivers/cxl/core/hdm.c

+62 −56
Original line number Diff line number Diff line
@@ -16,7 +16,10 @@ 
 * for enumerating these registers and capabilities.

 */



DECLARE_RWSEM(cxl_dpa_rwsem);

struct cxl_rwsem cxl_rwsem = {

	.region = __RWSEM_INITIALIZER(cxl_rwsem.region),

	.dpa = __RWSEM_INITIALIZER(cxl_rwsem.dpa),

};



static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,

			   int *target_map)
@@ -214,7 +217,7 @@ void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds) 
{

	struct resource *p1, *p2;



	guard(rwsem_read)(&cxl_dpa_rwsem);

	guard(rwsem_read)(&cxl_rwsem.dpa);

	for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {

		__cxl_dpa_debug(file, p1, 0);

		for (p2 = p1->child; p2; p2 = p2->sibling)
@@ -266,7 +269,7 @@ static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 
	struct resource *res = cxled->dpa_res;

	resource_size_t skip_start;



	lockdep_assert_held_write(&cxl_dpa_rwsem);

	lockdep_assert_held_write(&cxl_rwsem.dpa);



	/* save @skip_start, before @res is released */

	skip_start = res->start - cxled->skip;
@@ -281,7 +284,7 @@ static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 


static void cxl_dpa_release(void *cxled)

{

	guard(rwsem_write)(&cxl_dpa_rwsem);

	guard(rwsem_write)(&cxl_rwsem.dpa);

	__cxl_dpa_release(cxled);

}
@@ -293,7 +296,7 @@ static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 
{

	struct cxl_port *port = cxled_to_port(cxled);



	lockdep_assert_held_write(&cxl_dpa_rwsem);

	lockdep_assert_held_write(&cxl_rwsem.dpa);

	devm_remove_action(&port->dev, cxl_dpa_release, cxled);

	__cxl_dpa_release(cxled);

}
@@ -361,7 +364,7 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 
	struct resource *res;

	int rc;



	lockdep_assert_held_write(&cxl_dpa_rwsem);

	lockdep_assert_held_write(&cxl_rwsem.dpa);



	if (!len) {

		dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
@@ -470,7 +473,7 @@ int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info) 
{

	struct device *dev = cxlds->dev;



	guard(rwsem_write)(&cxl_dpa_rwsem);

	guard(rwsem_write)(&cxl_rwsem.dpa);



	if (cxlds->nr_partitions)

		return -EBUSY;
@@ -516,9 +519,8 @@ int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 
	struct cxl_port *port = cxled_to_port(cxled);

	int rc;



	down_write(&cxl_dpa_rwsem);

	scoped_guard(rwsem_write, &cxl_rwsem.dpa)

		rc = __cxl_dpa_reserve(cxled, base, len, skipped);

	up_write(&cxl_dpa_rwsem);



	if (rc)

		return rc;
@@ -529,7 +531,7 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, "CXL"); 


resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)

{

	guard(rwsem_read)(&cxl_dpa_rwsem);

	guard(rwsem_read)(&cxl_rwsem.dpa);

	if (cxled->dpa_res)

		return resource_size(cxled->dpa_res);
@@ -540,7 +542,7 @@ resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled) 
{

	resource_size_t base = -1;



	lockdep_assert_held(&cxl_dpa_rwsem);

	lockdep_assert_held(&cxl_rwsem.dpa);

	if (cxled->dpa_res)

		base = cxled->dpa_res->start;
@@ -559,7 +561,7 @@ int cxl_dpa_free(struct cxl_endpoint_decoder *cxled) 
	struct cxl_port *port = cxled_to_port(cxled);

	struct device *dev = &cxled->cxld.dev;



	guard(rwsem_write)(&cxl_dpa_rwsem);

	guard(rwsem_write)(&cxl_rwsem.dpa);

	if (!cxled->dpa_res)

		return 0;

	if (cxled->cxld.region) {
@@ -589,7 +591,7 @@ int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled, 
	struct device *dev = &cxled->cxld.dev;

	int part;



	guard(rwsem_write)(&cxl_dpa_rwsem);

	guard(rwsem_write)(&cxl_rwsem.dpa);

	if (cxled->cxld.flags & CXL_DECODER_F_ENABLE)

		return -EBUSY;
@@ -621,7 +623,7 @@ static int __cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size) 
	struct resource *p, *last;

	int part;



	guard(rwsem_write)(&cxl_dpa_rwsem);

	guard(rwsem_write)(&cxl_rwsem.dpa);

	if (cxled->cxld.region) {

		dev_dbg(dev, "decoder attached to %s\n",

			dev_name(&cxled->cxld.region->dev));
@@ -771,46 +773,12 @@ static int cxld_await_commit(void __iomem *hdm, int id) 
	return -ETIMEDOUT;

}



static int cxl_decoder_commit(struct cxl_decoder *cxld)

static void setup_hw_decoder(struct cxl_decoder *cxld, void __iomem *hdm)

{

	struct cxl_port *port = to_cxl_port(cxld->dev.parent);

	struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);

	void __iomem *hdm = cxlhdm->regs.hdm_decoder;

	int id = cxld->id, rc;

	int id = cxld->id;

	u64 base, size;

	u32 ctrl;



	if (cxld->flags & CXL_DECODER_F_ENABLE)

		return 0;



	if (cxl_num_decoders_committed(port) != id) {

		dev_dbg(&port->dev,

			"%s: out of order commit, expected decoder%d.%d\n",

			dev_name(&cxld->dev), port->id,

			cxl_num_decoders_committed(port));

		return -EBUSY;

	}



	/*

	 * For endpoint decoders hosted on CXL memory devices that

	 * support the sanitize operation, make sure sanitize is not in-flight.

	 */

	if (is_endpoint_decoder(&cxld->dev)) {

		struct cxl_endpoint_decoder *cxled =

			to_cxl_endpoint_decoder(&cxld->dev);

		struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);

		struct cxl_memdev_state *mds =

			to_cxl_memdev_state(cxlmd->cxlds);



		if (mds && mds->security.sanitize_active) {

			dev_dbg(&cxlmd->dev,

				"attempted to commit %s during sanitize\n",

				dev_name(&cxld->dev));

			return -EBUSY;

		}

	}



	down_read(&cxl_dpa_rwsem);

	/* common decoder settings */

	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));

	cxld_set_interleave(cxld, &ctrl);
@@ -844,7 +812,47 @@ static int cxl_decoder_commit(struct cxl_decoder *cxld) 
	}



	writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));

	up_read(&cxl_dpa_rwsem);

}



static int cxl_decoder_commit(struct cxl_decoder *cxld)

{

	struct cxl_port *port = to_cxl_port(cxld->dev.parent);

	struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);

	void __iomem *hdm = cxlhdm->regs.hdm_decoder;

	int id = cxld->id, rc;



	if (cxld->flags & CXL_DECODER_F_ENABLE)

		return 0;



	if (cxl_num_decoders_committed(port) != id) {

		dev_dbg(&port->dev,

			"%s: out of order commit, expected decoder%d.%d\n",

			dev_name(&cxld->dev), port->id,

			cxl_num_decoders_committed(port));

		return -EBUSY;

	}



	/*

	 * For endpoint decoders hosted on CXL memory devices that

	 * support the sanitize operation, make sure sanitize is not in-flight.

	 */

	if (is_endpoint_decoder(&cxld->dev)) {

		struct cxl_endpoint_decoder *cxled =

			to_cxl_endpoint_decoder(&cxld->dev);

		struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);

		struct cxl_memdev_state *mds =

			to_cxl_memdev_state(cxlmd->cxlds);



		if (mds && mds->security.sanitize_active) {

			dev_dbg(&cxlmd->dev,

				"attempted to commit %s during sanitize\n",

				dev_name(&cxld->dev));

			return -EBUSY;

		}

	}



	scoped_guard(rwsem_read, &cxl_rwsem.dpa)

		setup_hw_decoder(cxld, hdm);



	port->commit_end++;

	rc = cxld_await_commit(hdm, cxld->id);
@@ -882,7 +890,7 @@ void cxl_port_commit_reap(struct cxl_decoder *cxld) 
{

	struct cxl_port *port = to_cxl_port(cxld->dev.parent);



	lockdep_assert_held_write(&cxl_region_rwsem);

	lockdep_assert_held_write(&cxl_rwsem.region);



	/*

	 * Once the highest committed decoder is disabled, free any other
@@ -914,7 +922,6 @@ static void cxl_decoder_reset(struct cxl_decoder *cxld) 
			"%s: out of order reset, expected decoder%d.%d\n",

			dev_name(&cxld->dev), port->id, port->commit_end);



	down_read(&cxl_dpa_rwsem);

	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));

	ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;

	writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
@@ -923,7 +930,6 @@ static void cxl_decoder_reset(struct cxl_decoder *cxld) 
	writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));

	writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));

	writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));

	up_read(&cxl_dpa_rwsem);



	cxld->flags &= ~CXL_DECODER_F_ENABLE;
@@ -1032,7 +1038,7 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 
		else

			cxld->target_type = CXL_DECODER_DEVMEM;



		guard(rwsem_write)(&cxl_region_rwsem);

		guard(rwsem_write)(&cxl_rwsem.region);

		if (cxld->id != cxl_num_decoders_committed(port)) {

			dev_warn(&port->dev,

				 "decoder%d.%d: Committed out of order\n",

drivers/cxl/core/mbox.c

+6 −7
Original line number Diff line number Diff line
@@ -909,8 +909,8 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd, 
		 * translations. Take topology mutation locks and lookup

		 * { HPA, REGION } from { DPA, MEMDEV } in the event record.

		 */

		guard(rwsem_read)(&cxl_region_rwsem);

		guard(rwsem_read)(&cxl_dpa_rwsem);

		guard(rwsem_read)(&cxl_rwsem.region);

		guard(rwsem_read)(&cxl_rwsem.dpa);



		dpa = le64_to_cpu(evt->media_hdr.phys_addr) & CXL_DPA_MASK;

		cxlr = cxl_dpa_to_region(cxlmd, dpa);
@@ -1265,7 +1265,7 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd) 
	/* synchronize with cxl_mem_probe() and decoder write operations */

	guard(device)(&cxlmd->dev);

	endpoint = cxlmd->endpoint;

	guard(rwsem_read)(&cxl_region_rwsem);

	guard(rwsem_read)(&cxl_rwsem.region);

	/*

	 * Require an endpoint to be safe otherwise the driver can not

	 * be sure that the device is unmapped.
@@ -1401,8 +1401,8 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, 
	int nr_records = 0;

	int rc;



	rc = mutex_lock_interruptible(&mds->poison.lock);

	if (rc)

	ACQUIRE(mutex_intr, lock)(&mds->poison.mutex);

	if ((rc = ACQUIRE_ERR(mutex_intr, &lock)))

		return rc;



	po = mds->poison.list_out;
@@ -1437,7 +1437,6 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, 
		}

	} while (po->flags & CXL_POISON_FLAG_MORE);



	mutex_unlock(&mds->poison.lock);

	return rc;

}

EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, "CXL");
@@ -1473,7 +1472,7 @@ int cxl_poison_state_init(struct cxl_memdev_state *mds) 
		return rc;

	}



	mutex_init(&mds->poison.lock);

	mutex_init(&mds->poison.mutex);

	return 0;

}

EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, "CXL");