Commit 979c6342 authored by Keith Busch's avatar Keith Busch
Browse files

nvme-pci: add support for sgl metadata 



Supporting this mode allows creating and merging multi-segment metadata
requests that wouldn't be possible otherwise. It also allows directly
using user space requests that straddle physically discontiguous pages.

Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarKeith Busch <kbusch@kernel.org>
parent 5dd18f09

drivers/nvme/host/nvme.h

+7 −0
Original line number Diff line number Diff line
@@ -1126,6 +1126,13 @@ static inline bool nvme_ctrl_sgl_supported(struct nvme_ctrl *ctrl) 
	return ctrl->sgls & ((1 << 0) | (1 << 1));

}



static inline bool nvme_ctrl_meta_sgl_supported(struct nvme_ctrl *ctrl)

{

	if (ctrl->ops->flags & NVME_F_FABRICS)

		return true;

	return ctrl->sgls & NVME_CTRL_SGLS_MSDS;

}



#ifdef CONFIG_NVME_HOST_AUTH

int __init nvme_init_auth(void);

void __exit nvme_exit_auth(void);

drivers/nvme/host/pci.c

+129 −15
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ 
 */

#define NVME_MAX_KB_SZ	8192

#define NVME_MAX_SEGS	128

#define NVME_MAX_META_SEGS 15

#define NVME_MAX_NR_ALLOCATIONS	5



static int use_threaded_interrupts;
@@ -144,6 +145,7 @@ struct nvme_dev { 
	struct sg_table *hmb_sgt;



	mempool_t *iod_mempool;

	mempool_t *iod_meta_mempool;



	/* shadow doorbell buffer support: */

	__le32 *dbbuf_dbs;
@@ -239,6 +241,8 @@ struct nvme_iod { 
	dma_addr_t first_dma;

	dma_addr_t meta_dma;

	struct sg_table sgt;

	struct sg_table meta_sgt;

	union nvme_descriptor meta_list;

	union nvme_descriptor list[NVME_MAX_NR_ALLOCATIONS];

};
@@ -506,6 +510,14 @@ static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx) 
	spin_unlock(&nvmeq->sq_lock);

}



static inline bool nvme_pci_metadata_use_sgls(struct nvme_dev *dev,

					      struct request *req)

{

	if (!nvme_ctrl_meta_sgl_supported(&dev->ctrl))

		return false;

	return req->nr_integrity_segments > 1;

}



static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req,

				     int nseg)

{
@@ -518,6 +530,8 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req, 
		return false;

	if (!nvmeq->qid)

		return false;

	if (nvme_pci_metadata_use_sgls(dev, req))

		return true;

	if (!sgl_threshold || avg_seg_size < sgl_threshold)

		return false;

	return true;
@@ -780,7 +794,8 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, 
		struct bio_vec bv = req_bvec(req);



		if (!is_pci_p2pdma_page(bv.bv_page)) {

			if ((bv.bv_offset & (NVME_CTRL_PAGE_SIZE - 1)) +

			if (!nvme_pci_metadata_use_sgls(dev, req) &&

			    (bv.bv_offset & (NVME_CTRL_PAGE_SIZE - 1)) +

			     bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2)

				return nvme_setup_prp_simple(dev, req,

							     &cmnd->rw, &bv);
@@ -824,11 +839,69 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, 
	return ret;

}



static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req,

		struct nvme_command *cmnd)

static blk_status_t nvme_pci_setup_meta_sgls(struct nvme_dev *dev,

					     struct request *req)

{

	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);

	struct nvme_rw_command *cmnd = &iod->cmd.rw;

	struct nvme_sgl_desc *sg_list;

	struct scatterlist *sgl, *sg;

	unsigned int entries;

	dma_addr_t sgl_dma;

	int rc, i;



	iod->meta_sgt.sgl = mempool_alloc(dev->iod_meta_mempool, GFP_ATOMIC);

	if (!iod->meta_sgt.sgl)

		return BLK_STS_RESOURCE;



	sg_init_table(iod->meta_sgt.sgl, req->nr_integrity_segments);

	iod->meta_sgt.orig_nents = blk_rq_map_integrity_sg(req,

							   iod->meta_sgt.sgl);

	if (!iod->meta_sgt.orig_nents)

		goto out_free_sg;



	rc = dma_map_sgtable(dev->dev, &iod->meta_sgt, rq_dma_dir(req),

			     DMA_ATTR_NO_WARN);

	if (rc)

		goto out_free_sg;



	sg_list = dma_pool_alloc(dev->prp_small_pool, GFP_ATOMIC, &sgl_dma);

	if (!sg_list)

		goto out_unmap_sg;



	entries = iod->meta_sgt.nents;

	iod->meta_list.sg_list = sg_list;

	iod->meta_dma = sgl_dma;



	cmnd->flags = NVME_CMD_SGL_METASEG;

	cmnd->metadata = cpu_to_le64(sgl_dma);



	sgl = iod->meta_sgt.sgl;

	if (entries == 1) {

		nvme_pci_sgl_set_data(sg_list, sgl);

		return BLK_STS_OK;

	}



	sgl_dma += sizeof(*sg_list);

	nvme_pci_sgl_set_seg(sg_list, sgl_dma, entries);

	for_each_sg(sgl, sg, entries, i)

		nvme_pci_sgl_set_data(&sg_list[i + 1], sg);



	return BLK_STS_OK;



out_unmap_sg:

	dma_unmap_sgtable(dev->dev, &iod->meta_sgt, rq_dma_dir(req), 0);

out_free_sg:

	mempool_free(iod->meta_sgt.sgl, dev->iod_meta_mempool);

	return BLK_STS_RESOURCE;

}



static blk_status_t nvme_pci_setup_meta_mptr(struct nvme_dev *dev,

					     struct request *req)

{

	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);

	struct bio_vec bv = rq_integrity_vec(req);

	struct nvme_command *cmnd = &iod->cmd;



	iod->meta_dma = dma_map_bvec(dev->dev, &bv, rq_dma_dir(req), 0);

	if (dma_mapping_error(dev->dev, iod->meta_dma))
@@ -837,6 +910,13 @@ static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req, 
	return BLK_STS_OK;

}



static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req)

{

	if (nvme_pci_metadata_use_sgls(dev, req))

		return nvme_pci_setup_meta_sgls(dev, req);

	return nvme_pci_setup_meta_mptr(dev, req);

}



static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)

{

	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
@@ -845,6 +925,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req) 
	iod->aborted = false;

	iod->nr_allocations = -1;

	iod->sgt.nents = 0;

	iod->meta_sgt.nents = 0;



	ret = nvme_setup_cmd(req->q->queuedata, req);

	if (ret)
@@ -857,7 +938,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req) 
	}



	if (blk_integrity_rq(req)) {

		ret = nvme_map_metadata(dev, req, &iod->cmd);

		ret = nvme_map_metadata(dev, req);

		if (ret)

			goto out_unmap_data;

	}
@@ -955,18 +1036,32 @@ static void nvme_queue_rqs(struct rq_list *rqlist) 
	*rqlist = requeue_list;

}



static __always_inline void nvme_pci_unmap_rq(struct request *req)

static __always_inline void nvme_unmap_metadata(struct nvme_dev *dev,

						struct request *req)

{

	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;

	struct nvme_dev *dev = nvmeq->dev;



	if (blk_integrity_rq(req)) {

	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);



	if (!iod->meta_sgt.nents) {

		dma_unmap_page(dev->dev, iod->meta_dma,

			       rq_integrity_vec(req).bv_len, rq_dma_dir(req));

			       rq_integrity_vec(req).bv_len,

			       rq_dma_dir(req));

		return;

	}



	dma_pool_free(dev->prp_small_pool, iod->meta_list.sg_list,

		      iod->meta_dma);

	dma_unmap_sgtable(dev->dev, &iod->meta_sgt, rq_dma_dir(req), 0);

	mempool_free(iod->meta_sgt.sgl, dev->iod_meta_mempool);

}



static __always_inline void nvme_pci_unmap_rq(struct request *req)

{

	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;

	struct nvme_dev *dev = nvmeq->dev;



	if (blk_integrity_rq(req))

		nvme_unmap_metadata(dev, req);



	if (blk_rq_nr_phys_segments(req))

		nvme_unmap_data(dev, req);

}
@@ -2761,6 +2856,7 @@ static void nvme_release_prp_pools(struct nvme_dev *dev) 


static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev)

{

	size_t meta_size = sizeof(struct scatterlist) * (NVME_MAX_META_SEGS + 1);

	size_t alloc_size = sizeof(struct scatterlist) * NVME_MAX_SEGS;



	dev->iod_mempool = mempool_create_node(1,
@@ -2769,7 +2865,18 @@ static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev) 
			dev_to_node(dev->dev));

	if (!dev->iod_mempool)

		return -ENOMEM;



	dev->iod_meta_mempool = mempool_create_node(1,

			mempool_kmalloc, mempool_kfree,

			(void *)meta_size, GFP_KERNEL,

			dev_to_node(dev->dev));

	if (!dev->iod_meta_mempool)

		goto free;



	return 0;

free:

	mempool_destroy(dev->iod_mempool);

	return -ENOMEM;

}



static void nvme_free_tagset(struct nvme_dev *dev)
@@ -2834,6 +2941,11 @@ static void nvme_reset_work(struct work_struct *work) 
	if (result)

		goto out;



	if (nvme_ctrl_meta_sgl_supported(&dev->ctrl))

		dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS;

	else

		dev->ctrl.max_integrity_segments = 1;



	nvme_dbbuf_dma_alloc(dev);



	result = nvme_setup_host_mem(dev);
@@ -3101,11 +3213,6 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev, 
	dev->ctrl.max_hw_sectors = min_t(u32,

		NVME_MAX_KB_SZ << 1, dma_opt_mapping_size(&pdev->dev) >> 9);

	dev->ctrl.max_segments = NVME_MAX_SEGS;



	/*

	 * There is no support for SGLs for metadata (yet), so we are limited to

	 * a single integrity segment for the separate metadata pointer.

	 */

	dev->ctrl.max_integrity_segments = 1;

	return dev;
@@ -3168,6 +3275,11 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 
	if (result)

		goto out_disable;



	if (nvme_ctrl_meta_sgl_supported(&dev->ctrl))

		dev->ctrl.max_integrity_segments = NVME_MAX_META_SEGS;

	else

		dev->ctrl.max_integrity_segments = 1;



	nvme_dbbuf_dma_alloc(dev);



	result = nvme_setup_host_mem(dev);
@@ -3210,6 +3322,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 
	nvme_free_queues(dev, 0);

out_release_iod_mempool:

	mempool_destroy(dev->iod_mempool);

	mempool_destroy(dev->iod_meta_mempool);

out_release_prp_pools:

	nvme_release_prp_pools(dev);

out_dev_unmap:
@@ -3275,6 +3388,7 @@ static void nvme_remove(struct pci_dev *pdev) 
	nvme_dbbuf_dma_free(dev);

	nvme_free_queues(dev, 0);

	mempool_destroy(dev->iod_mempool);

	mempool_destroy(dev->iod_meta_mempool);

	nvme_release_prp_pools(dev);

	nvme_dev_unmap(dev);

	nvme_uninit_ctrl(&dev->ctrl);

include/linux/nvme.h

+1 −0
Original line number Diff line number Diff line
@@ -389,6 +389,7 @@ enum { 
	NVME_CTRL_CTRATT_PREDICTABLE_LAT	= 1 << 5,

	NVME_CTRL_CTRATT_NAMESPACE_GRANULARITY	= 1 << 7,

	NVME_CTRL_CTRATT_UUID_LIST		= 1 << 9,

	NVME_CTRL_SGLS_MSDS                     = 1 << 19,

};



struct nvme_lbaf {