Commit 63a5c7a4 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Keith Busch
Browse files

nvme-pci: use dma_alloc_noncontigous if possible 



Use dma_alloc_noncontigous to allocate a single IOVA-contigous segment
when backed by an IOMMU.  This allow to easily use bigger segments and
avoids running into segment limits if we can avoid it.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarKeith Busch <kbusch@kernel.org>
parent 3c2fb1ca

drivers/nvme/host/pci.c

+53 −5
Original line number Diff line number Diff line
@@ -141,6 +141,7 @@ struct nvme_dev { 
	struct nvme_ctrl ctrl;

	u32 last_ps;

	bool hmb;

	struct sg_table *hmb_sgt;



	mempool_t *iod_mempool;
@@ -1952,7 +1953,7 @@ static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits) 
	return ret;

}



static void nvme_free_host_mem(struct nvme_dev *dev)

static void nvme_free_host_mem_multi(struct nvme_dev *dev)

{

	int i;
@@ -1967,6 +1968,16 @@ static void nvme_free_host_mem(struct nvme_dev *dev) 


	kfree(dev->host_mem_desc_bufs);

	dev->host_mem_desc_bufs = NULL;

}



static void nvme_free_host_mem(struct nvme_dev *dev)

{

	if (dev->hmb_sgt)

		dma_free_noncontiguous(dev->dev, dev->host_mem_size,

				dev->hmb_sgt, DMA_BIDIRECTIONAL);

	else

		nvme_free_host_mem_multi(dev);



	dma_free_coherent(dev->dev, dev->host_mem_descs_size,

			dev->host_mem_descs, dev->host_mem_descs_dma);

	dev->host_mem_descs = NULL;
@@ -1974,7 +1985,33 @@ static void nvme_free_host_mem(struct nvme_dev *dev) 
	dev->nr_host_mem_descs = 0;

}



static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,

static int nvme_alloc_host_mem_single(struct nvme_dev *dev, u64 size)

{

	dev->hmb_sgt = dma_alloc_noncontiguous(dev->dev, size,

				DMA_BIDIRECTIONAL, GFP_KERNEL, 0);

	if (!dev->hmb_sgt)

		return -ENOMEM;



	dev->host_mem_descs = dma_alloc_coherent(dev->dev,

			sizeof(*dev->host_mem_descs), &dev->host_mem_descs_dma,

			GFP_KERNEL);

	if (!dev->host_mem_descs) {

		dma_free_noncontiguous(dev->dev, dev->host_mem_size,

				dev->hmb_sgt, DMA_BIDIRECTIONAL);

		dev->hmb_sgt = NULL;

		return -ENOMEM;

	}

	dev->host_mem_size = size;

	dev->host_mem_descs_size = sizeof(*dev->host_mem_descs);

	dev->nr_host_mem_descs = 1;



	dev->host_mem_descs[0].addr =

		cpu_to_le64(dev->hmb_sgt->sgl->dma_address);

	dev->host_mem_descs[0].size = cpu_to_le32(size / NVME_CTRL_PAGE_SIZE);

	return 0;

}



static int nvme_alloc_host_mem_multi(struct nvme_dev *dev, u64 preferred,

		u32 chunk_size)

{

	struct nvme_host_mem_buf_desc *descs;
@@ -2049,9 +2086,18 @@ static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred) 
	u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);

	u64 chunk_size;



	/*

	 * If there is an IOMMU that can merge pages, try a virtually

	 * non-contiguous allocation for a single segment first.

	 */

	if (!(PAGE_SIZE & dma_get_merge_boundary(dev->dev))) {

		if (!nvme_alloc_host_mem_single(dev, preferred))

			return 0;

	}



	/* start big and work our way down */

	for (chunk_size = min_chunk; chunk_size >= hmminds; chunk_size /= 2) {

		if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {

		if (!nvme_alloc_host_mem_multi(dev, preferred, chunk_size)) {

			if (!min || dev->host_mem_size >= min)

				return 0;

			nvme_free_host_mem(dev);
@@ -2099,8 +2145,10 @@ static int nvme_setup_host_mem(struct nvme_dev *dev) 
		}



		dev_info(dev->ctrl.device,

			"allocated %lld MiB host memory buffer.\n",

			dev->host_mem_size >> ilog2(SZ_1M));

			"allocated %lld MiB host memory buffer (%u segment%s).\n",

			dev->host_mem_size >> ilog2(SZ_1M),

			dev->nr_host_mem_descs,

			str_plural(dev->nr_host_mem_descs));

	}



	ret = nvme_set_host_mem(dev, enable_bits);