Commit 4c1f3a7d authored by Mike Christie's avatar Mike Christie Committed by Michael S. Tsirkin
Browse files

vhost-scsi: Reduce mem use by moving upages to per queue 



Each worker thread can process 1 command at a time so there's no need to
allocate a upages array per cmd. This patch moves it to per queue. Even a
small device with 128 cmds and 1 queue this brings mem use for the array
from

2 MB = 8 bytes per page pointer * 2048 pointers * 128 cmds

to

16K = 8 bytes per pointer * 2048 * 1 queue

Signed-off-by: default avatarMike Christie <michael.christie@oracle.com>
Message-Id: <20241203191705.19431-2-michael.christie@oracle.com>
Signed-off-by: default avatarMichael S. Tsirkin <mst@redhat.com>
Acked-by: default avatarStefan Hajnoczi <stefanha@redhat.com>
parent fc80842a

drivers/vhost/scsi.c

+10 −14
Original line number Diff line number Diff line
@@ -83,7 +83,6 @@ struct vhost_scsi_cmd { 
	/* Pointer to the SGL formatted memory from virtio-scsi */

	struct scatterlist *tvc_sgl;

	struct scatterlist *tvc_prot_sgl;

	struct page **tvc_upages;

	/* Pointer to response header iovec */

	struct iovec *tvc_resp_iov;

	/* Pointer to vhost_scsi for our device */
@@ -187,6 +186,7 @@ struct vhost_scsi_virtqueue { 
	struct vhost_scsi_cmd *scsi_cmds;

	struct sbitmap scsi_tags;

	int max_cmds;

	struct page **upages;



	struct vhost_work completion_work;

	struct llist_head completion_list;
@@ -619,7 +619,6 @@ vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg, 
	struct vhost_scsi_nexus *tv_nexus;

	struct scatterlist *sg, *prot_sg;

	struct iovec *tvc_resp_iov;

	struct page **pages;

	int tag;



	tv_nexus = tpg->tpg_nexus;
@@ -637,12 +636,10 @@ vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg, 
	cmd = &svq->scsi_cmds[tag];

	sg = cmd->tvc_sgl;

	prot_sg = cmd->tvc_prot_sgl;

	pages = cmd->tvc_upages;

	tvc_resp_iov = cmd->tvc_resp_iov;

	memset(cmd, 0, sizeof(*cmd));

	cmd->tvc_sgl = sg;

	cmd->tvc_prot_sgl = prot_sg;

	cmd->tvc_upages = pages;

	cmd->tvc_se_cmd.map_tag = tag;

	cmd->tvc_tag = scsi_tag;

	cmd->tvc_lun = lun;
@@ -669,7 +666,9 @@ vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd, 
		      struct scatterlist *sgl,

		      bool is_prot)

{

	struct page **pages = cmd->tvc_upages;

	struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,

					struct vhost_scsi_virtqueue, vq);

	struct page **pages = svq->upages;

	struct scatterlist *sg = sgl;

	ssize_t bytes, mapped_bytes;

	size_t offset, mapped_offset;
@@ -1598,11 +1597,11 @@ static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq) 


		kfree(tv_cmd->tvc_sgl);

		kfree(tv_cmd->tvc_prot_sgl);

		kfree(tv_cmd->tvc_upages);

		kfree(tv_cmd->tvc_resp_iov);

	}



	sbitmap_free(&svq->scsi_tags);

	kfree(svq->upages);

	kfree(svq->scsi_cmds);

	svq->scsi_cmds = NULL;

}
@@ -1628,6 +1627,11 @@ static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds) 
		return -ENOMEM;

	}



	svq->upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES, sizeof(struct page *),

			      GFP_KERNEL);

	if (!svq->upages)

		goto out;



	for (i = 0; i < max_cmds; i++) {

		tv_cmd = &svq->scsi_cmds[i];
@@ -1639,14 +1643,6 @@ static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds) 
			goto out;

		}



		tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES,

					     sizeof(struct page *),

					     GFP_KERNEL);

		if (!tv_cmd->tvc_upages) {

			pr_err("Unable to allocate tv_cmd->tvc_upages\n");

			goto out;

		}



		tv_cmd->tvc_resp_iov = kcalloc(UIO_MAXIOV,

					       sizeof(struct iovec),

					       GFP_KERNEL);