Commit 6e5f6a86 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost 

Pull virtio updates from Michael Tsirkin:
 "vhost,virtio and vdpa features, fixes, and cleanups:

   - mac vlan filter and stats support in mlx5 vdpa

   - irq hardening in virtio

   - performance improvements in virtio crypto

   - polling i/o support in virtio blk

   - ASID support in vhost

   - fixes, cleanups all over the place"

* tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (64 commits)
  vdpa: ifcvf: set pci driver data in probe
  vdpa/mlx5: Add RX MAC VLAN filter support
  vdpa/mlx5: Remove flow counter from steering
  vhost: rename vhost_work_dev_flush
  vhost-test: drop flush after vhost_dev_cleanup
  vhost-scsi: drop flush after vhost_dev_cleanup
  vhost_vsock: simplify vhost_vsock_flush()
  vhost_test: remove vhost_test_flush_vq()
  vhost_net: get rid of vhost_net_flush_vq() and extra flush calls
  vhost: flush dev once during vhost_dev_stop
  vhost: get rid of vhost_poll_flush() wrapper
  vhost-vdpa: return -EFAULT on copy_to_user() failure
  vdpasim: Off by one in vdpasim_set_group_asid()
  virtio: Directly use ida_alloc()/free()
  virtio: use WARN_ON() to warning illegal status value
  virtio: harden vring IRQ
  virtio: allow to unbreak virtqueue
  virtio-ccw: implement synchronize_cbs()
  virtio-mmio: implement synchronize_cbs()
  virtio-pci: implement synchronize_cbs()
  ...
parents 6f6ebb98 bd8bb9ae

drivers/block/virtio_blk.c

+201 −19
Original line number Diff line number Diff line
@@ -37,6 +37,10 @@ MODULE_PARM_DESC(num_request_queues, 
		 "0 for no limit. "

		 "Values > nr_cpu_ids truncated to nr_cpu_ids.");



static unsigned int poll_queues;

module_param(poll_queues, uint, 0644);

MODULE_PARM_DESC(poll_queues, "The number of dedicated virtqueues for polling I/O");



static int major;

static DEFINE_IDA(vd_index_ida);
@@ -74,6 +78,7 @@ struct virtio_blk { 


	/* num of vqs */

	int num_vqs;

	int io_queues[HCTX_MAX_TYPES];

	struct virtio_blk_vq *vqs;

};
@@ -96,8 +101,7 @@ static inline blk_status_t virtblk_result(struct virtblk_req *vbr) 
	}

}



static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr,

		struct scatterlist *data_sg, bool have_data)

static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr)

{

	struct scatterlist hdr, status, *sgs[3];

	unsigned int num_out = 0, num_in = 0;
@@ -105,11 +109,11 @@ static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr, 
	sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr));

	sgs[num_out++] = &hdr;



	if (have_data) {

	if (vbr->sg_table.nents) {

		if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT))

			sgs[num_out++] = data_sg;

			sgs[num_out++] = vbr->sg_table.sgl;

		else

			sgs[num_out + num_in++] = data_sg;

			sgs[num_out + num_in++] = vbr->sg_table.sgl;

	}



	sg_init_one(&status, &vbr->status, sizeof(vbr->status));
@@ -299,6 +303,28 @@ static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx) 
		virtqueue_notify(vq->vq);

}



static blk_status_t virtblk_prep_rq(struct blk_mq_hw_ctx *hctx,

					struct virtio_blk *vblk,

					struct request *req,

					struct virtblk_req *vbr)

{

	blk_status_t status;



	status = virtblk_setup_cmd(vblk->vdev, req, vbr);

	if (unlikely(status))

		return status;



	blk_mq_start_request(req);



	vbr->sg_table.nents = virtblk_map_data(hctx, req, vbr);

	if (unlikely(vbr->sg_table.nents < 0)) {

		virtblk_cleanup_cmd(req);

		return BLK_STS_RESOURCE;

	}



	return BLK_STS_OK;

}



static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,

			   const struct blk_mq_queue_data *bd)

{
@@ -306,26 +332,17 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, 
	struct request *req = bd->rq;

	struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);

	unsigned long flags;

	int num;

	int qid = hctx->queue_num;

	bool notify = false;

	blk_status_t status;

	int err;



	status = virtblk_setup_cmd(vblk->vdev, req, vbr);

	status = virtblk_prep_rq(hctx, vblk, req, vbr);

	if (unlikely(status))

		return status;



	blk_mq_start_request(req);



	num = virtblk_map_data(hctx, req, vbr);

	if (unlikely(num < 0)) {

		virtblk_cleanup_cmd(req);

		return BLK_STS_RESOURCE;

	}



	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);

	err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg_table.sgl, num);

	err = virtblk_add_req(vblk->vqs[qid].vq, vbr);

	if (err) {

		virtqueue_kick(vblk->vqs[qid].vq);

		/* Don't stop the queue if -ENOMEM: we may have failed to
@@ -355,6 +372,75 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, 
	return BLK_STS_OK;

}



static bool virtblk_prep_rq_batch(struct request *req)

{

	struct virtio_blk *vblk = req->mq_hctx->queue->queuedata;

	struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);



	req->mq_hctx->tags->rqs[req->tag] = req;



	return virtblk_prep_rq(req->mq_hctx, vblk, req, vbr) == BLK_STS_OK;

}



static bool virtblk_add_req_batch(struct virtio_blk_vq *vq,

					struct request **rqlist,

					struct request **requeue_list)

{

	unsigned long flags;

	int err;

	bool kick;



	spin_lock_irqsave(&vq->lock, flags);



	while (!rq_list_empty(*rqlist)) {

		struct request *req = rq_list_pop(rqlist);

		struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);



		err = virtblk_add_req(vq->vq, vbr);

		if (err) {

			virtblk_unmap_data(req, vbr);

			virtblk_cleanup_cmd(req);

			rq_list_add(requeue_list, req);

		}

	}



	kick = virtqueue_kick_prepare(vq->vq);

	spin_unlock_irqrestore(&vq->lock, flags);



	return kick;

}



static void virtio_queue_rqs(struct request **rqlist)

{

	struct request *req, *next, *prev = NULL;

	struct request *requeue_list = NULL;



	rq_list_for_each_safe(rqlist, req, next) {

		struct virtio_blk_vq *vq = req->mq_hctx->driver_data;

		bool kick;



		if (!virtblk_prep_rq_batch(req)) {

			rq_list_move(rqlist, &requeue_list, req, prev);

			req = prev;

			if (!req)

				continue;

		}



		if (!next || req->mq_hctx != next->mq_hctx) {

			req->rq_next = NULL;

			kick = virtblk_add_req_batch(vq, rqlist, &requeue_list);

			if (kick)

				virtqueue_notify(vq->vq);



			*rqlist = next;

			prev = NULL;

		} else

			prev = req;

	}



	*rqlist = requeue_list;

}



/* return id (s/n) string for *disk to *id_str

 */

static int virtblk_get_id(struct gendisk *disk, char *id_str)
@@ -512,6 +598,7 @@ static int init_vq(struct virtio_blk *vblk) 
	const char **names;

	struct virtqueue **vqs;

	unsigned short num_vqs;

	unsigned int num_poll_vqs;

	struct virtio_device *vdev = vblk->vdev;

	struct irq_affinity desc = { 0, };
@@ -520,6 +607,7 @@ static int init_vq(struct virtio_blk *vblk) 
				   &num_vqs);

	if (err)

		num_vqs = 1;



	if (!err && !num_vqs) {

		dev_err(&vdev->dev, "MQ advertised but zero queues reported\n");

		return -EINVAL;
@@ -529,6 +617,17 @@ static int init_vq(struct virtio_blk *vblk) 
			min_not_zero(num_request_queues, nr_cpu_ids),

			num_vqs);



	num_poll_vqs = min_t(unsigned int, poll_queues, num_vqs - 1);



	vblk->io_queues[HCTX_TYPE_DEFAULT] = num_vqs - num_poll_vqs;

	vblk->io_queues[HCTX_TYPE_READ] = 0;

	vblk->io_queues[HCTX_TYPE_POLL] = num_poll_vqs;



	dev_info(&vdev->dev, "%d/%d/%d default/read/poll queues\n",

				vblk->io_queues[HCTX_TYPE_DEFAULT],

				vblk->io_queues[HCTX_TYPE_READ],

				vblk->io_queues[HCTX_TYPE_POLL]);



	vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);

	if (!vblk->vqs)

		return -ENOMEM;
@@ -541,12 +640,18 @@ static int init_vq(struct virtio_blk *vblk) 
		goto out;

	}



	for (i = 0; i < num_vqs; i++) {

	for (i = 0; i < num_vqs - num_poll_vqs; i++) {

		callbacks[i] = virtblk_done;

		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);

		names[i] = vblk->vqs[i].name;

	}



	for (; i < num_vqs; i++) {

		callbacks[i] = NULL;

		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req_poll.%d", i);

		names[i] = vblk->vqs[i].name;

	}



	/* Discover virtqueues and write information to configuration.  */

	err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc);

	if (err)
@@ -692,16 +797,90 @@ static const struct attribute_group *virtblk_attr_groups[] = { 
static int virtblk_map_queues(struct blk_mq_tag_set *set)

{

	struct virtio_blk *vblk = set->driver_data;

	int i, qoff;



	for (i = 0, qoff = 0; i < set->nr_maps; i++) {

		struct blk_mq_queue_map *map = &set->map[i];



		map->nr_queues = vblk->io_queues[i];

		map->queue_offset = qoff;

		qoff += map->nr_queues;



		if (map->nr_queues == 0)

			continue;



		/*

		 * Regular queues have interrupts and hence CPU affinity is

		 * defined by the core virtio code, but polling queues have

		 * no interrupts so we let the block layer assign CPU affinity.

		 */

		if (i == HCTX_TYPE_POLL)

			blk_mq_map_queues(&set->map[i]);

		else

			blk_mq_virtio_map_queues(&set->map[i], vblk->vdev, 0);

	}



	return 0;

}



	return blk_mq_virtio_map_queues(&set->map[HCTX_TYPE_DEFAULT],

					vblk->vdev, 0);

static void virtblk_complete_batch(struct io_comp_batch *iob)

{

	struct request *req;



	rq_list_for_each(&iob->req_list, req) {

		virtblk_unmap_data(req, blk_mq_rq_to_pdu(req));

		virtblk_cleanup_cmd(req);

	}

	blk_mq_end_request_batch(iob);

}



static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)

{

	struct virtio_blk *vblk = hctx->queue->queuedata;

	struct virtio_blk_vq *vq = hctx->driver_data;

	struct virtblk_req *vbr;

	unsigned long flags;

	unsigned int len;

	int found = 0;



	spin_lock_irqsave(&vq->lock, flags);



	while ((vbr = virtqueue_get_buf(vq->vq, &len)) != NULL) {

		struct request *req = blk_mq_rq_from_pdu(vbr);



		found++;

		if (!blk_mq_add_to_batch(req, iob, vbr->status,

						virtblk_complete_batch))

			blk_mq_complete_request(req);

	}



	if (found)

		blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);



	spin_unlock_irqrestore(&vq->lock, flags);



	return found;

}



static int virtblk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,

			  unsigned int hctx_idx)

{

	struct virtio_blk *vblk = data;

	struct virtio_blk_vq *vq = &vblk->vqs[hctx_idx];



	WARN_ON(vblk->tag_set.tags[hctx_idx] != hctx->tags);

	hctx->driver_data = vq;

	return 0;

}



static const struct blk_mq_ops virtio_mq_ops = {

	.queue_rq	= virtio_queue_rq,

	.queue_rqs	= virtio_queue_rqs,

	.commit_rqs	= virtio_commit_rqs,

	.init_hctx	= virtblk_init_hctx,

	.complete	= virtblk_request_done,

	.map_queues	= virtblk_map_queues,

	.poll		= virtblk_poll,

};



static unsigned int virtblk_queue_depth;
@@ -778,6 +957,9 @@ static int virtblk_probe(struct virtio_device *vdev) 
		sizeof(struct scatterlist) * VIRTIO_BLK_INLINE_SG_CNT;

	vblk->tag_set.driver_data = vblk;

	vblk->tag_set.nr_hw_queues = vblk->num_vqs;

	vblk->tag_set.nr_maps = 1;

	if (vblk->io_queues[HCTX_TYPE_POLL])

		vblk->tag_set.nr_maps = 3;



	err = blk_mq_alloc_tag_set(&vblk->tag_set);

	if (err)

drivers/crypto/virtio/virtio_crypto_akcipher_algs.c

+50 −45
Original line number Diff line number Diff line
@@ -90,9 +90,12 @@ static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request * 
	}



	akcipher_req = vc_akcipher_req->akcipher_req;

	if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY)

	if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY) {

		/* actuall length maybe less than dst buffer */

		akcipher_req->dst_len = len - sizeof(vc_req->status);

		sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst),

				    vc_akcipher_req->dst_buf, akcipher_req->dst_len);

	}

	virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error);

}
@@ -103,54 +106,56 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher 
	struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3];

	struct virtio_crypto *vcrypto = ctx->vcrypto;

	uint8_t *pkey;

	unsigned int inlen;

	int err;

	unsigned int num_out = 0, num_in = 0;

	struct virtio_crypto_op_ctrl_req *ctrl;

	struct virtio_crypto_session_input *input;

	struct virtio_crypto_ctrl_request *vc_ctrl_req;



	pkey = kmemdup(key, keylen, GFP_ATOMIC);

	if (!pkey)

		return -ENOMEM;



	spin_lock(&vcrypto->ctrl_lock);

	memcpy(&vcrypto->ctrl.header, header, sizeof(vcrypto->ctrl.header));

	memcpy(&vcrypto->ctrl.u, para, sizeof(vcrypto->ctrl.u));

	vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR);

	vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL);

	if (!vc_ctrl_req) {

		err = -ENOMEM;

		goto out;

	}



	ctrl = &vc_ctrl_req->ctrl;

	memcpy(&ctrl->header, header, sizeof(ctrl->header));

	memcpy(&ctrl->u, para, sizeof(ctrl->u));

	input = &vc_ctrl_req->input;

	input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR);



	sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl));

	sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl));

	sgs[num_out++] = &outhdr_sg;



	sg_init_one(&key_sg, pkey, keylen);

	sgs[num_out++] = &key_sg;



	sg_init_one(&inhdr_sg, &vcrypto->input, sizeof(vcrypto->input));

	sg_init_one(&inhdr_sg, input, sizeof(*input));

	sgs[num_out + num_in++] = &inhdr_sg;



	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC);

	err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req);

	if (err < 0)

		goto out;



	virtqueue_kick(vcrypto->ctrl_vq);

	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) &&

	       !virtqueue_is_broken(vcrypto->ctrl_vq))

		cpu_relax();



	if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) {

	if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) {

		pr_err("virtio_crypto: Create session failed status: %u\n",

			le32_to_cpu(input->status));

		err = -EINVAL;

		goto out;

	}



	ctx->session_id = le64_to_cpu(vcrypto->input.session_id);

	ctx->session_id = le64_to_cpu(input->session_id);

	ctx->session_valid = true;

	err = 0;



out:

	spin_unlock(&vcrypto->ctrl_lock);

	kfree(vc_ctrl_req);

	kfree_sensitive(pkey);



	if (err < 0)

		pr_err("virtio_crypto: Create session failed status: %u\n",

			le32_to_cpu(vcrypto->input.status));



	return err;

}
@@ -159,37 +164,41 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe 
	struct scatterlist outhdr_sg, inhdr_sg, *sgs[2];

	struct virtio_crypto_destroy_session_req *destroy_session;

	struct virtio_crypto *vcrypto = ctx->vcrypto;

	unsigned int num_out = 0, num_in = 0, inlen;

	unsigned int num_out = 0, num_in = 0;

	int err;

	struct virtio_crypto_op_ctrl_req *ctrl;

	struct virtio_crypto_inhdr *ctrl_status;

	struct virtio_crypto_ctrl_request *vc_ctrl_req;



	spin_lock(&vcrypto->ctrl_lock);

	if (!ctx->session_valid) {

		err = 0;

		goto out;

	}

	vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR;

	vcrypto->ctrl.header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION);

	vcrypto->ctrl.header.queue_id = 0;

	if (!ctx->session_valid)

		return 0;



	vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL);

	if (!vc_ctrl_req)

		return -ENOMEM;



	destroy_session = &vcrypto->ctrl.u.destroy_session;

	ctrl_status = &vc_ctrl_req->ctrl_status;

	ctrl_status->status = VIRTIO_CRYPTO_ERR;

	ctrl = &vc_ctrl_req->ctrl;

	ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION);

	ctrl->header.queue_id = 0;



	destroy_session = &ctrl->u.destroy_session;

	destroy_session->session_id = cpu_to_le64(ctx->session_id);



	sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl));

	sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl));

	sgs[num_out++] = &outhdr_sg;



	sg_init_one(&inhdr_sg, &vcrypto->ctrl_status.status, sizeof(vcrypto->ctrl_status.status));

	sg_init_one(&inhdr_sg, &ctrl_status->status, sizeof(ctrl_status->status));

	sgs[num_out + num_in++] = &inhdr_sg;



	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC);

	err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req);

	if (err < 0)

		goto out;



	virtqueue_kick(vcrypto->ctrl_vq);

	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) &&

	       !virtqueue_is_broken(vcrypto->ctrl_vq))

		cpu_relax();



	if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) {

	if (ctrl_status->status != VIRTIO_CRYPTO_OK) {

		pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",

			ctrl_status->status, destroy_session->session_id);

		err = -EINVAL;

		goto out;

	}
@@ -198,11 +207,7 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe 
	ctx->session_valid = false;



out:

	spin_unlock(&vcrypto->ctrl_lock);

	if (err < 0) {

		pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",

			vcrypto->ctrl_status.status, destroy_session->session_id);

	}

	kfree(vc_ctrl_req);



	return err;

}

drivers/crypto/virtio/virtio_crypto_common.h

+16 −5
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include <crypto/aead.h>

#include <crypto/aes.h>

#include <crypto/engine.h>

#include <uapi/linux/virtio_crypto.h>





/* Internal representation of a data virtqueue */
@@ -65,11 +66,6 @@ struct virtio_crypto { 
	/* Maximum size of per request */

	u64 max_size;



	/* Control VQ buffers: protected by the ctrl_lock */

	struct virtio_crypto_op_ctrl_req ctrl;

	struct virtio_crypto_session_input input;

	struct virtio_crypto_inhdr ctrl_status;



	unsigned long status;

	atomic_t ref_count;

	struct list_head list;
@@ -85,6 +81,18 @@ struct virtio_crypto_sym_session_info { 
	__u64 session_id;

};



/*

 * Note: there are padding fields in request, clear them to zero before

 *       sending to host to avoid to divulge any information.

 * Ex, virtio_crypto_ctrl_request::ctrl::u::destroy_session::padding[48]

 */

struct virtio_crypto_ctrl_request {

	struct virtio_crypto_op_ctrl_req ctrl;

	struct virtio_crypto_session_input input;

	struct virtio_crypto_inhdr ctrl_status;

	struct completion compl;

};



struct virtio_crypto_request;

typedef void (*virtio_crypto_data_callback)

		(struct virtio_crypto_request *vc_req, int len);
@@ -134,5 +142,8 @@ int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto); 
void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto);

int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto);

void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto);

int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterlist *sgs[],

				  unsigned int out_sgs, unsigned int in_sgs,

				  struct virtio_crypto_ctrl_request *vc_ctrl_req);



#endif /* _VIRTIO_CRYPTO_COMMON_H */

drivers/crypto/virtio/virtio_crypto_core.c

+53 −2
Original line number Diff line number Diff line
@@ -22,6 +22,56 @@ virtcrypto_clear_request(struct virtio_crypto_request *vc_req) 
	}

}



static void virtio_crypto_ctrlq_callback(struct virtio_crypto_ctrl_request *vc_ctrl_req)

{

	complete(&vc_ctrl_req->compl);

}



static void virtcrypto_ctrlq_callback(struct virtqueue *vq)

{

	struct virtio_crypto *vcrypto = vq->vdev->priv;

	struct virtio_crypto_ctrl_request *vc_ctrl_req;

	unsigned long flags;

	unsigned int len;



	spin_lock_irqsave(&vcrypto->ctrl_lock, flags);

	do {

		virtqueue_disable_cb(vq);

		while ((vc_ctrl_req = virtqueue_get_buf(vq, &len)) != NULL) {

			spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);

			virtio_crypto_ctrlq_callback(vc_ctrl_req);

			spin_lock_irqsave(&vcrypto->ctrl_lock, flags);

		}

		if (unlikely(virtqueue_is_broken(vq)))

			break;

	} while (!virtqueue_enable_cb(vq));

	spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);

}



int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterlist *sgs[],

		unsigned int out_sgs, unsigned int in_sgs,

		struct virtio_crypto_ctrl_request *vc_ctrl_req)

{

	int err;

	unsigned long flags;



	init_completion(&vc_ctrl_req->compl);



	spin_lock_irqsave(&vcrypto->ctrl_lock, flags);

	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, out_sgs, in_sgs, vc_ctrl_req, GFP_ATOMIC);

	if (err < 0) {

		spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);

		return err;

	}



	virtqueue_kick(vcrypto->ctrl_vq);

	spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);



	wait_for_completion(&vc_ctrl_req->compl);



	return 0;

}



static void virtcrypto_dataq_callback(struct virtqueue *vq)

{

	struct virtio_crypto *vcrypto = vq->vdev->priv;
@@ -73,7 +123,7 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi) 
		goto err_names;



	/* Parameters for control virtqueue */

	callbacks[total_vqs - 1] = NULL;

	callbacks[total_vqs - 1] = virtcrypto_ctrlq_callback;

	names[total_vqs - 1] = "controlq";



	/* Allocate/initialize parameters for data virtqueues */
@@ -94,7 +144,8 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi) 
		spin_lock_init(&vi->data_vq[i].lock);

		vi->data_vq[i].vq = vqs[i];

		/* Initialize crypto engine */

		vi->data_vq[i].engine = crypto_engine_alloc_init(dev, 1);

		vi->data_vq[i].engine = crypto_engine_alloc_init_and_set(dev, true, NULL, true,

						virtqueue_get_vring_size(vqs[i]));

		if (!vi->data_vq[i].engine) {

			ret = -ENOMEM;

			goto err_engine;

drivers/crypto/virtio/virtio_crypto_skcipher_algs.c

+63 −77
Original line number Diff line number Diff line
@@ -118,11 +118,14 @@ static int virtio_crypto_alg_skcipher_init_session( 
		int encrypt)

{

	struct scatterlist outhdr, key_sg, inhdr, *sgs[3];

	unsigned int tmp;

	struct virtio_crypto *vcrypto = ctx->vcrypto;

	int op = encrypt ? VIRTIO_CRYPTO_OP_ENCRYPT : VIRTIO_CRYPTO_OP_DECRYPT;

	int err;

	unsigned int num_out = 0, num_in = 0;

	struct virtio_crypto_op_ctrl_req *ctrl;

	struct virtio_crypto_session_input *input;

	struct virtio_crypto_sym_create_session_req *sym_create_session;

	struct virtio_crypto_ctrl_request *vc_ctrl_req;



	/*

	 * Avoid to do DMA from the stack, switch to using
@@ -133,26 +136,29 @@ static int virtio_crypto_alg_skcipher_init_session( 
	if (!cipher_key)

		return -ENOMEM;



	spin_lock(&vcrypto->ctrl_lock);

	vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL);

	if (!vc_ctrl_req) {

		err = -ENOMEM;

		goto out;

	}



	/* Pad ctrl header */

	vcrypto->ctrl.header.opcode =

		cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION);

	vcrypto->ctrl.header.algo = cpu_to_le32(alg);

	ctrl = &vc_ctrl_req->ctrl;

	ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION);

	ctrl->header.algo = cpu_to_le32(alg);

	/* Set the default dataqueue id to 0 */

	vcrypto->ctrl.header.queue_id = 0;

	ctrl->header.queue_id = 0;



	vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR);

	input = &vc_ctrl_req->input;

	input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR);

	/* Pad cipher's parameters */

	vcrypto->ctrl.u.sym_create_session.op_type =

		cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);

	vcrypto->ctrl.u.sym_create_session.u.cipher.para.algo =

		vcrypto->ctrl.header.algo;

	vcrypto->ctrl.u.sym_create_session.u.cipher.para.keylen =

		cpu_to_le32(keylen);

	vcrypto->ctrl.u.sym_create_session.u.cipher.para.op =

		cpu_to_le32(op);



	sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl));

	sym_create_session = &ctrl->u.sym_create_session;

	sym_create_session->op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);

	sym_create_session->u.cipher.para.algo = ctrl->header.algo;

	sym_create_session->u.cipher.para.keylen = cpu_to_le32(keylen);

	sym_create_session->u.cipher.para.op = cpu_to_le32(op);



	sg_init_one(&outhdr, ctrl, sizeof(*ctrl));

	sgs[num_out++] = &outhdr;



	/* Set key */
@@ -160,45 +166,30 @@ static int virtio_crypto_alg_skcipher_init_session( 
	sgs[num_out++] = &key_sg;



	/* Return status and session id back */

	sg_init_one(&inhdr, &vcrypto->input, sizeof(vcrypto->input));

	sg_init_one(&inhdr, input, sizeof(*input));

	sgs[num_out + num_in++] = &inhdr;



	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out,

				num_in, vcrypto, GFP_ATOMIC);

	if (err < 0) {

		spin_unlock(&vcrypto->ctrl_lock);

		kfree_sensitive(cipher_key);

		return err;

	}

	virtqueue_kick(vcrypto->ctrl_vq);

	err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req);

	if (err < 0)

		goto out;



	/*

	 * Trapping into the hypervisor, so the request should be

	 * handled immediately.

	 */

	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) &&

	       !virtqueue_is_broken(vcrypto->ctrl_vq))

		cpu_relax();



	if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) {

		spin_unlock(&vcrypto->ctrl_lock);

	if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) {

		pr_err("virtio_crypto: Create session failed status: %u\n",

			le32_to_cpu(vcrypto->input.status));

		kfree_sensitive(cipher_key);

		return -EINVAL;

			le32_to_cpu(input->status));

		err = -EINVAL;

		goto out;

	}



	if (encrypt)

		ctx->enc_sess_info.session_id =

			le64_to_cpu(vcrypto->input.session_id);

		ctx->enc_sess_info.session_id = le64_to_cpu(input->session_id);

	else

		ctx->dec_sess_info.session_id =

			le64_to_cpu(vcrypto->input.session_id);



	spin_unlock(&vcrypto->ctrl_lock);

		ctx->dec_sess_info.session_id = le64_to_cpu(input->session_id);



	err = 0;

out:

	kfree(vc_ctrl_req);

	kfree_sensitive(cipher_key);

	return 0;

	return err;

}



static int virtio_crypto_alg_skcipher_close_session(
@@ -206,60 +197,55 @@ static int virtio_crypto_alg_skcipher_close_session( 
		int encrypt)

{

	struct scatterlist outhdr, status_sg, *sgs[2];

	unsigned int tmp;

	struct virtio_crypto_destroy_session_req *destroy_session;

	struct virtio_crypto *vcrypto = ctx->vcrypto;

	int err;

	unsigned int num_out = 0, num_in = 0;

	struct virtio_crypto_op_ctrl_req *ctrl;

	struct virtio_crypto_inhdr *ctrl_status;

	struct virtio_crypto_ctrl_request *vc_ctrl_req;



	spin_lock(&vcrypto->ctrl_lock);

	vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR;

	vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL);

	if (!vc_ctrl_req)

		return -ENOMEM;



	ctrl_status = &vc_ctrl_req->ctrl_status;

	ctrl_status->status = VIRTIO_CRYPTO_ERR;

	/* Pad ctrl header */

	vcrypto->ctrl.header.opcode =

		cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION);

	ctrl = &vc_ctrl_req->ctrl;

	ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION);

	/* Set the default virtqueue id to 0 */

	vcrypto->ctrl.header.queue_id = 0;

	ctrl->header.queue_id = 0;



	destroy_session = &vcrypto->ctrl.u.destroy_session;

	destroy_session = &ctrl->u.destroy_session;



	if (encrypt)

		destroy_session->session_id =

			cpu_to_le64(ctx->enc_sess_info.session_id);

		destroy_session->session_id = cpu_to_le64(ctx->enc_sess_info.session_id);

	else

		destroy_session->session_id =

			cpu_to_le64(ctx->dec_sess_info.session_id);

		destroy_session->session_id = cpu_to_le64(ctx->dec_sess_info.session_id);



	sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl));

	sg_init_one(&outhdr, ctrl, sizeof(*ctrl));

	sgs[num_out++] = &outhdr;



	/* Return status and session id back */

	sg_init_one(&status_sg, &vcrypto->ctrl_status.status,

		sizeof(vcrypto->ctrl_status.status));

	sg_init_one(&status_sg, &ctrl_status->status, sizeof(ctrl_status->status));

	sgs[num_out + num_in++] = &status_sg;



	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out,

			num_in, vcrypto, GFP_ATOMIC);

	if (err < 0) {

		spin_unlock(&vcrypto->ctrl_lock);

		return err;

	}

	virtqueue_kick(vcrypto->ctrl_vq);



	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) &&

	       !virtqueue_is_broken(vcrypto->ctrl_vq))

		cpu_relax();

	err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req);

	if (err < 0)

		goto out;



	if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) {

		spin_unlock(&vcrypto->ctrl_lock);

	if (ctrl_status->status != VIRTIO_CRYPTO_OK) {

		pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",

			vcrypto->ctrl_status.status,

			destroy_session->session_id);

			ctrl_status->status, destroy_session->session_id);



		return -EINVAL;

	}

	spin_unlock(&vcrypto->ctrl_lock);



	return 0;

	err = 0;

out:

	kfree(vc_ctrl_req);

	return err;

}



static int virtio_crypto_alg_skcipher_init_sessions(