Commit 97bc3633 authored by Shirley Ma's avatar Shirley Ma Committed by David S. Miller
Browse files

macvtap: macvtapTX zero-copy support 



Only 128 bytes is copied, the rest of data is DMA mapped directly from
userspace.

Signed-off-by: default avatarShirley Ma <xma@...ibm.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent a6686f2f

drivers/net/macvtap.c

+121 −11
Original line number Diff line number Diff line
@@ -60,6 +60,7 @@ static struct proto macvtap_proto = { 
 */

static dev_t macvtap_major;

#define MACVTAP_NUM_DEVS 65536

#define GOODCOPY_LEN 128

static struct class *macvtap_class;

static struct cdev macvtap_cdev;
@@ -340,6 +341,7 @@ static int macvtap_open(struct inode *inode, struct file *file) 
{

	struct net *net = current->nsproxy->net_ns;

	struct net_device *dev = dev_get_by_index(net, iminor(inode));

	struct macvlan_dev *vlan = netdev_priv(dev);

	struct macvtap_queue *q;

	int err;
@@ -369,6 +371,16 @@ static int macvtap_open(struct inode *inode, struct file *file) 
	q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP;

	q->vnet_hdr_sz = sizeof(struct virtio_net_hdr);



	/*

	 * so far only KVM virtio_net uses macvtap, enable zero copy between

	 * guest kernel and host kernel when lower device supports zerocopy

	 */

	if (vlan) {

		if ((vlan->lowerdev->features & NETIF_F_HIGHDMA) &&

		    (vlan->lowerdev->features & NETIF_F_SG))

			sock_set_flag(&q->sk, SOCK_ZEROCOPY);

	}



	err = macvtap_set_queue(dev, file, q);

	if (err)

		sock_put(&q->sk);
@@ -433,6 +445,80 @@ static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad, 
	return skb;

}



/* set skb frags from iovec, this can move to core network code for reuse */

static int zerocopy_sg_from_iovec(struct sk_buff *skb, const struct iovec *from,

				  int offset, size_t count)

{

	int len = iov_length(from, count) - offset;

	int copy = skb_headlen(skb);

	int size, offset1 = 0;

	int i = 0;

	skb_frag_t *f;



	/* Skip over from offset */

	while (count && (offset >= from->iov_len)) {

		offset -= from->iov_len;

		++from;

		--count;

	}



	/* copy up to skb headlen */

	while (count && (copy > 0)) {

		size = min_t(unsigned int, copy, from->iov_len - offset);

		if (copy_from_user(skb->data + offset1, from->iov_base + offset,

				   size))

			return -EFAULT;

		if (copy > size) {

			++from;

			--count;

		}

		copy -= size;

		offset1 += size;

		offset = 0;

	}



	if (len == offset1)

		return 0;



	while (count--) {

		struct page *page[MAX_SKB_FRAGS];

		int num_pages;

		unsigned long base;



		len = from->iov_len - offset1;

		if (!len) {

			offset1 = 0;

			++from;

			continue;

		}

		base = (unsigned long)from->iov_base + offset1;

		size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT;

		num_pages = get_user_pages_fast(base, size, 0, &page[i]);

		if ((num_pages != size) ||

		    (num_pages > MAX_SKB_FRAGS - skb_shinfo(skb)->nr_frags))

			/* put_page is in skb free */

			return -EFAULT;

		skb->data_len += len;

		skb->len += len;

		skb->truesize += len;

		atomic_add(len, &skb->sk->sk_wmem_alloc);

		while (len) {

			f = &skb_shinfo(skb)->frags[i];

			f->page = page[i];

			f->page_offset = base & ~PAGE_MASK;

			f->size = min_t(int, len, PAGE_SIZE - f->page_offset);

			skb_shinfo(skb)->nr_frags++;

			/* increase sk_wmem_alloc */

			base += f->size;

			len -= f->size;

			i++;

		}

		offset1 = 0;

		++from;

	}

	return 0;

}



/*

 * macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should

 * be shared with the tun/tap driver.
@@ -517,16 +603,18 @@ static int macvtap_skb_to_vnet_hdr(const struct sk_buff *skb, 




/* Get packet from user space buffer */

static ssize_t macvtap_get_user(struct macvtap_queue *q,

				const struct iovec *iv, size_t count,

				int noblock)

static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,

				const struct iovec *iv, unsigned long total_len,

				size_t count, int noblock)

{

	struct sk_buff *skb;

	struct macvlan_dev *vlan;

	size_t len = count;

	unsigned long len = total_len;

	int err;

	struct virtio_net_hdr vnet_hdr = { 0 };

	int vnet_hdr_len = 0;

	int copylen;

	bool zerocopy = false;



	if (q->flags & IFF_VNET_HDR) {

		vnet_hdr_len = q->vnet_hdr_sz;
@@ -554,12 +642,31 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, 
	if (unlikely(len < ETH_HLEN))

		goto err;



	skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, len, vnet_hdr.hdr_len,

				noblock, &err);

	if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY))

		zerocopy = true;



	if (zerocopy) {

		/* There are 256 bytes to be copied in skb, so there is enough

		 * room for skb expand head in case it is used.

		 * The rest buffer is mapped from userspace.

		 */

		copylen = vnet_hdr.hdr_len;

		if (!copylen)

			copylen = GOODCOPY_LEN;

	} else

		copylen = len;



	skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen,

				vnet_hdr.hdr_len, noblock, &err);

	if (!skb)

		goto err;



	err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len, len);

	if (zerocopy) {

		err = zerocopy_sg_from_iovec(skb, iv, vnet_hdr_len, count);

		skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;

	} else

		err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len,

						   len);

	if (err)

		goto err_kfree;
@@ -575,13 +682,16 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, 


	rcu_read_lock_bh();

	vlan = rcu_dereference_bh(q->vlan);

	/* copy skb_ubuf_info for callback when skb has no error */

	if (zerocopy)

		skb_shinfo(skb)->destructor_arg = m->msg_control;

	if (vlan)

		macvlan_start_xmit(skb, vlan->dev);

	else

		kfree_skb(skb);

	rcu_read_unlock_bh();



	return count;

	return total_len;



err_kfree:

	kfree_skb(skb);
@@ -603,7 +713,7 @@ static ssize_t macvtap_aio_write(struct kiocb *iocb, const struct iovec *iv, 
	ssize_t result = -ENOLINK;

	struct macvtap_queue *q = file->private_data;



	result = macvtap_get_user(q, iv, iov_length(iv, count),

	result = macvtap_get_user(q, NULL, iv, iov_length(iv, count), count,

				  file->f_flags & O_NONBLOCK);

	return result;

}
@@ -817,7 +927,7 @@ static int macvtap_sendmsg(struct kiocb *iocb, struct socket *sock, 
			   struct msghdr *m, size_t total_len)

{

	struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock);

	return macvtap_get_user(q, m->msg_iov, total_len,

	return macvtap_get_user(q, m, m->msg_iov, total_len, m->msg_iovlen,

			    m->msg_flags & MSG_DONTWAIT);

}