Merge tag 'vfs-6.17-rc1.iomap' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs

     struct dax_device   *dax_dev;

     void                *inline_data;

     void                *private;

     const struct iomap_folio_ops *folio_ops;

     u64                 validity_cookie;

 };

   <https://lore.kernel.org/all/20180619164137.13720-7-hch@lst.de/>`_.

   This value will be passed unchanged to ``->iomap_end``.

 * ``folio_ops`` will be covered in the section on pagecache operations.

 * ``validity_cookie`` is a magic freshness value set by the filesystem

   that should be used to detect stale mappings.

   For pagecache operations this is critical for correct operation

 * ``bmap``

 * ``swap_activate``

``struct iomap_folio_ops``

``struct iomap_write_ops``

--------------------------

The ``->iomap_begin`` function for pagecache operations may set the

``struct iomap::folio_ops`` field to an ops structure to override

default behaviors of iomap:

.. code-block:: c

 struct iomap_folio_ops {

 struct iomap_write_ops {

     struct folio *(*get_folio)(struct iomap_iter *iter, loff_t pos,

                                unsigned len);

     void (*put_folio)(struct inode *inode, loff_t pos, unsigned copied,

                       struct folio *folio);

     bool (*iomap_valid)(struct inode *inode, const struct iomap *iomap);

     int (*read_folio_range)(const struct iomap_iter *iter,

     			struct folio *folio, loff_t pos, size_t len);

 };

iomap calls these functions:

    ``->iomap_valid``, then the iomap should considered stale and the

    validation failed.

  - ``read_folio_range``: Called to synchronously read in the range that will

    be written to. If this function is not provided, iomap will default to

    submitting a bio read request.

These ``struct kiocb`` flags are significant for buffered I/O with iomap:

 * ``IOCB_NOWAIT``: Turns on ``IOMAP_NOWAIT``.

It does not lock ``i_rwsem`` or ``invalidate_lock``.

The dirty bit will be cleared for all folios run through the

``->map_blocks`` machinery described below even if the writeback fails.

``->writeback_range`` machinery described below even if the writeback fails.

This is to prevent dirty folio clots when storage devices fail; an

``-EIO`` is recorded for userspace to collect via ``fsync``.

.. code-block:: c

 struct iomap_writeback_ops {

     int (*map_blocks)(struct iomap_writepage_ctx *wpc, struct inode *inode,

                       loff_t offset, unsigned len);

     int (*submit_ioend)(struct iomap_writepage_ctx *wpc, int status);

     void (*discard_folio)(struct folio *folio, loff_t pos);

    int (*writeback_range)(struct iomap_writepage_ctx *wpc,

        struct folio *folio, u64 pos, unsigned int len, u64 end_pos);

    int (*writeback_submit)(struct iomap_writepage_ctx *wpc, int error);

 };

The fields are as follows:

  - ``map_blocks``: Sets ``wpc->iomap`` to the space mapping of the file

  - ``writeback_range``: Sets ``wpc->iomap`` to the space mapping of the file

    range (in bytes) given by ``offset`` and ``len``.

    iomap calls this function for each dirty fs block in each dirty folio,

    though it will `reuse mappings

    This revalidation must be open-coded by the filesystem; it is

    unclear if ``iomap::validity_cookie`` can be reused for this

    purpose.

    This function must be supplied by the filesystem.

  - ``submit_ioend``: Allows the file systems to hook into writeback bio

    submission.

    This might include pre-write space accounting updates, or installing

    a custom ``->bi_end_io`` function for internal purposes, such as

    deferring the ioend completion to a workqueue to run metadata update

    transactions from process context before submitting the bio.

    This function is optional.

  - ``discard_folio``: iomap calls this function after ``->map_blocks``

    fails to schedule I/O for any part of a dirty folio.

    The function should throw away any reservations that may have been

    made for the write.

    If this methods fails to schedule I/O for any part of a dirty folio, it

    should throw away any reservations that may have been made for the write.

    The folio will be marked clean and an ``-EIO`` recorded in the

    pagecache.

    Filesystems can use this callback to `remove

    <https://lore.kernel.org/all/20201029163313.1766967-1-bfoster@redhat.com/>`_

    delalloc reservations to avoid having delalloc reservations for

    clean pagecache.

    This function is optional.

    This function must be supplied by the filesystem.

  - ``writeback_submit``: Submit the previous built writeback context.

    Block based file systems should use the iomap_ioend_writeback_submit

    helper, other file system can implement their own.

    File systems can optionall to hook into writeback bio submission.

    This might include pre-write space accounting updates, or installing

    a custom ``->bi_end_io`` function for internal purposes, such as

    deferring the ioend completion to a workqueue to run metadata update

    transactions from process context before submitting the bio.

    This function must be supplied by the filesystem.

Pagecache Writeback Completion

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

the address space.

This can happen in interrupt or process context, depending on the

storage device.

Filesystems that need to update internal bookkeeping (e.g. unwritten

extent conversions) should provide a ``->submit_ioend`` function to

set ``struct iomap_end::bio::bi_end_io`` to its own function.

extent conversions) should set their own bi_end_io on the bios

submitted by ``->submit_writeback``

This function should call ``iomap_finish_ioends`` after finishing its

own work (e.g. unwritten extent conversion).

	iomap_readahead(rac, &blkdev_iomap_ops);

}

static int blkdev_map_blocks(struct iomap_writepage_ctx *wpc,

		struct inode *inode, loff_t offset, unsigned int len)

static ssize_t blkdev_writeback_range(struct iomap_writepage_ctx *wpc,

		struct folio *folio, u64 offset, unsigned int len, u64 end_pos)

{

	loff_t isize = i_size_read(inode);

	loff_t isize = i_size_read(wpc->inode);

	if (WARN_ON_ONCE(offset >= isize))

		return -EIO;

	if (offset >= wpc->iomap.offset &&

	    offset < wpc->iomap.offset + wpc->iomap.length)

		return 0;

	return blkdev_iomap_begin(inode, offset, isize - offset,

	if (offset < wpc->iomap.offset ||

	    offset >= wpc->iomap.offset + wpc->iomap.length) {

		int error;

		error = blkdev_iomap_begin(wpc->inode, offset, isize - offset,

				IOMAP_WRITE, &wpc->iomap, NULL);

		if (error)

			return error;

	}

	return iomap_add_to_ioend(wpc, folio, offset, end_pos, len);

}

static const struct iomap_writeback_ops blkdev_writeback_ops = {

	.map_blocks		= blkdev_map_blocks,

	.writeback_range	= blkdev_writeback_range,

	.writeback_submit	= iomap_ioend_writeback_submit,

};

static int blkdev_writepages(struct address_space *mapping,

		struct writeback_control *wbc)

{

	struct iomap_writepage_ctx wpc = { };

	struct iomap_writepage_ctx wpc = {

		.inode		= mapping->host,

		.wbc		= wbc,

		.ops		= &blkdev_writeback_ops

	};

	return iomap_writepages(mapping, wbc, &wpc, &blkdev_writeback_ops);

	return iomap_writepages(&wpc);

}

const struct address_space_operations def_blk_aops = {

static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from)

{

	return iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL);

	return iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL,

			NULL);

}

/*

config FUSE_FS

	tristate "FUSE (Filesystem in Userspace) support"

	select FS_POSIX_ACL

	select FS_IOMAP

	help

	  With FUSE it is possible to implement a fully functional filesystem

	  in a userspace program.

#include <linux/filelock.h>

#include <linux/splice.h>

#include <linux/task_io_accounting_ops.h>

#include <linux/iomap.h>

static int fuse_send_open(struct fuse_mount *fm, u64 nodeid,

			  unsigned int open_flags, int opcode,

	}

}

static int fuse_do_readfolio(struct file *file, struct folio *folio)

static int fuse_do_readfolio(struct file *file, struct folio *folio,

			     size_t off, size_t len)

{

	struct inode *inode = folio->mapping->host;

	struct fuse_mount *fm = get_fuse_mount(inode);

	loff_t pos = folio_pos(folio);

	struct fuse_folio_desc desc = { .length = folio_size(folio) };

	loff_t pos = folio_pos(folio) + off;

	struct fuse_folio_desc desc = {

		.offset = off,

		.length = len,

	};

	struct fuse_io_args ia = {

		.ap.args.page_zeroing = true,

		.ap.args.out_pages = true,

	if (res < desc.length)

		fuse_short_read(inode, attr_ver, res, &ia.ap);

	folio_mark_uptodate(folio);

	return 0;

}

	if (fuse_is_bad(inode))

		goto out;

	err = fuse_do_readfolio(file, folio);

	err = fuse_do_readfolio(file, folio, 0, folio_size(folio));

	if (!err)

		folio_mark_uptodate(folio);

	fuse_invalidate_atime(inode);

 out:

	folio_unlock(folio);

	return err;

}

static int fuse_iomap_read_folio_range(const struct iomap_iter *iter,

				       struct folio *folio, loff_t pos,

				       size_t len)

{

	struct file *file = iter->private;

	size_t off = offset_in_folio(folio, pos);

	return fuse_do_readfolio(file, folio, off, len);

}

static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,

			       int err)

{

	}

}

static const struct iomap_write_ops fuse_iomap_write_ops = {

	.read_folio_range = fuse_iomap_read_folio_range,

};

static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length,

			    unsigned int flags, struct iomap *iomap,

			    struct iomap *srcmap)

{

	iomap->type = IOMAP_MAPPED;

	iomap->length = length;

	iomap->offset = offset;

	return 0;

}

static const struct iomap_ops fuse_iomap_ops = {

	.iomap_begin	= fuse_iomap_begin,

};

static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)

{

	struct file *file = iocb->ki_filp;

	struct inode *inode = mapping->host;

	ssize_t err, count;

	struct fuse_conn *fc = get_fuse_conn(inode);

	bool writeback = false;

	if (fc->writeback_cache) {

		/* Update size (EOF optimization) and mode (SUID clearing) */

		if (err)

			return err;

		if (fc->handle_killpriv_v2 &&

		    setattr_should_drop_suidgid(idmap,

						file_inode(file))) {

			goto writethrough;

		}

		return generic_file_write_iter(iocb, from);

		if (!fc->handle_killpriv_v2 ||

		    !setattr_should_drop_suidgid(idmap, file_inode(file)))

			writeback = true;

	}

writethrough:

	inode_lock(inode);

	err = count = generic_write_checks(iocb, from);

			goto out;

		written = direct_write_fallback(iocb, from, written,

				fuse_perform_write(iocb, from));

	} else if (writeback) {

		/*

		 * Use iomap so that we can do granular uptodate reads

		 * and granular dirty tracking for large folios.

		 */

		written = iomap_file_buffered_write(iocb, from,

						    &fuse_iomap_ops,

						    &fuse_iomap_write_ops,

						    file);

	} else {

		written = fuse_perform_write(iocb, from);

	}

		 * scope of the fi->lock alleviates xarray lock

		 * contention and noticeably improves performance.

		 */

		folio_end_writeback(ap->folios[i]);

		iomap_finish_folio_write(inode, ap->folios[i], 1);

		dec_wb_stat(&bdi->wb, WB_WRITEBACK);

		wb_writeout_inc(&bdi->wb);

	}

}

static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio,

					  uint32_t folio_index)

					  uint32_t folio_index, loff_t offset, unsigned len)

{

	struct inode *inode = folio->mapping->host;

	struct fuse_args_pages *ap = &wpa->ia.ap;

	ap->folios[folio_index] = folio;

	ap->descs[folio_index].offset = 0;

	ap->descs[folio_index].length = folio_size(folio);

	ap->descs[folio_index].offset = offset;

	ap->descs[folio_index].length = len;

	inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);

}

static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio,

							     size_t offset,

							     struct fuse_file *ff)

{

	struct inode *inode = folio->mapping->host;

		return NULL;

	fuse_writepage_add_to_bucket(fc, wpa);

	fuse_write_args_fill(&wpa->ia, ff, folio_pos(folio), 0);

	fuse_write_args_fill(&wpa->ia, ff, folio_pos(folio) + offset, 0);

	wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;

	wpa->inode = inode;

	wpa->ia.ff = ff;

	return wpa;

}

static int fuse_writepage_locked(struct folio *folio)

{

	struct address_space *mapping = folio->mapping;

	struct inode *inode = mapping->host;

	struct fuse_inode *fi = get_fuse_inode(inode);

	struct fuse_writepage_args *wpa;

	struct fuse_args_pages *ap;

	struct fuse_file *ff;

	int error = -EIO;

	ff = fuse_write_file_get(fi);

	if (!ff)

		goto err;

	wpa = fuse_writepage_args_setup(folio, ff);

	error = -ENOMEM;

	if (!wpa)

		goto err_writepage_args;

	ap = &wpa->ia.ap;

	ap->num_folios = 1;

	folio_start_writeback(folio);

	fuse_writepage_args_page_fill(wpa, folio, 0);

	spin_lock(&fi->lock);

	list_add_tail(&wpa->queue_entry, &fi->queued_writes);

	fuse_flush_writepages(inode);

	spin_unlock(&fi->lock);

	return 0;

err_writepage_args:

	fuse_file_put(ff, false);

err:

	mapping_set_error(folio->mapping, error);

	return error;

}

struct fuse_fill_wb_data {

	struct fuse_writepage_args *wpa;

	struct fuse_file *ff;

	struct inode *inode;

	unsigned int max_folios;

	unsigned int nr_pages;

	/*

	 * nr_bytes won't overflow since fuse_writepage_need_send() caps

	 * wb requests to never exceed fc->max_pages (which has an upper bound

	 * of U16_MAX).

	 */

	unsigned int nr_bytes;

};

static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)

static bool fuse_pages_realloc(struct fuse_fill_wb_data *data,

			       unsigned int max_pages)

{

	struct fuse_args_pages *ap = &data->wpa->ia.ap;

	struct fuse_conn *fc = get_fuse_conn(data->inode);

	struct folio **folios;

	struct fuse_folio_desc *descs;

	unsigned int nfolios = min_t(unsigned int,

				     max_t(unsigned int, data->max_folios * 2,

					   FUSE_DEFAULT_MAX_PAGES_PER_REQ),

				    fc->max_pages);

				    max_pages);

	WARN_ON(nfolios <= data->max_folios);

	folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs);

	return true;

}

static void fuse_writepages_send(struct fuse_fill_wb_data *data)

static void fuse_writepages_send(struct inode *inode,

				 struct fuse_fill_wb_data *data)

{

	struct fuse_writepage_args *wpa = data->wpa;

	struct inode *inode = data->inode;

	struct fuse_inode *fi = get_fuse_inode(inode);

	spin_lock(&fi->lock);

	spin_unlock(&fi->lock);

}

static bool fuse_writepage_need_send(struct fuse_conn *fc, struct folio *folio,

				     struct fuse_args_pages *ap,

static bool fuse_writepage_need_send(struct fuse_conn *fc, loff_t pos,

				     unsigned len, struct fuse_args_pages *ap,

				     struct fuse_fill_wb_data *data)

{

	struct folio *prev_folio;

	struct fuse_folio_desc prev_desc;

	unsigned bytes = data->nr_bytes + len;

	loff_t prev_pos;

	WARN_ON(!ap->num_folios);

	/* Reached max pages */

	if (data->nr_pages + folio_nr_pages(folio) > fc->max_pages)

	if ((bytes + PAGE_SIZE - 1) >> PAGE_SHIFT > fc->max_pages)

		return true;

	/* Reached max write bytes */

	if ((data->nr_pages * PAGE_SIZE) + folio_size(folio) > fc->max_write)

	if (bytes > fc->max_write)

		return true;

	/* Discontinuity */

	if (folio_next_index(ap->folios[ap->num_folios - 1]) != folio->index)

	prev_folio = ap->folios[ap->num_folios - 1];

	prev_desc = ap->descs[ap->num_folios - 1];

	prev_pos = folio_pos(prev_folio) + prev_desc.offset + prev_desc.length;

	if (prev_pos != pos)

		return true;

	/* Need to grow the pages array?  If so, did the expansion fail? */

	if (ap->num_folios == data->max_folios && !fuse_pages_realloc(data))

	if (ap->num_folios == data->max_folios &&

	    !fuse_pages_realloc(data, fc->max_pages))

		return true;

	return false;

}

static int fuse_writepages_fill(struct folio *folio,

		struct writeback_control *wbc, void *_data)

static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc,

					  struct folio *folio, u64 pos,

					  unsigned len, u64 end_pos)

{

	struct fuse_fill_wb_data *data = _data;

	struct fuse_fill_wb_data *data = wpc->wb_ctx;

	struct fuse_writepage_args *wpa = data->wpa;

	struct fuse_args_pages *ap = &wpa->ia.ap;

	struct inode *inode = data->inode;

	struct inode *inode = wpc->inode;

	struct fuse_inode *fi = get_fuse_inode(inode);

	struct fuse_conn *fc = get_fuse_conn(inode);

	int err;

	loff_t offset = offset_in_folio(folio, pos);

	WARN_ON_ONCE(!data);

	/* len will always be page aligned */

	WARN_ON_ONCE(len & (PAGE_SIZE - 1));

	if (!data->ff) {

		err = -EIO;

		data->ff = fuse_write_file_get(fi);

		if (!data->ff)

			goto out_unlock;

			return -EIO;

	}

	if (wpa && fuse_writepage_need_send(fc, folio, ap, data)) {

		fuse_writepages_send(data);

	if (wpa && fuse_writepage_need_send(fc, pos, len, ap, data)) {

		fuse_writepages_send(inode, data);

		data->wpa = NULL;

		data->nr_pages = 0;

		data->nr_bytes = 0;

	}

	if (data->wpa == NULL) {

		err = -ENOMEM;

		wpa = fuse_writepage_args_setup(folio, data->ff);

		wpa = fuse_writepage_args_setup(folio, offset, data->ff);

		if (!wpa)

			goto out_unlock;

			return -ENOMEM;

		fuse_file_get(wpa->ia.ff);

		data->max_folios = 1;

		ap = &wpa->ia.ap;

	}

	folio_start_writeback(folio);

	fuse_writepage_args_page_fill(wpa, folio, ap->num_folios);

	data->nr_pages += folio_nr_pages(folio);

	iomap_start_folio_write(inode, folio, 1);

	fuse_writepage_args_page_fill(wpa, folio, ap->num_folios,

				      offset, len);

	data->nr_bytes += len;

	err = 0;

	ap->num_folios++;

	if (!data->wpa)

		data->wpa = wpa;

out_unlock:

	folio_unlock(folio);

	return err;

	return len;

}

static int fuse_writepages(struct address_space *mapping,

			   struct writeback_control *wbc)

static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc,

				       int error)

{

	struct inode *inode = mapping->host;

	struct fuse_conn *fc = get_fuse_conn(inode);

	struct fuse_fill_wb_data data;

	int err;

	err = -EIO;

	if (fuse_is_bad(inode))

		goto out;

	if (wbc->sync_mode == WB_SYNC_NONE &&

	    fc->num_background >= fc->congestion_threshold)

		return 0;

	data.inode = inode;

	data.wpa = NULL;

	data.ff = NULL;

	data.nr_pages = 0;

	struct fuse_fill_wb_data *data = wpc->wb_ctx;

	err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);

	if (data.wpa) {

		WARN_ON(!data.wpa->ia.ap.num_folios);

		fuse_writepages_send(&data);

	}

	if (data.ff)

		fuse_file_put(data.ff, false);

	WARN_ON_ONCE(!data);

out:

	return err;

	if (data->wpa) {

		WARN_ON(!data->wpa->ia.ap.num_folios);

		fuse_writepages_send(wpc->inode, data);

	}

/*

 * It's worthy to make sure that space is reserved on disk for the write,

 * but how to implement it without killing performance need more thinking.

 */

static int fuse_write_begin(const struct kiocb *iocb,

			    struct address_space *mapping,

			    loff_t pos, unsigned len, struct folio **foliop,

			    void **fsdata)

{

	pgoff_t index = pos >> PAGE_SHIFT;

	struct file *file = iocb->ki_filp;

	struct fuse_conn *fc = get_fuse_conn(file_inode(file));

	struct folio *folio;

	loff_t fsize;

	int err = -ENOMEM;

	WARN_ON(!fc->writeback_cache);

	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,

			mapping_gfp_mask(mapping));

	if (IS_ERR(folio))

		goto error;

	if (data->ff)

		fuse_file_put(data->ff, false);