Unverified Commit 84208b8f authored by Christian Brauner's avatar Christian Brauner
Browse files

Merge patch series "symlink length caching" 

Mateusz Guzik <mjguzik@gmail.com> says:

quote:
    When utilized it dodges strlen() in vfs_readlink(), giving about 1.5%
    speed up when issuing readlink on /initrd.img on ext4.

The size is stored in a union with i_devices, which is never looked at
unless the inode is for a device.

ext4 and tmpfs are patched, other filesystems can also get there with
some more work.

benchmark:
plug into will-it-scale into tests/readlink1.c:

char *testcase_description = "readlink /initrd.img";

void testcase(unsigned long long *iterations, unsigned long nr)
{
        char *tmplink = "/initrd.img";
        char buf[1024];

        while (1) {
                int error = readlink(tmplink, buf, sizeof(buf));
                assert(error > 0);

                (*iterations)++;
        }
}

* patches from https://lore.kernel.org/r/20241120112037.822078-1-mjguzik@gmail.com:
  tmpfs: use inode_set_cached_link()
  ext4: use inode_set_cached_link()
  vfs: support caching symlink lengths in inodes

Link: https://lore.kernel.org/r/20241120112037.822078-1-mjguzik@gmail.com


Signed-off-by: default avatarChristian Brauner <brauner@kernel.org>
parents 135ec43e 657e726e

fs/ext4/inode.c

+2 −1
Original line number Diff line number Diff line
@@ -5006,10 +5006,11 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, 
		if (IS_ENCRYPTED(inode)) {

			inode->i_op = &ext4_encrypted_symlink_inode_operations;

		} else if (ext4_inode_is_fast_symlink(inode)) {

			inode->i_link = (char *)ei->i_data;

			inode->i_op = &ext4_fast_symlink_inode_operations;

			nd_terminate_link(ei->i_data, inode->i_size,

				sizeof(ei->i_data) - 1);

			inode_set_cached_link(inode, (char *)ei->i_data,

					      inode->i_size);

		} else {

			inode->i_op = &ext4_symlink_inode_operations;

		}

fs/ext4/namei.c

+3 −1
Original line number Diff line number Diff line
@@ -3418,7 +3418,6 @@ static int ext4_symlink(struct mnt_idmap *idmap, struct inode *dir, 
			inode->i_op = &ext4_symlink_inode_operations;

		} else {

			inode->i_op = &ext4_fast_symlink_inode_operations;

			inode->i_link = (char *)&EXT4_I(inode)->i_data;

		}

	}
@@ -3434,6 +3433,9 @@ static int ext4_symlink(struct mnt_idmap *idmap, struct inode *dir, 
		       disk_link.len);

		inode->i_size = disk_link.len - 1;

		EXT4_I(inode)->i_disksize = inode->i_size;

		if (!IS_ENCRYPTED(inode))

			inode_set_cached_link(inode, (char *)&EXT4_I(inode)->i_data,

					      inode->i_size);

	}

	err = ext4_add_nondir(handle, dentry, &inode);

	if (handle)

fs/namei.c

+19 −15
Original line number Diff line number Diff line
@@ -5272,19 +5272,16 @@ SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newna 
				getname(newname), 0);

}



int readlink_copy(char __user *buffer, int buflen, const char *link)

int readlink_copy(char __user *buffer, int buflen, const char *link, int linklen)

{

	int len = PTR_ERR(link);

	if (IS_ERR(link))

		goto out;

	int copylen;



	len = strlen(link);

	if (len > (unsigned) buflen)

		len = buflen;

	if (copy_to_user(buffer, link, len))

		len = -EFAULT;

out:

	return len;

	copylen = linklen;

	if (unlikely(copylen > (unsigned) buflen))

		copylen = buflen;

	if (copy_to_user(buffer, link, copylen))

		copylen = -EFAULT;

	return copylen;

}



/**
@@ -5304,6 +5301,9 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen) 
	const char *link;

	int res;



	if (inode->i_opflags & IOP_CACHED_LINK)

		return readlink_copy(buffer, buflen, inode->i_link, inode->i_linklen);



	if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {

		if (unlikely(inode->i_op->readlink))

			return inode->i_op->readlink(dentry, buffer, buflen);
@@ -5322,7 +5322,7 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen) 
		if (IS_ERR(link))

			return PTR_ERR(link);

	}

	res = readlink_copy(buffer, buflen, link);

	res = readlink_copy(buffer, buflen, link, strlen(link));

	do_delayed_call(&done);

	return res;

}
@@ -5391,10 +5391,14 @@ EXPORT_SYMBOL(page_put_link); 


int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)

{

	const char *link;

	int res;



	DEFINE_DELAYED_CALL(done);

	int res = readlink_copy(buffer, buflen,

				page_get_link(dentry, d_inode(dentry),

					      &done));

	link = page_get_link(dentry, d_inode(dentry), &done);

	res = PTR_ERR(link);

	if (!IS_ERR(link))

		res = readlink_copy(buffer, buflen, link, strlen(link));

	do_delayed_call(&done);

	return res;

}

fs/proc/namespaces.c

+1 −1
Original line number Diff line number Diff line
@@ -83,7 +83,7 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl 
	if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {

		res = ns_get_name(name, sizeof(name), task, ns_ops);

		if (res >= 0)

			res = readlink_copy(buffer, buflen, name);

			res = readlink_copy(buffer, buflen, name, strlen(name));

	}

	put_task_struct(task);

	return res;

include/linux/fs.h

+13 −2
Original line number Diff line number Diff line
@@ -626,6 +626,7 @@ is_uncached_acl(struct posix_acl *acl) 
#define IOP_XATTR	0x0008

#define IOP_DEFAULT_READLINK	0x0010

#define IOP_MGTIME	0x0020

#define IOP_CACHED_LINK	0x0040



/*

 * Keep mostly read-only and often accessed (especially for
@@ -723,7 +724,10 @@ struct inode { 
	};

	struct file_lock_context	*i_flctx;

	struct address_space	i_data;

	union {

		struct list_head	i_devices;

		int			i_linklen;

	};

	union {

		struct pipe_inode_info	*i_pipe;

		struct cdev		*i_cdev;
@@ -749,6 +753,13 @@ struct inode { 
	void			*i_private; /* fs or device private pointer */

} __randomize_layout;



static inline void inode_set_cached_link(struct inode *inode, char *link, int linklen)

{

	inode->i_link = link;

	inode->i_linklen = linklen;

	inode->i_opflags |= IOP_CACHED_LINK;

}



/*

 * Get bit address from inode->i_state to use with wait_var_event()

 * infrastructre.
@@ -3351,7 +3362,7 @@ extern const struct file_operations generic_ro_fops; 


#define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))



extern int readlink_copy(char __user *, int, const char *);

extern int readlink_copy(char __user *, int, const char *, int);

extern int page_readlink(struct dentry *, char __user *, int);

extern const char *page_get_link(struct dentry *, struct inode *,

				 struct delayed_call *);