Commit 2c52e894 authored by Mykyta Yatsenko's avatar Mykyta Yatsenko Committed by Alexei Starovoitov
Browse files

bpf: dispatch to sleepable file dynptr 



File dynptr reads may sleep when the requested folios are not in
the page cache. To avoid sleeping in non-sleepable contexts while still
supporting valid sleepable use, given that dynptrs are non-sleepable by
default, enable sleeping only when bpf_dynptr_from_file() is invoked
from a sleepable context.

This change:
  * Introduces a sleepable constructor: bpf_dynptr_from_file_sleepable()
  * Override non-sleepable constructor with sleepable if it's always
  called in sleepable context

Signed-off-by: default avatarMykyta Yatsenko <yatsenko@meta.com>
Acked-by: default avatarEduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251026203853.135105-10-mykyta.yatsenko5@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent d869d56c

include/linux/bpf.h

+3 −0
Original line number Diff line number Diff line
@@ -670,6 +670,9 @@ static inline bool bpf_map_has_internal_structs(struct bpf_map *map) 


void bpf_map_free_internal_structs(struct bpf_map *map, void *obj);



int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags,

				   struct bpf_dynptr *ptr__uninit);



extern const struct bpf_map_ops bpf_map_offload_ops;



/* bpf_type_flag contains a set of flags that are applicable to the values of

kernel/bpf/helpers.c

+5 −0
Original line number Diff line number Diff line
@@ -4336,6 +4336,11 @@ __bpf_kfunc int bpf_dynptr_from_file(struct file *file, u32 flags, struct bpf_dy 
	return make_file_dynptr(file, flags, false, (struct bpf_dynptr_kern *)ptr__uninit);

}



int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags, struct bpf_dynptr *ptr__uninit)

{

	return make_file_dynptr(file, flags, true, (struct bpf_dynptr_kern *)ptr__uninit);

}



__bpf_kfunc int bpf_dynptr_file_discard(struct bpf_dynptr *dynptr)

{

	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)dynptr;

kernel/bpf/verifier.c

+7 −3
Original line number Diff line number Diff line
@@ -3124,7 +3124,8 @@ struct bpf_kfunc_btf_tab { 
	u32 nr_descs;

};









static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc);









static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc,









			    int insn_idx);

















static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)









{








@@ -21869,7 +21870,7 @@ static int fixup_call_args(struct bpf_verifier_env *env)







}

















/* replace a generic kfunc with a specialized version if necessary */









static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc)









static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, int insn_idx)









{









	struct bpf_prog *prog = env->prog;









	bool seen_direct_write;








@@ -21904,6 +21905,9 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc







	} else if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr]) {









		if (bpf_lsm_has_d_inode_locked(prog))









			addr = (unsigned long)bpf_remove_dentry_xattr_locked;









	} else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) {









		if (!env->insn_aux_data[insn_idx].non_sleepable)









			addr = (unsigned long)bpf_dynptr_from_file_sleepable;









	}

















set_imm:








@@ -21963,7 +21967,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,







		return -EFAULT;









	}

















	err = specialize_kfunc(env, desc);









	err = specialize_kfunc(env, desc, insn_idx);









	if (err)









		return err;