Commit 2d419c44 authored by Menglong Dong's avatar Menglong Dong Committed by Alexei Starovoitov
Browse files

bpf: add fsession support 



The fsession is something that similar to kprobe session. It allow to
attach a single BPF program to both the entry and the exit of the target
functions.

Introduce the struct bpf_fsession_link, which allows to add the link to
both the fentry and fexit progs_hlist of the trampoline.

Signed-off-by: default avatarMenglong Dong <dongml2@chinatelecom.cn>
Co-developed-by: default avatarLeon Hwang <leon.hwang@linux.dev>
Signed-off-by: default avatarLeon Hwang <leon.hwang@linux.dev>
Link: https://lore.kernel.org/r/20260124062008.8657-2-dongml2@chinatelecom.cn


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent c7900f22

include/linux/bpf.h

+19 −0
Original line number Diff line number Diff line
@@ -1309,6 +1309,7 @@ enum bpf_tramp_prog_type { 
	BPF_TRAMP_MODIFY_RETURN,

	BPF_TRAMP_MAX,

	BPF_TRAMP_REPLACE, /* more than MAX */

	BPF_TRAMP_FSESSION,

};



struct bpf_tramp_image {
@@ -1875,6 +1876,11 @@ struct bpf_tracing_link { 
	struct bpf_prog *tgt_prog;

};



struct bpf_fsession_link {

	struct bpf_tracing_link link;

	struct bpf_tramp_link fexit;

};



struct bpf_raw_tp_link {

	struct bpf_link link;

	struct bpf_raw_event_map *btp;
@@ -2169,6 +2175,19 @@ static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_op 


#endif



static inline int bpf_fsession_cnt(struct bpf_tramp_links *links)

{

	struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY];

	int cnt = 0;



	for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) {

		if (fentries.links[i]->link.prog->expected_attach_type == BPF_TRACE_FSESSION)

			cnt++;

	}



	return cnt;

}



int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,

			       const struct bpf_ctx_arg_aux *info, u32 cnt);

include/uapi/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -1145,6 +1145,7 @@ enum bpf_attach_type { 
	BPF_NETKIT_PEER,

	BPF_TRACE_KPROBE_SESSION,

	BPF_TRACE_UPROBE_SESSION,

	BPF_TRACE_FSESSION,

	__MAX_BPF_ATTACH_TYPE

};

kernel/bpf/btf.c

+2 −0
Original line number Diff line number Diff line
@@ -6219,6 +6219,7 @@ static int btf_validate_prog_ctx_type(struct bpf_verifier_log *log, const struct 
		case BPF_TRACE_FENTRY:

		case BPF_TRACE_FEXIT:

		case BPF_MODIFY_RETURN:

		case BPF_TRACE_FSESSION:

			/* allow u64* as ctx */

			if (btf_is_int(t) && t->size == 8)

				return 0;
@@ -6820,6 +6821,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, 
			fallthrough;

		case BPF_LSM_CGROUP:

		case BPF_TRACE_FEXIT:

		case BPF_TRACE_FSESSION:

			/* When LSM programs are attached to void LSM hooks

			 * they use FEXIT trampolines and when attached to

			 * int LSM hooks, they use MODIFY_RETURN trampolines.

kernel/bpf/syscall.c

+17 −1
Original line number Diff line number Diff line
@@ -3577,6 +3577,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, 
	case BPF_PROG_TYPE_TRACING:

		if (prog->expected_attach_type != BPF_TRACE_FENTRY &&

		    prog->expected_attach_type != BPF_TRACE_FEXIT &&

		    prog->expected_attach_type != BPF_TRACE_FSESSION &&

		    prog->expected_attach_type != BPF_MODIFY_RETURN) {

			err = -EINVAL;

			goto out_put_prog;
@@ -3626,7 +3627,21 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, 
		key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);

	}



	if (prog->expected_attach_type == BPF_TRACE_FSESSION) {

		struct bpf_fsession_link *fslink;



		fslink = kzalloc(sizeof(*fslink), GFP_USER);

		if (fslink) {

			bpf_link_init(&fslink->fexit.link, BPF_LINK_TYPE_TRACING,

				      &bpf_tracing_link_lops, prog, attach_type);

			fslink->fexit.cookie = bpf_cookie;

			link = &fslink->link;

		} else {

			link = NULL;

		}

	} else {

		link = kzalloc(sizeof(*link), GFP_USER);

	}

	if (!link) {

		err = -ENOMEM;

		goto out_put_prog;
@@ -4350,6 +4365,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) 
	case BPF_TRACE_RAW_TP:

	case BPF_TRACE_FENTRY:

	case BPF_TRACE_FEXIT:

	case BPF_TRACE_FSESSION:

	case BPF_MODIFY_RETURN:

		return BPF_PROG_TYPE_TRACING;

	case BPF_LSM_MAC:

kernel/bpf/trampoline.c

+45 −8
Original line number Diff line number Diff line
@@ -109,10 +109,17 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) 
	enum bpf_attach_type eatype = prog->expected_attach_type;

	enum bpf_prog_type ptype = prog->type;



	return (ptype == BPF_PROG_TYPE_TRACING &&

		(eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||

		 eatype == BPF_MODIFY_RETURN)) ||

		(ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);

	switch (ptype) {

	case BPF_PROG_TYPE_TRACING:

		if (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||

		    eatype == BPF_MODIFY_RETURN || eatype == BPF_TRACE_FSESSION)

			return true;

		return false;

	case BPF_PROG_TYPE_LSM:

		return eatype == BPF_LSM_MAC;

	default:

		return false;

	}

}



void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym)
@@ -559,6 +566,8 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) 
		return BPF_TRAMP_MODIFY_RETURN;

	case BPF_TRACE_FEXIT:

		return BPF_TRAMP_FEXIT;

	case BPF_TRACE_FSESSION:

		return BPF_TRAMP_FSESSION;

	case BPF_LSM_MAC:

		if (!prog->aux->attach_func_proto->type)

			/* The function returns void, we cannot modify its
@@ -594,8 +603,10 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, 
				      struct bpf_trampoline *tr,

				      struct bpf_prog *tgt_prog)

{

	struct bpf_fsession_link *fslink = NULL;

	enum bpf_tramp_prog_type kind;

	struct bpf_tramp_link *link_exiting;

	struct hlist_head *prog_list;

	int err = 0;

	int cnt = 0, i;
@@ -621,25 +632,44 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, 
					  BPF_MOD_JUMP, NULL,

					  link->link.prog->bpf_func);

	}

	if (kind == BPF_TRAMP_FSESSION) {

		prog_list = &tr->progs_hlist[BPF_TRAMP_FENTRY];

		cnt++;

	} else {

		prog_list = &tr->progs_hlist[kind];

	}

	if (cnt >= BPF_MAX_TRAMP_LINKS)

		return -E2BIG;

	if (!hlist_unhashed(&link->tramp_hlist))

		/* prog already linked */

		return -EBUSY;

	hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) {

	hlist_for_each_entry(link_exiting, prog_list, tramp_hlist) {

		if (link_exiting->link.prog != link->link.prog)

			continue;

		/* prog already linked */

		return -EBUSY;

	}



	hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);

	hlist_add_head(&link->tramp_hlist, prog_list);

	if (kind == BPF_TRAMP_FSESSION) {

		tr->progs_cnt[BPF_TRAMP_FENTRY]++;

		fslink = container_of(link, struct bpf_fsession_link, link.link);

		hlist_add_head(&fslink->fexit.tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]);

		tr->progs_cnt[BPF_TRAMP_FEXIT]++;

	} else {

		tr->progs_cnt[kind]++;

	}

	err = bpf_trampoline_update(tr, true /* lock_direct_mutex */);

	if (err) {

		hlist_del_init(&link->tramp_hlist);

		if (kind == BPF_TRAMP_FSESSION) {

			tr->progs_cnt[BPF_TRAMP_FENTRY]--;

			hlist_del_init(&fslink->fexit.tramp_hlist);

			tr->progs_cnt[BPF_TRAMP_FEXIT]--;

		} else {

			tr->progs_cnt[kind]--;

		}

	}

	return err;

}
@@ -672,6 +702,13 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, 
		guard(mutex)(&tgt_prog->aux->ext_mutex);

		tgt_prog->aux->is_extended = false;

		return err;

	} else if (kind == BPF_TRAMP_FSESSION) {

		struct bpf_fsession_link *fslink =

			container_of(link, struct bpf_fsession_link, link.link);



		hlist_del_init(&fslink->fexit.tramp_hlist);

		tr->progs_cnt[BPF_TRAMP_FEXIT]--;

		kind = BPF_TRAMP_FENTRY;

	}

	hlist_del_init(&link->tramp_hlist);

	tr->progs_cnt[kind]--;