Commit ee88bddf authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf 

Pull bpf fixes from Alexei Starovoitov:

 - Fix use-after-free in libbpf when map is resized (Adin Scannell)

 - Fix verifier assumptions about 2nd argument of bpf_sysctl_get_name
   (Jerome Marchand)

 - Fix verifier assumption of nullness of d_inode in dentry (Song Liu)

 - Fix global starvation of LRU map (Willem de Bruijn)

 - Fix potential NULL dereference in btf_dump__free (Yuan Chen)

* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf:
  selftests/bpf: adapt one more case in test_lru_map to the new target_free
  libbpf: Fix possible use-after-free for externs
  selftests/bpf: Convert test_sysctl to prog_tests
  bpf: Specify access type of bpf_sysctl_get_name args
  libbpf: Fix null pointer dereference in btf_dump__free on allocation failure
  bpf: Adjust free target to avoid global starvation of LRU map
  bpf: Mark dentry->d_inode as trusted_or_null
parents c5c2a8b4 5e9388f7

Documentation/bpf/map_hash.rst

+7 −1
Original line number Diff line number Diff line
@@ -233,10 +233,16 @@ attempts in order to enforce the LRU property which have increasing impacts on 
other CPUs involved in the following operation attempts:



- Attempt to use CPU-local state to batch operations

- Attempt to fetch free nodes from global lists

- Attempt to fetch ``target_free`` free nodes from global lists

- Attempt to pull any node from a global list and remove it from the hashmap

- Attempt to pull any node from any CPU's list and remove it from the hashmap



The number of nodes to borrow from the global list in a batch, ``target_free``,

depends on the size of the map. Larger batch size reduces lock contention, but

may also exhaust the global structure. The value is computed at map init to

avoid exhaustion, by limiting aggregate reservation by all CPUs to half the map

size. With a minimum of a single element and maximum budget of 128 at a time.



This algorithm is described visually in the following diagram. See the

description in commit 3a08c2fd7634 ("bpf: LRU List") for a full explanation of

the corresponding operations:

Documentation/bpf/map_lru_hash_update.dot

+3 −3
Original line number Diff line number Diff line
@@ -35,18 +35,18 @@ digraph { 
  fn_bpf_lru_list_pop_free_to_local [shape=rectangle,fillcolor=2,

    label="Flush local pending,

    Rotate Global list, move

    LOCAL_FREE_TARGET

    target_free

    from global -> local"]

  // Also corresponds to:

  // fn__local_list_flush()

  // fn_bpf_lru_list_rotate()

  fn___bpf_lru_node_move_to_free[shape=diamond,fillcolor=2,

    label="Able to free\nLOCAL_FREE_TARGET\nnodes?"]

    label="Able to free\ntarget_free\nnodes?"]



  fn___bpf_lru_list_shrink_inactive [shape=rectangle,fillcolor=3,

    label="Shrink inactive list

      up to remaining

      LOCAL_FREE_TARGET

      target_free

      (global LRU -> local)"]

  fn___bpf_lru_list_shrink [shape=diamond,fillcolor=2,

    label="> 0 entries in\nlocal free list?"]

kernel/bpf/bpf_lru_list.c

+6 −3
Original line number Diff line number Diff line
@@ -337,12 +337,12 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, 
				 list) {

		__bpf_lru_node_move_to_free(l, node, local_free_list(loc_l),

					    BPF_LRU_LOCAL_LIST_T_FREE);

		if (++nfree == LOCAL_FREE_TARGET)

		if (++nfree == lru->target_free)

			break;

	}



	if (nfree < LOCAL_FREE_TARGET)

		__bpf_lru_list_shrink(lru, l, LOCAL_FREE_TARGET - nfree,

	if (nfree < lru->target_free)

		__bpf_lru_list_shrink(lru, l, lru->target_free - nfree,

				      local_free_list(loc_l),

				      BPF_LRU_LOCAL_LIST_T_FREE);
@@ -577,6 +577,9 @@ static void bpf_common_lru_populate(struct bpf_lru *lru, void *buf, 
		list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]);

		buf += elem_size;

	}



	lru->target_free = clamp((nr_elems / num_possible_cpus()) / 2,

				 1, LOCAL_FREE_TARGET);

}



static void bpf_percpu_lru_populate(struct bpf_lru *lru, void *buf,

kernel/bpf/bpf_lru_list.h

+1 −0
Original line number Diff line number Diff line
@@ -58,6 +58,7 @@ struct bpf_lru { 
	del_from_htab_func del_from_htab;

	void *del_arg;

	unsigned int hash_offset;

	unsigned int target_free;

	unsigned int nr_scans;

	bool percpu;

};

kernel/bpf/cgroup.c

+1 −1
Original line number Diff line number Diff line
@@ -2134,7 +2134,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { 
	.gpl_only	= false,

	.ret_type	= RET_INTEGER,

	.arg1_type	= ARG_PTR_TO_CTX,

	.arg2_type	= ARG_PTR_TO_MEM,

	.arg2_type	= ARG_PTR_TO_MEM | MEM_WRITE,

	.arg3_type	= ARG_CONST_SIZE,

	.arg4_type	= ARG_ANYTHING,

};