Merge tag 'probes-v6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace

F:	lib/objagg.c

F:	lib/test_objagg.c

OBJPOOL

M:	Matt Wu <wuqiang.matt@bytedance.com>

S:	Supported

F:	include/linux/objpool.h

F:	lib/objpool.c

F:	lib/test_objpool.c

OBJTOOL

M:	Josh Poimboeuf <jpoimboe@kernel.org>

M:	Peter Zijlstra <peterz@infradead.org>

/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */

#ifndef FREELIST_H

#define FREELIST_H

#include <linux/atomic.h>

/*

 * Copyright: cameron@moodycamel.com

 *

 * A simple CAS-based lock-free free list. Not the fastest thing in the world

 * under heavy contention, but simple and correct (assuming nodes are never

 * freed until after the free list is destroyed), and fairly speedy under low

 * contention.

 *

 * Adapted from: https://moodycamel.com/blog/2014/solving-the-aba-problem-for-lock-free-free-lists

 */

struct freelist_node {

	atomic_t		refs;

	struct freelist_node	*next;

};

struct freelist_head {

	struct freelist_node	*head;

};

#define REFS_ON_FREELIST 0x80000000

#define REFS_MASK	 0x7FFFFFFF

static inline void __freelist_add(struct freelist_node *node, struct freelist_head *list)

{

	/*

	 * Since the refcount is zero, and nobody can increase it once it's

	 * zero (except us, and we run only one copy of this method per node at

	 * a time, i.e. the single thread case), then we know we can safely

	 * change the next pointer of the node; however, once the refcount is

	 * back above zero, then other threads could increase it (happens under

	 * heavy contention, when the refcount goes to zero in between a load

	 * and a refcount increment of a node in try_get, then back up to

	 * something non-zero, then the refcount increment is done by the other

	 * thread) -- so if the CAS to add the node to the actual list fails,

	 * decrese the refcount and leave the add operation to the next thread

	 * who puts the refcount back to zero (which could be us, hence the

	 * loop).

	 */

	struct freelist_node *head = READ_ONCE(list->head);

	for (;;) {

		WRITE_ONCE(node->next, head);

		atomic_set_release(&node->refs, 1);

		if (!try_cmpxchg_release(&list->head, &head, node)) {

			/*

			 * Hmm, the add failed, but we can only try again when

			 * the refcount goes back to zero.

			 */

			if (atomic_fetch_add_release(REFS_ON_FREELIST - 1, &node->refs) == 1)

				continue;

		}

		return;

	}

}

static inline void freelist_add(struct freelist_node *node, struct freelist_head *list)

{

	/*

	 * We know that the should-be-on-freelist bit is 0 at this point, so

	 * it's safe to set it using a fetch_add.

	 */

	if (!atomic_fetch_add_release(REFS_ON_FREELIST, &node->refs)) {

		/*

		 * Oh look! We were the last ones referencing this node, and we

		 * know we want to add it to the free list, so let's do it!

		 */

		__freelist_add(node, list);

	}

}

static inline struct freelist_node *freelist_try_get(struct freelist_head *list)

{

	struct freelist_node *prev, *next, *head = smp_load_acquire(&list->head);

	unsigned int refs;

	while (head) {

		prev = head;

		refs = atomic_read(&head->refs);

		if ((refs & REFS_MASK) == 0 ||

		    !atomic_try_cmpxchg_acquire(&head->refs, &refs, refs+1)) {

			head = smp_load_acquire(&list->head);

			continue;

		}

		/*

		 * Good, reference count has been incremented (it wasn't at

		 * zero), which means we can read the next and not worry about

		 * it changing between now and the time we do the CAS.

		 */

		next = READ_ONCE(head->next);

		if (try_cmpxchg_acquire(&list->head, &head, next)) {

			/*

			 * Yay, got the node. This means it was on the list,

			 * which means should-be-on-freelist must be false no

			 * matter the refcount (because nobody else knows it's

			 * been taken off yet, it can't have been put back on).

			 */

			WARN_ON_ONCE(atomic_read(&head->refs) & REFS_ON_FREELIST);

			/*

			 * Decrease refcount twice, once for our ref, and once

			 * for the list's ref.

			 */

			atomic_fetch_add(-2, &head->refs);

			return head;

		}

		/*

		 * OK, the head must have changed on us, but we still need to decrement

		 * the refcount we increased.

		 */

		refs = atomic_fetch_add(-1, &prev->refs);

		if (refs == REFS_ON_FREELIST + 1)

			__freelist_add(prev, list);

	}

	return NULL;

}

#endif /* FREELIST_H */

#include <linux/rcupdate.h>

#include <linux/mutex.h>

#include <linux/ftrace.h>

#include <linux/refcount.h>

#include <linux/freelist.h>

#include <linux/objpool.h>

#include <linux/rethook.h>

#include <asm/kprobes.h>

 */

struct kretprobe_holder {

	struct kretprobe	*rp;

	refcount_t		ref;

	struct objpool_head	pool;

};

struct kretprobe {

#ifdef CONFIG_KRETPROBE_ON_RETHOOK

	struct rethook *rh;

#else

	struct freelist_head freelist;

	struct kretprobe_holder *rph;

#endif

};

#ifdef CONFIG_KRETPROBE_ON_RETHOOK

	struct rethook_node node;

#else

	union {

		struct freelist_node freelist;

	struct rcu_head rcu;

	};

	struct llist_node llist;

	struct kretprobe_holder *rph;

	kprobe_opcode_t *ret_addr;

/* SPDX-License-Identifier: GPL-2.0 */

#ifndef _LINUX_OBJPOOL_H

#define _LINUX_OBJPOOL_H

#include <linux/types.h>

#include <linux/refcount.h>

/*

 * objpool: ring-array based lockless MPMC queue

 *

 * Copyright: wuqiang.matt@bytedance.com,mhiramat@kernel.org

 *

 * objpool is a scalable implementation of high performance queue for

 * object allocation and reclamation, such as kretprobe instances.

 *

 * With leveraging percpu ring-array to mitigate hot spots of memory

 * contention, it delivers near-linear scalability for high parallel

 * scenarios. The objpool is best suited for the following cases:

 * 1) Memory allocation or reclamation are prohibited or too expensive

 * 2) Consumers are of different priorities, such as irqs and threads

 *

 * Limitations:

 * 1) Maximum objects (capacity) is fixed after objpool creation

 * 2) All pre-allocated objects are managed in percpu ring array,

 *    which consumes more memory than linked lists

 */

/**

 * struct objpool_slot - percpu ring array of objpool

 * @head: head sequence of the local ring array (to retrieve at)

 * @tail: tail sequence of the local ring array (to append at)

 * @last: the last sequence number marked as ready for retrieve

 * @mask: bits mask for modulo capacity to compute array indexes

 * @entries: object entries on this slot

 *

 * Represents a cpu-local array-based ring buffer, its size is specialized

 * during initialization of object pool. The percpu objpool node is to be

 * allocated from local memory for NUMA system, and to be kept compact in

 * continuous memory: CPU assigned number of objects are stored just after

 * the body of objpool_node.

 *

 * Real size of the ring array is far too smaller than the value range of

 * head and tail, typed as uint32_t: [0, 2^32), so only lower bits (mask)

 * of head and tail are used as the actual position in the ring array. In

 * general the ring array is acting like a small sliding window, which is

 * always moving forward in the loop of [0, 2^32).

 */

struct objpool_slot {

	uint32_t            head;

	uint32_t            tail;

	uint32_t            last;

	uint32_t            mask;

	void               *entries[];

} __packed;

struct objpool_head;

/*

 * caller-specified callback for object initial setup, it's only called

 * once for each object (just after the memory allocation of the object)

 */

typedef int (*objpool_init_obj_cb)(void *obj, void *context);

/* caller-specified cleanup callback for objpool destruction */

typedef int (*objpool_fini_cb)(struct objpool_head *head, void *context);

/**

 * struct objpool_head - object pooling metadata

 * @obj_size:   object size, aligned to sizeof(void *)

 * @nr_objs:    total objs (to be pre-allocated with objpool)

 * @nr_cpus:    local copy of nr_cpu_ids

 * @capacity:   max objs can be managed by one objpool_slot

 * @gfp:        gfp flags for kmalloc & vmalloc

 * @ref:        refcount of objpool

 * @flags:      flags for objpool management

 * @cpu_slots:  pointer to the array of objpool_slot

 * @release:    resource cleanup callback

 * @context:    caller-provided context

 */

struct objpool_head {

	int                     obj_size;

	int                     nr_objs;

	int                     nr_cpus;

	int                     capacity;

	gfp_t                   gfp;

	refcount_t              ref;

	unsigned long           flags;

	struct objpool_slot   **cpu_slots;

	objpool_fini_cb         release;

	void                   *context;

};

#define OBJPOOL_NR_OBJECT_MAX	(1UL << 24) /* maximum numbers of total objects */

#define OBJPOOL_OBJECT_SIZE_MAX	(1UL << 16) /* maximum size of an object */

/**

 * objpool_init() - initialize objpool and pre-allocated objects

 * @pool:    the object pool to be initialized, declared by caller

 * @nr_objs: total objects to be pre-allocated by this object pool

 * @object_size: size of an object (should be > 0)

 * @gfp:     flags for memory allocation (via kmalloc or vmalloc)

 * @context: user context for object initialization callback

 * @objinit: object initialization callback for extra setup

 * @release: cleanup callback for extra cleanup task

 *

 * return value: 0 for success, otherwise error code

 *

 * All pre-allocated objects are to be zeroed after memory allocation.

 * Caller could do extra initialization in objinit callback. objinit()

 * will be called just after slot allocation and called only once for

 * each object. After that the objpool won't touch any content of the

 * objects. It's caller's duty to perform reinitialization after each

 * pop (object allocation) or do clearance before each push (object

 * reclamation).

 */

int objpool_init(struct objpool_head *pool, int nr_objs, int object_size,

		 gfp_t gfp, void *context, objpool_init_obj_cb objinit,

		 objpool_fini_cb release);

/**

 * objpool_pop() - allocate an object from objpool

 * @pool: object pool

 *

 * return value: object ptr or NULL if failed

 */

void *objpool_pop(struct objpool_head *pool);

/**

 * objpool_push() - reclaim the object and return back to objpool

 * @obj:  object ptr to be pushed to objpool

 * @pool: object pool

 *

 * return: 0 or error code (it fails only when user tries to push

 * the same object multiple times or wrong "objects" into objpool)

 */

int objpool_push(void *obj, struct objpool_head *pool);

/**

 * objpool_drop() - discard the object and deref objpool

 * @obj:  object ptr to be discarded

 * @pool: object pool

 *

 * return: 0 if objpool was released; -EAGAIN if there are still

 *         outstanding objects

 *

 * objpool_drop is normally for the release of outstanding objects

 * after objpool cleanup (objpool_fini). Thinking of this example:

 * kretprobe is unregistered and objpool_fini() is called to release

 * all remained objects, but there are still objects being used by

 * unfinished kretprobes (like blockable function: sys_accept). So

 * only when the last outstanding object is dropped could the whole

 * objpool be released along with the call of objpool_drop()

 */

int objpool_drop(void *obj, struct objpool_head *pool);

/**

 * objpool_free() - release objpool forcely (all objects to be freed)

 * @pool: object pool to be released

 */

void objpool_free(struct objpool_head *pool);

/**

 * objpool_fini() - deref object pool (also releasing unused objects)

 * @pool: object pool to be dereferenced

 *

 * objpool_fini() will try to release all remained free objects and

 * then drop an extra reference of the objpool. If all objects are

 * already returned to objpool (so called synchronous use cases),

 * the objpool itself will be freed together. But if there are still

 * outstanding objects (so called asynchronous use cases, such like

 * blockable kretprobe), the objpool won't be released until all

 * the outstanding objects are dropped, but the caller must assure

 * there are no concurrent objpool_push() on the fly. Normally RCU

 * is being required to make sure all ongoing objpool_push() must

 * be finished before calling objpool_fini(), so does test_objpool,

 * kretprobe or rethook

 */

void objpool_fini(struct objpool_head *pool);

#endif /* _LINUX_OBJPOOL_H */

#define _LINUX_RETHOOK_H

#include <linux/compiler.h>

#include <linux/freelist.h>

#include <linux/objpool.h>

#include <linux/kallsyms.h>

#include <linux/llist.h>

#include <linux/rcupdate.h>

#include <linux/refcount.h>

struct rethook_node;

struct rethook {

	void			*data;

	rethook_handler_t	handler;

	struct freelist_head	pool;

	refcount_t		ref;

	struct objpool_head	pool;

	struct rcu_head		rcu;

};

/**

 * struct rethook_node - The rethook shadow-stack entry node.

 * @freelist: The freelist, linked to struct rethook::pool.

 * @rcu: The rcu_head for deferred freeing.

 * @llist: The llist, linked to a struct task_struct::rethooks.

 * @rethook: The pointer to the struct rethook.

 * on each entry of the shadow stack.

 */

struct rethook_node {

	union {

		struct freelist_node freelist;

	struct rcu_head		rcu;

	};

	struct llist_node	llist;

	struct rethook		*rethook;

	unsigned long		ret_addr;

	unsigned long		frame;

};

struct rethook *rethook_alloc(void *data, rethook_handler_t handler);

struct rethook *rethook_alloc(void *data, rethook_handler_t handler, int size, int num);

void rethook_stop(struct rethook *rh);

void rethook_free(struct rethook *rh);

void rethook_add_node(struct rethook *rh, struct rethook_node *node);

struct rethook_node *rethook_try_get(struct rethook *rh);

void rethook_recycle(struct rethook_node *node);

void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount);

#endif

#endif