Commit 56882250 authored by Lorenzo Stoakes's avatar Lorenzo Stoakes Committed by Andrew Morton
Browse files

mm: add atomic VMA flags and set VM_MAYBE_GUARD as such 

This patch adds the ability to atomically set VMA flags with only the mmap
read/VMA read lock held.

As this could be hugely problematic for VMA flags in general given that
all other accesses are non-atomic and serialised by the mmap/VMA locks, we
implement this with a strict allow-list - that is, only designated flags
are allowed to do this.

We make VM_MAYBE_GUARD one of these flags.

Link: https://lkml.kernel.org/r/97e57abed09f2663077ed7a36fb8206e243171a9.1763460113.git.lorenzo.stoakes@oracle.com


Signed-off-by: default avatarLorenzo Stoakes <lorenzo.stoakes@oracle.com>
Reviewed-by: default avatarPedro Falcato <pfalcato@suse.de>
Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarDavid Hildenbrand (Red Hat) <david@kernel.org>
Reviewed-by: default avatarLance Yang <lance.yang@linux.dev>
Cc: Andrei Vagin <avagin@gmail.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Jann Horn <jannh@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: "Masami Hiramatsu (Google)" <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Nico Pache <npache@redhat.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 5dba5cc2

include/linux/mm.h

+44 −0
Original line number Diff line number Diff line
@@ -518,6 +518,9 @@ extern unsigned int kobjsize(const void *objp); 
/* This mask represents all the VMA flag bits used by mlock */

#define VM_LOCKED_MASK	(VM_LOCKED | VM_LOCKONFAULT)



/* These flags can be updated atomically via VMA/mmap read lock. */

#define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD



/* Arch-specific flags to clear when updating VM flags on protection change */

#ifndef VM_ARCH_CLEAR

# define VM_ARCH_CLEAR	VM_NONE
@@ -860,6 +863,47 @@ static inline void vm_flags_mod(struct vm_area_struct *vma, 
	__vm_flags_mod(vma, set, clear);

}



static inline bool __vma_flag_atomic_valid(struct vm_area_struct *vma,

				       int bit)

{

	const vm_flags_t mask = BIT(bit);



	/* Only specific flags are permitted */

	if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED)))

		return false;



	return true;

}



/*

 * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific

 * valid flags are allowed to do this.

 */

static inline void vma_flag_set_atomic(struct vm_area_struct *vma, int bit)

{

	/* mmap read lock/VMA read lock must be held. */

	if (!rwsem_is_locked(&vma->vm_mm->mmap_lock))

		vma_assert_locked(vma);



	if (__vma_flag_atomic_valid(vma, bit))

		set_bit(bit, &ACCESS_PRIVATE(vma, __vm_flags));

}



/*

 * Test for VMA flag atomically. Requires no locks. Only specific valid flags

 * are allowed to do this.

 *

 * This is necessarily racey, so callers must ensure that serialisation is

 * achieved through some other means, or that races are permissible.

 */

static inline bool vma_flag_test_atomic(struct vm_area_struct *vma, int bit)

{

	if (__vma_flag_atomic_valid(vma, bit))

		return test_bit(bit, &vma->vm_flags);



	return false;

}



static inline void vma_set_anonymous(struct vm_area_struct *vma)

{

	vma->vm_ops = NULL;