Commit 8b72f5a9 authored by Heiko Carstens's avatar Heiko Carstens
Browse files

s390/mm: Reimplement lazy ASCE handling 



Reduce system call overhead time (round trip time for invoking a
non-existent system call) by 25%.

With the removal of set_fs() [1] lazy control register handling was removed
in order to keep kernel entry and exit simple. However this made system
calls slower.

With the conversion to generic entry [2] and numerous follow up changes
which simplified the entry code significantly, adding support for lazy asce
handling doesn't add much complexity to the entry code anymore.

In particular this means:

- On kernel entry the primary asce is not modified and contains the user
  asce

- Kernel accesses which require secondary-space mode (for example futex
  operations) are surrounded by enable_sacf_uaccess() and
  disable_sacf_uaccess() calls. enable_sacf_uaccess() sets the primary asce
  to kernel asce so that the sacf instruction can be used to switch to
  secondary-space mode. The primary asce is changed back to user asce with
  disable_sacf_uaccess().

The state of the control register which contains the primary asce is
reflected with a new TIF_ASCE_PRIMARY bit. This is required on context
switch so that the correct asce is restored for the scheduled in process.

In result address spaces are now setup like this:

CPU running in               | %cr1 ASCE | %cr7 ASCE | %cr13 ASCE
-----------------------------|-----------|-----------|-----------
user space                   |  user     |  user     |  kernel
kernel (no sacf)             |  user     |  user     |  kernel
kernel (during sacf uaccess) |  kernel   |  user     |  kernel
kernel (kvm guest execution) |  guest    |  user     |  kernel

In result cr1 control register content is not changed except for:
- futex system calls
- legacy s390 PCI system calls
- the kvm specific cmpxchg_user_key() uaccess helper

This leads to faster system call execution.

[1] 87d59863 ("s390/mm: remove set_fs / rework address space handling")
[2] 56e62a73 ("s390: convert to generic entry")

Reviewed-by: default avatarAlexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
parent 8ffd015d

arch/s390/include/asm/asce.h

0 → 100644
+36 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */



#ifndef _ASM_S390_ASCE_H

#define _ASM_S390_ASCE_H



#include <linux/thread_info.h>

#include <linux/irqflags.h>

#include <asm/lowcore.h>

#include <asm/ctlreg.h>



static inline bool enable_sacf_uaccess(void)

{

	unsigned long flags;



	if (test_thread_flag(TIF_ASCE_PRIMARY))

		return true;

	local_irq_save(flags);

	local_ctl_load(1, &get_lowcore()->kernel_asce);

	set_thread_flag(TIF_ASCE_PRIMARY);

	local_irq_restore(flags);

	return false;

}



static inline void disable_sacf_uaccess(bool previous)

{

	unsigned long flags;



	if (previous)

		return;

	local_irq_save(flags);

	local_ctl_load(1, &get_lowcore()->user_asce);

	clear_thread_flag(TIF_ASCE_PRIMARY);

	local_irq_restore(flags);

}



#endif /* _ASM_S390_ASCE_H */

arch/s390/include/asm/futex.h

+6 −0
Original line number Diff line number Diff line
@@ -13,9 +13,11 @@ 
static uaccess_kmsan_or_inline int						\

__futex_atomic_##name(int oparg, int *old, u32 __user *uaddr)			\

{										\

	bool sacf_flag;								\

	int rc, new;								\

										\

	instrument_copy_from_user_before(old, uaddr, sizeof(*old));		\

	sacf_flag = enable_sacf_uaccess();					\

	asm_inline volatile(							\

		"	sacf	256\n"						\

		"0:	l	%[old],%[uaddr]\n"				\
@@ -32,6 +34,7 @@ __futex_atomic_##name(int oparg, int *old, u32 __user *uaddr) \ 
		  [new] "=&d" (new), [uaddr] "+Q" (*uaddr)			\

		: [oparg] "d" (oparg)						\

		: "cc");							\

	disable_sacf_uaccess(sacf_flag);					\

	if (!rc)								\

		instrument_copy_from_user_after(old, uaddr, sizeof(*old), 0);	\

	return rc;								\
@@ -75,9 +78,11 @@ int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) 
static uaccess_kmsan_or_inline

int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval)

{

	bool sacf_flag;

	int rc;



	instrument_copy_from_user_before(uval, uaddr, sizeof(*uval));

	sacf_flag = enable_sacf_uaccess();

	asm_inline volatile(

		"	sacf	256\n"

		"0:	cs	%[old],%[new],%[uaddr]\n"
@@ -88,6 +93,7 @@ int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 
		: [rc] "=d" (rc), [old] "+d" (oldval), [uaddr] "+Q" (*uaddr)

		: [new] "d" (newval)

		: "cc", "memory");

	disable_sacf_uaccess(sacf_flag);

	*uval = oldval;

	instrument_copy_from_user_after(uval, uaddr, sizeof(*uval), 0);

	return rc;

arch/s390/include/asm/mmu_context.h

+18 −1
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include <linux/mm_types.h>

#include <asm/tlbflush.h>

#include <asm/ctlreg.h>

#include <asm/asce.h>

#include <asm-generic/mm_hooks.h>



#define init_new_context init_new_context
@@ -77,7 +78,8 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct * 
	else

		get_lowcore()->user_asce.val = next->context.asce;

	cpumask_set_cpu(cpu, &next->context.cpu_attach_mask);

	/* Clear previous user-ASCE from CR7 */

	/* Clear previous user-ASCE from CR1 and CR7 */

	local_ctl_load(1, &s390_invalid_asce);

	local_ctl_load(7, &s390_invalid_asce);

	if (prev != next)

		cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask);
@@ -99,6 +101,7 @@ static inline void finish_arch_post_lock_switch(void) 
{

	struct task_struct *tsk = current;

	struct mm_struct *mm = tsk->mm;

	unsigned long flags;



	if (mm) {

		preempt_disable();
@@ -108,16 +111,30 @@ static inline void finish_arch_post_lock_switch(void) 
		__tlb_flush_mm_lazy(mm);

		preempt_enable();

	}

	local_irq_save(flags);

	if (test_thread_flag(TIF_ASCE_PRIMARY))

		local_ctl_load(1, &get_lowcore()->kernel_asce);

	else

		local_ctl_load(1, &get_lowcore()->user_asce);

	local_ctl_load(7, &get_lowcore()->user_asce);

	local_irq_restore(flags);

}



#define activate_mm activate_mm

static inline void activate_mm(struct mm_struct *prev,

                               struct mm_struct *next)

{

	unsigned long flags;



	switch_mm(prev, next, current);

	cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));

	local_irq_save(flags);

	if (test_thread_flag(TIF_ASCE_PRIMARY))

		local_ctl_load(1, &get_lowcore()->kernel_asce);

	else

		local_ctl_load(1, &get_lowcore()->user_asce);

	local_ctl_load(7, &get_lowcore()->user_asce);

	local_irq_restore(flags);

}



#include <asm-generic/mmu_context.h>

arch/s390/include/asm/ptrace.h

+0 −1
Original line number Diff line number Diff line
@@ -126,7 +126,6 @@ struct pt_regs { 
		struct tpi_info tpi_info;

	};

	unsigned long flags;

	unsigned long cr1;

	unsigned long last_break;

};

arch/s390/include/asm/thread_info.h

+2 −0
Original line number Diff line number Diff line
@@ -64,6 +64,7 @@ void arch_setup_new_exec(void); 
#define TIF_NEED_RESCHED_LAZY	3	/* lazy rescheduling needed */

#define TIF_UPROBE		4	/* breakpointed or single-stepping */

#define TIF_PATCH_PENDING	5	/* pending live patching update */

#define TIF_ASCE_PRIMARY	6	/* primary asce is kernel asce */

#define TIF_NOTIFY_SIGNAL	7	/* signal notifications exist */

#define TIF_GUARDED_STORAGE	8	/* load guarded storage control block */

#define TIF_ISOLATE_BP_GUEST	9	/* Run KVM guests with isolated BP */
@@ -85,6 +86,7 @@ void arch_setup_new_exec(void); 
#define _TIF_NEED_RESCHED_LAZY	BIT(TIF_NEED_RESCHED_LAZY)

#define _TIF_UPROBE		BIT(TIF_UPROBE)

#define _TIF_PATCH_PENDING	BIT(TIF_PATCH_PENDING)

#define _TIF_ASCE_PRIMARY	BIT(TIF_ASCE_PRIMARY)

#define _TIF_NOTIFY_SIGNAL	BIT(TIF_NOTIFY_SIGNAL)

#define _TIF_GUARDED_STORAGE	BIT(TIF_GUARDED_STORAGE)

#define _TIF_ISOLATE_BP_GUEST	BIT(TIF_ISOLATE_BP_GUEST)