Commit e3b78a7a authored by David Kaplan's avatar David Kaplan Committed by Borislav Petkov (AMD)
Browse files

x86/bugs: Restructure retbleed mitigation 



Restructure retbleed mitigation to use select/update/apply functions to create
consistent vulnerability handling.  The retbleed_update_mitigation()
simplifies the dependency between spectre_v2 and retbleed.

The command line options now directly select a preferred mitigation
which simplifies the logic.

Signed-off-by: default avatarDavid Kaplan <david.kaplan@amd.com>
Signed-off-by: default avatarBorislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: default avatarJosh Poimboeuf <jpoimboe@kernel.org>
Link: https://lore.kernel.org/20250418161721.1855190-11-david.kaplan@amd.com
parent 83d4b193

arch/x86/kernel/cpu/bugs.c

+98 −97
Original line number Diff line number Diff line
@@ -57,6 +57,8 @@ static void __init spectre_v1_select_mitigation(void); 
static void __init spectre_v1_apply_mitigation(void);

static void __init spectre_v2_select_mitigation(void);

static void __init retbleed_select_mitigation(void);

static void __init retbleed_update_mitigation(void);

static void __init retbleed_apply_mitigation(void);

static void __init spectre_v2_user_select_mitigation(void);

static void __init ssb_select_mitigation(void);

static void __init l1tf_select_mitigation(void);
@@ -187,11 +189,6 @@ void __init cpu_select_mitigations(void) 
	/* Select the proper CPU mitigations before patching alternatives: */

	spectre_v1_select_mitigation();

	spectre_v2_select_mitigation();

	/*

	 * retbleed_select_mitigation() relies on the state set by

	 * spectre_v2_select_mitigation(); specifically it wants to know about

	 * spectre_v2=ibrs.

	 */

	retbleed_select_mitigation();

	/*

	 * spectre_v2_user_select_mitigation() relies on the state set by
@@ -219,12 +216,14 @@ void __init cpu_select_mitigations(void) 
	 * After mitigations are selected, some may need to update their

	 * choices.

	 */

	retbleed_update_mitigation();

	mds_update_mitigation();

	taa_update_mitigation();

	mmio_update_mitigation();

	rfds_update_mitigation();



	spectre_v1_apply_mitigation();

	retbleed_apply_mitigation();

	mds_apply_mitigation();

	taa_apply_mitigation();

	mmio_apply_mitigation();
@@ -1085,6 +1084,7 @@ enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 


enum retbleed_mitigation {

	RETBLEED_MITIGATION_NONE,

	RETBLEED_MITIGATION_AUTO,

	RETBLEED_MITIGATION_UNRET,

	RETBLEED_MITIGATION_IBPB,

	RETBLEED_MITIGATION_IBRS,
@@ -1092,14 +1092,6 @@ enum retbleed_mitigation { 
	RETBLEED_MITIGATION_STUFF,

};



enum retbleed_mitigation_cmd {

	RETBLEED_CMD_OFF,

	RETBLEED_CMD_AUTO,

	RETBLEED_CMD_UNRET,

	RETBLEED_CMD_IBPB,

	RETBLEED_CMD_STUFF,

};



static const char * const retbleed_strings[] = {

	[RETBLEED_MITIGATION_NONE]	= "Vulnerable",

	[RETBLEED_MITIGATION_UNRET]	= "Mitigation: untrained return thunk",
@@ -1110,9 +1102,7 @@ static const char * const retbleed_strings[] = { 
};



static enum retbleed_mitigation retbleed_mitigation __ro_after_init =

	RETBLEED_MITIGATION_NONE;

static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init =

	IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_CMD_AUTO : RETBLEED_CMD_OFF;

	IS_ENABLED(CONFIG_MITIGATION_RETBLEED) ? RETBLEED_MITIGATION_AUTO : RETBLEED_MITIGATION_NONE;



static int __ro_after_init retbleed_nosmt = false;
@@ -1129,15 +1119,15 @@ static int __init retbleed_parse_cmdline(char *str) 
		}



		if (!strcmp(str, "off")) {

			retbleed_cmd = RETBLEED_CMD_OFF;

			retbleed_mitigation = RETBLEED_MITIGATION_NONE;

		} else if (!strcmp(str, "auto")) {

			retbleed_cmd = RETBLEED_CMD_AUTO;

			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;

		} else if (!strcmp(str, "unret")) {

			retbleed_cmd = RETBLEED_CMD_UNRET;

			retbleed_mitigation = RETBLEED_MITIGATION_UNRET;

		} else if (!strcmp(str, "ibpb")) {

			retbleed_cmd = RETBLEED_CMD_IBPB;

			retbleed_mitigation = RETBLEED_MITIGATION_IBPB;

		} else if (!strcmp(str, "stuff")) {

			retbleed_cmd = RETBLEED_CMD_STUFF;

			retbleed_mitigation = RETBLEED_MITIGATION_STUFF;

		} else if (!strcmp(str, "nosmt")) {

			retbleed_nosmt = true;

		} else if (!strcmp(str, "force")) {
@@ -1158,57 +1148,44 @@ early_param("retbleed", retbleed_parse_cmdline); 


static void __init retbleed_select_mitigation(void)

{

	bool mitigate_smt = false;



	if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())

		return;



	switch (retbleed_cmd) {

	case RETBLEED_CMD_OFF:

	if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) {

		retbleed_mitigation = RETBLEED_MITIGATION_NONE;

		return;

	}



	case RETBLEED_CMD_UNRET:

		if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) {

			retbleed_mitigation = RETBLEED_MITIGATION_UNRET;

		} else {

	switch (retbleed_mitigation) {

	case RETBLEED_MITIGATION_UNRET:

		if (!IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) {

			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;

			pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n");

			goto do_cmd_auto;

		}

		break;



	case RETBLEED_CMD_IBPB:

	case RETBLEED_MITIGATION_IBPB:

		if (!boot_cpu_has(X86_FEATURE_IBPB)) {

			pr_err("WARNING: CPU does not support IBPB.\n");

			goto do_cmd_auto;

		} else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {

			retbleed_mitigation = RETBLEED_MITIGATION_IBPB;

		} else {

			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;

		} else if (!IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {

			pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");

			goto do_cmd_auto;

			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;

		}

		break;



	case RETBLEED_CMD_STUFF:

		if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) &&

		    spectre_v2_enabled == SPECTRE_V2_RETPOLINE) {

			if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {

				pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n");

				goto do_cmd_auto;

			}

			retbleed_mitigation = RETBLEED_MITIGATION_STUFF;



		} else {

			if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING))

				pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n");

			else

	case RETBLEED_MITIGATION_STUFF:

		if (!IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)) {

			pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n");



			goto do_cmd_auto;

			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;

		} else if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {

			pr_err("WARNING: retbleed=stuff only supported for Intel CPUs.\n");

			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;

		}

		break;

	default:

		break;

	}



	if (retbleed_mitigation != RETBLEED_MITIGATION_AUTO)

		return;



do_cmd_auto:

	case RETBLEED_CMD_AUTO:

	/* Intel mitigation selected in retbleed_update_mitigation() */

	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||

	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {

		if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY))
@@ -1216,18 +1193,64 @@ static void __init retbleed_select_mitigation(void) 
		else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) &&

			 boot_cpu_has(X86_FEATURE_IBPB))

			retbleed_mitigation = RETBLEED_MITIGATION_IBPB;

		else

			retbleed_mitigation = RETBLEED_MITIGATION_NONE;

	}

}



static void __init retbleed_update_mitigation(void)

{

	if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())

		return;



	if (retbleed_mitigation == RETBLEED_MITIGATION_NONE)

		goto out;



	/*

		 * The Intel mitigation (IBRS or eIBRS) was already selected in

		 * spectre_v2_select_mitigation().  'retbleed_mitigation' will

		 * be set accordingly below.

	 * retbleed=stuff is only allowed on Intel.  If stuffing can't be used

	 * then a different mitigation will be selected below.

	 */



	if (retbleed_mitigation == RETBLEED_MITIGATION_STUFF) {

		if (spectre_v2_enabled != SPECTRE_V2_RETPOLINE) {

			pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n");

			retbleed_mitigation = RETBLEED_MITIGATION_AUTO;

		}

	}

	/*

	 * Let IBRS trump all on Intel without affecting the effects of the

	 * retbleed= cmdline option except for call depth based stuffing

	 */

	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {

		switch (spectre_v2_enabled) {

		case SPECTRE_V2_IBRS:

			retbleed_mitigation = RETBLEED_MITIGATION_IBRS;

			break;

		case SPECTRE_V2_EIBRS:

		case SPECTRE_V2_EIBRS_RETPOLINE:

		case SPECTRE_V2_EIBRS_LFENCE:

			retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;

			break;

		default:

			if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF)

				pr_err(RETBLEED_INTEL_MSG);

		}

		/* If nothing has set the mitigation yet, default to NONE. */

		if (retbleed_mitigation == RETBLEED_MITIGATION_AUTO)

			retbleed_mitigation = RETBLEED_MITIGATION_NONE;

	}

out:

	pr_info("%s\n", retbleed_strings[retbleed_mitigation]);

}





static void __init retbleed_apply_mitigation(void)

{

	bool mitigate_smt = false;



	switch (retbleed_mitigation) {

	case RETBLEED_MITIGATION_NONE:

		return;



	case RETBLEED_MITIGATION_UNRET:

		setup_force_cpu_cap(X86_FEATURE_RETHUNK);

		setup_force_cpu_cap(X86_FEATURE_UNRET);
@@ -1277,28 +1300,6 @@ static void __init retbleed_select_mitigation(void) 
	if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&

	    (retbleed_nosmt || cpu_mitigations_auto_nosmt()))

		cpu_smt_disable(false);



	/*

	 * Let IBRS trump all on Intel without affecting the effects of the

	 * retbleed= cmdline option except for call depth based stuffing

	 */

	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {

		switch (spectre_v2_enabled) {

		case SPECTRE_V2_IBRS:

			retbleed_mitigation = RETBLEED_MITIGATION_IBRS;

			break;

		case SPECTRE_V2_EIBRS:

		case SPECTRE_V2_EIBRS_RETPOLINE:

		case SPECTRE_V2_EIBRS_LFENCE:

			retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;

			break;

		default:

			if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF)

				pr_err(RETBLEED_INTEL_MSG);

		}

	}



	pr_info("%s\n", retbleed_strings[retbleed_mitigation]);

}



#undef pr_fmt
@@ -1855,8 +1856,8 @@ static void __init spectre_v2_select_mitigation(void) 


		if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) &&

		    boot_cpu_has_bug(X86_BUG_RETBLEED) &&

		    retbleed_cmd != RETBLEED_CMD_OFF &&

		    retbleed_cmd != RETBLEED_CMD_STUFF &&

		    retbleed_mitigation != RETBLEED_MITIGATION_NONE &&

		    retbleed_mitigation != RETBLEED_MITIGATION_STUFF &&

		    boot_cpu_has(X86_FEATURE_IBRS) &&

		    boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {

			mode = SPECTRE_V2_IBRS;
@@ -1964,7 +1965,7 @@ static void __init spectre_v2_select_mitigation(void) 
	    (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||

	     boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) {



		if (retbleed_cmd != RETBLEED_CMD_IBPB) {

		if (retbleed_mitigation != RETBLEED_MITIGATION_IBPB) {

			setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW);

			pr_info("Enabling Speculation Barrier for firmware calls\n");

		}