Loading arch/arm64/include/asm/arch_gicv3.h +7 −1 Original line number Diff line number Diff line Loading @@ -153,7 +153,7 @@ static inline bool gic_prio_masking_enabled(void) static inline void gic_pmr_mask_irqs(void) { BUILD_BUG_ON(GICD_INT_DEF_PRI < (GIC_PRIO_IRQOFF | BUILD_BUG_ON(GICD_INT_DEF_PRI < (__GIC_PRIO_IRQOFF | GIC_PRIO_PSR_I_SET)); BUILD_BUG_ON(GICD_INT_DEF_PRI >= GIC_PRIO_IRQON); /* Loading @@ -162,6 +162,12 @@ static inline void gic_pmr_mask_irqs(void) * are applied to IRQ priorities */ BUILD_BUG_ON((0x80 | (GICD_INT_DEF_PRI >> 1)) >= GIC_PRIO_IRQON); /* * Same situation as above, but now we make sure that we can mask * regular interrupts. */ BUILD_BUG_ON((0x80 | (GICD_INT_DEF_PRI >> 1)) < (__GIC_PRIO_IRQOFF_NS | GIC_PRIO_PSR_I_SET)); gic_write_pmr(GIC_PRIO_IRQOFF); } Loading arch/arm64/include/asm/ptrace.h +13 −1 Original line number Diff line number Diff line Loading @@ -31,9 +31,21 @@ * interrupt disabling temporarily does not rely on IRQ priorities. */ #define GIC_PRIO_IRQON 0xe0 #define GIC_PRIO_IRQOFF (GIC_PRIO_IRQON & ~0x80) #define __GIC_PRIO_IRQOFF (GIC_PRIO_IRQON & ~0x80) #define __GIC_PRIO_IRQOFF_NS 0xa0 #define GIC_PRIO_PSR_I_SET (1 << 4) #define GIC_PRIO_IRQOFF \ ({ \ extern struct static_key_false gic_nonsecure_priorities;\ u8 __prio = __GIC_PRIO_IRQOFF; \ \ if (static_branch_unlikely(&gic_nonsecure_priorities)) \ __prio = __GIC_PRIO_IRQOFF_NS; \ \ __prio; \ }) /* Additional SPSR bits not exposed in the UABI */ #define PSR_MODE_THREAD_BIT (1 << 0) #define PSR_IL_BIT (1 << 20) Loading arch/arm64/kernel/image-vars.h +2 −0 Original line number Diff line number Diff line Loading @@ -101,6 +101,8 @@ KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); /* Static key checked in pmr_sync(). */ #ifdef CONFIG_ARM64_PSEUDO_NMI KVM_NVHE_ALIAS(gic_pmr_sync); /* Static key checked in GIC_PRIO_IRQOFF. */ KVM_NVHE_ALIAS(gic_nonsecure_priorities); #endif /* EL2 exception handling */ Loading drivers/irqchip/irq-gic-v3.c +46 −14 Original line number Diff line number Diff line Loading @@ -75,16 +75,14 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); * * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure * EL1 are subject to a similar operation thus matching the priorities presented * from the (re)distributor when security is enabled. * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, * these values are unchanched by the GIC. * * see GICv3/GICv4 Architecture Specification (IHI0069D): * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt * priorities. * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 * interrupt. * * For now, we only support pseudo-NMIs if we have non-secure view of * priorities. */ static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); Loading @@ -97,6 +95,9 @@ static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); DEFINE_STATIC_KEY_FALSE(gic_pmr_sync); EXPORT_SYMBOL(gic_pmr_sync); DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); EXPORT_SYMBOL(gic_nonsecure_priorities); /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ static refcount_t *ppi_nmi_refs; Loading Loading @@ -932,14 +933,20 @@ static void gic_cpu_sys_reg_init(void) /* Set priority mask register */ if (!gic_prio_masking_enabled()) { write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); } else { } else if (gic_supports_nmi()) { /* * Mismatch configuration with boot CPU, the system is likely * to die as interrupt masking will not work properly on all * CPUs * * The boot CPU calls this function before enabling NMI support, * and as a result we'll never see this warning in the boot path * for that CPU. */ WARN_ON(gic_supports_nmi() && group0 && !gic_dist_security_disabled()); if (static_branch_unlikely(&gic_nonsecure_priorities)) WARN_ON(!group0 || gic_dist_security_disabled()); else WARN_ON(group0 && !gic_dist_security_disabled()); } /* Loading Loading @@ -1544,11 +1551,6 @@ static void gic_enable_nmi_support(void) if (!gic_prio_masking_enabled()) return; if (gic_has_group0() && !gic_dist_security_disabled()) { pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n"); return; } ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); if (!ppi_nmi_refs) return; Loading @@ -1564,8 +1566,38 @@ static void gic_enable_nmi_support(void) if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) static_branch_enable(&gic_pmr_sync); pr_info("%s ICC_PMR_EL1 synchronisation\n", static_branch_unlikely(&gic_pmr_sync) ? "Forcing" : "Relaxing"); pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed"); /* * How priority values are used by the GIC depends on two things: * the security state of the GIC (controlled by the GICD_CTRL.DS bit) * and if Group 0 interrupts can be delivered to Linux in the non-secure * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the * the ICC_PMR_EL1 register and the priority that software assigns to * interrupts: * * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority * ----------------------------------------------------------- * 1 | - | unchanged | unchanged * ----------------------------------------------------------- * 0 | 1 | non-secure | non-secure * ----------------------------------------------------------- * 0 | 0 | unchanged | non-secure * * where non-secure means that the value is right-shifted by one and the * MSB bit set, to make it fit in the non-secure priority range. * * In the first two cases, where ICC_PMR_EL1 and the interrupt priority * are both either modified or unchanged, we can use the same set of * priorities. * * In the last case, where only the interrupt priorities are modified to * be in the non-secure range, we use a different PMR value to mask IRQs * and the rest of the values that we use remain unchanged. */ if (gic_has_group0() && !gic_dist_security_disabled()) static_branch_enable(&gic_nonsecure_priorities); static_branch_enable(&supports_pseudo_nmis); Loading Loading
arch/arm64/include/asm/arch_gicv3.h +7 −1 Original line number Diff line number Diff line Loading @@ -153,7 +153,7 @@ static inline bool gic_prio_masking_enabled(void) static inline void gic_pmr_mask_irqs(void) { BUILD_BUG_ON(GICD_INT_DEF_PRI < (GIC_PRIO_IRQOFF | BUILD_BUG_ON(GICD_INT_DEF_PRI < (__GIC_PRIO_IRQOFF | GIC_PRIO_PSR_I_SET)); BUILD_BUG_ON(GICD_INT_DEF_PRI >= GIC_PRIO_IRQON); /* Loading @@ -162,6 +162,12 @@ static inline void gic_pmr_mask_irqs(void) * are applied to IRQ priorities */ BUILD_BUG_ON((0x80 | (GICD_INT_DEF_PRI >> 1)) >= GIC_PRIO_IRQON); /* * Same situation as above, but now we make sure that we can mask * regular interrupts. */ BUILD_BUG_ON((0x80 | (GICD_INT_DEF_PRI >> 1)) < (__GIC_PRIO_IRQOFF_NS | GIC_PRIO_PSR_I_SET)); gic_write_pmr(GIC_PRIO_IRQOFF); } Loading
arch/arm64/include/asm/ptrace.h +13 −1 Original line number Diff line number Diff line Loading @@ -31,9 +31,21 @@ * interrupt disabling temporarily does not rely on IRQ priorities. */ #define GIC_PRIO_IRQON 0xe0 #define GIC_PRIO_IRQOFF (GIC_PRIO_IRQON & ~0x80) #define __GIC_PRIO_IRQOFF (GIC_PRIO_IRQON & ~0x80) #define __GIC_PRIO_IRQOFF_NS 0xa0 #define GIC_PRIO_PSR_I_SET (1 << 4) #define GIC_PRIO_IRQOFF \ ({ \ extern struct static_key_false gic_nonsecure_priorities;\ u8 __prio = __GIC_PRIO_IRQOFF; \ \ if (static_branch_unlikely(&gic_nonsecure_priorities)) \ __prio = __GIC_PRIO_IRQOFF_NS; \ \ __prio; \ }) /* Additional SPSR bits not exposed in the UABI */ #define PSR_MODE_THREAD_BIT (1 << 0) #define PSR_IL_BIT (1 << 20) Loading
arch/arm64/kernel/image-vars.h +2 −0 Original line number Diff line number Diff line Loading @@ -101,6 +101,8 @@ KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); /* Static key checked in pmr_sync(). */ #ifdef CONFIG_ARM64_PSEUDO_NMI KVM_NVHE_ALIAS(gic_pmr_sync); /* Static key checked in GIC_PRIO_IRQOFF. */ KVM_NVHE_ALIAS(gic_nonsecure_priorities); #endif /* EL2 exception handling */ Loading
drivers/irqchip/irq-gic-v3.c +46 −14 Original line number Diff line number Diff line Loading @@ -75,16 +75,14 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); * * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure * EL1 are subject to a similar operation thus matching the priorities presented * from the (re)distributor when security is enabled. * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, * these values are unchanched by the GIC. * * see GICv3/GICv4 Architecture Specification (IHI0069D): * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt * priorities. * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 * interrupt. * * For now, we only support pseudo-NMIs if we have non-secure view of * priorities. */ static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); Loading @@ -97,6 +95,9 @@ static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); DEFINE_STATIC_KEY_FALSE(gic_pmr_sync); EXPORT_SYMBOL(gic_pmr_sync); DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); EXPORT_SYMBOL(gic_nonsecure_priorities); /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ static refcount_t *ppi_nmi_refs; Loading Loading @@ -932,14 +933,20 @@ static void gic_cpu_sys_reg_init(void) /* Set priority mask register */ if (!gic_prio_masking_enabled()) { write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); } else { } else if (gic_supports_nmi()) { /* * Mismatch configuration with boot CPU, the system is likely * to die as interrupt masking will not work properly on all * CPUs * * The boot CPU calls this function before enabling NMI support, * and as a result we'll never see this warning in the boot path * for that CPU. */ WARN_ON(gic_supports_nmi() && group0 && !gic_dist_security_disabled()); if (static_branch_unlikely(&gic_nonsecure_priorities)) WARN_ON(!group0 || gic_dist_security_disabled()); else WARN_ON(group0 && !gic_dist_security_disabled()); } /* Loading Loading @@ -1544,11 +1551,6 @@ static void gic_enable_nmi_support(void) if (!gic_prio_masking_enabled()) return; if (gic_has_group0() && !gic_dist_security_disabled()) { pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n"); return; } ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); if (!ppi_nmi_refs) return; Loading @@ -1564,8 +1566,38 @@ static void gic_enable_nmi_support(void) if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) static_branch_enable(&gic_pmr_sync); pr_info("%s ICC_PMR_EL1 synchronisation\n", static_branch_unlikely(&gic_pmr_sync) ? "Forcing" : "Relaxing"); pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed"); /* * How priority values are used by the GIC depends on two things: * the security state of the GIC (controlled by the GICD_CTRL.DS bit) * and if Group 0 interrupts can be delivered to Linux in the non-secure * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the * the ICC_PMR_EL1 register and the priority that software assigns to * interrupts: * * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority * ----------------------------------------------------------- * 1 | - | unchanged | unchanged * ----------------------------------------------------------- * 0 | 1 | non-secure | non-secure * ----------------------------------------------------------- * 0 | 0 | unchanged | non-secure * * where non-secure means that the value is right-shifted by one and the * MSB bit set, to make it fit in the non-secure priority range. * * In the first two cases, where ICC_PMR_EL1 and the interrupt priority * are both either modified or unchanged, we can use the same set of * priorities. * * In the last case, where only the interrupt priorities are modified to * be in the non-secure range, we use a different PMR value to mask IRQs * and the rest of the values that we use remain unchanged. */ if (gic_has_group0() && !gic_dist_security_disabled()) static_branch_enable(&gic_nonsecure_priorities); static_branch_enable(&supports_pseudo_nmis); Loading
Marc Zyngier <maz@kernel.org>Marc Zyngier <maz@kernel.org>