From 0f80cad3124f986d0e46c14d46b8da06d87a2bf4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Apr 2019 13:03:51 +0100 Subject: [PATCH 01/11] arm64: Restrict ARM64_ERRATUM_1188873 mitigation to AArch32 We currently deal with ARM64_ERRATUM_1188873 by always trapping EL0 accesses for both instruction sets. Although nothing wrong comes out of that, people trying to squeeze the last drop of performance from buggy HW find this over the top. Oh well. Let's change the mitigation by flipping the counter enable bit on return to userspace. Non-broken HW gets an extra branch on the fast path, which is hopefully not the end of the world. The arch timer workaround is also removed. Acked-by: Daniel Lezcano Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/kernel/entry.S | 19 +++++++++++++++++-- drivers/clocksource/arm_arch_timer.c | 15 --------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index c50a7a75f2e0f7..1a7811b7e3c461 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -336,6 +336,21 @@ alternative_if ARM64_WORKAROUND_845719 alternative_else_nop_endif #endif 3: +#ifdef CONFIG_ARM64_ERRATUM_1188873 +alternative_if_not ARM64_WORKAROUND_1188873 + b 4f +alternative_else_nop_endif + /* + * if (x22.mode32 == cntkctl_el1.el0vcten) + * cntkctl_el1.el0vcten = ~cntkctl_el1.el0vcten + */ + mrs x1, cntkctl_el1 + eon x0, x1, x22, lsr #3 + tbz x0, #1, 4f + eor x1, x1, #2 // ARCH_TIMER_USR_VCT_ACCESS_EN + msr cntkctl_el1, x1 +4: +#endif apply_ssbd 0, x0, x1 .endif @@ -362,11 +377,11 @@ alternative_else_nop_endif .if \el == 0 alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - bne 4f + bne 5f msr far_el1, x30 tramp_alias x30, tramp_exit_native br x30 -4: +5: tramp_alias x30, tramp_exit_compat br x30 #endif diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index aa4ec53281cea5..da11a9508b7783 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -319,13 +319,6 @@ static u64 notrace arm64_858921_read_cntvct_el0(void) } #endif -#ifdef CONFIG_ARM64_ERRATUM_1188873 -static u64 notrace arm64_1188873_read_cntvct_el0(void) -{ - return read_sysreg(cntvct_el0); -} -#endif - #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1 /* * The low bits of the counter registers are indeterminate while bit 10 or @@ -457,14 +450,6 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = { .read_cntvct_el0 = arm64_858921_read_cntvct_el0, }, #endif -#ifdef CONFIG_ARM64_ERRATUM_1188873 - { - .match_type = ate_match_local_cap_id, - .id = (void *)ARM64_WORKAROUND_1188873, - .desc = "ARM erratum 1188873", - .read_cntvct_el0 = arm64_1188873_read_cntvct_el0, - }, -#endif #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1 { .match_type = ate_match_dt, From c2b5bba3967a000764e9148e6f020d776b7ecd82 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Apr 2019 13:03:52 +0100 Subject: [PATCH 02/11] arm64: Make ARM64_ERRATUM_1188873 depend on COMPAT Since ARM64_ERRATUM_1188873 only affects AArch32 EL0, it makes some sense that it should depend on COMPAT. Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 7e34b9eba5de15..560f2a860637b3 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -477,6 +477,7 @@ config ARM64_ERRATUM_1024718 config ARM64_ERRATUM_1188873 bool "Cortex-A76: MRC read following MRRC read of specific Generic Timer in AArch32 might give incorrect result" default y + depends on COMPAT select ARM_ARCH_TIMER_OOL_WORKAROUND help This option adds work arounds for ARM Cortex-A76 erratum 1188873 From 0cf57b86859c49381addb3ce47be70aadf5fd2c0 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Apr 2019 13:03:53 +0100 Subject: [PATCH 03/11] arm64: Add part number for Neoverse N1 New CPU, new part number. You know the drill. Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 5f1437099b9979..2602bae334fb70 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -89,6 +89,7 @@ #define ARM_CPU_PART_CORTEX_A35 0xD04 #define ARM_CPU_PART_CORTEX_A55 0xD05 #define ARM_CPU_PART_CORTEX_A76 0xD0B +#define ARM_CPU_PART_NEOVERSE_N1 0xD0C #define APM_CPU_PART_POTENZA 0x000 @@ -118,6 +119,7 @@ #define MIDR_CORTEX_A35 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35) #define MIDR_CORTEX_A55 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55) #define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) +#define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) From 6989303a3b2d864fd8e17d3fa3365d3e9649a598 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 15 Apr 2019 13:03:54 +0100 Subject: [PATCH 04/11] arm64: Apply ARM64_ERRATUM_1188873 to Neoverse-N1 Neoverse-N1 is also affected by ARM64_ERRATUM_1188873, so let's add it to the list of affected CPUs. Signed-off-by: Marc Zyngier [will: Update silicon-errata.txt] Signed-off-by: Will Deacon --- Documentation/arm64/silicon-errata.txt | 1 + arch/arm64/Kconfig | 11 ++++++----- arch/arm64/kernel/cpu_errata.c | 13 +++++++++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt index d1e2bb801e1bdb..d5a124d7e24203 100644 --- a/Documentation/arm64/silicon-errata.txt +++ b/Documentation/arm64/silicon-errata.txt @@ -61,6 +61,7 @@ stable kernels. | ARM | Cortex-A76 | #1188873 | ARM64_ERRATUM_1188873 | | ARM | Cortex-A76 | #1165522 | ARM64_ERRATUM_1165522 | | ARM | Cortex-A76 | #1286807 | ARM64_ERRATUM_1286807 | +| ARM | Neoverse-N1 | #1188873 | ARM64_ERRATUM_1188873 | | ARM | MMU-500 | #841119,#826419 | N/A | | | | | | | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 | diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 560f2a860637b3..fcda4e21fa8fea 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -475,16 +475,17 @@ config ARM64_ERRATUM_1024718 If unsure, say Y. config ARM64_ERRATUM_1188873 - bool "Cortex-A76: MRC read following MRRC read of specific Generic Timer in AArch32 might give incorrect result" + bool "Cortex-A76/Neoverse-N1: MRC read following MRRC read of specific Generic Timer in AArch32 might give incorrect result" default y depends on COMPAT select ARM_ARCH_TIMER_OOL_WORKAROUND help - This option adds work arounds for ARM Cortex-A76 erratum 1188873 + This option adds work arounds for ARM Cortex-A76/Neoverse-N1 + erratum 1188873 - Affected Cortex-A76 cores (r0p0, r1p0, r2p0) could cause - register corruption when accessing the timer registers from - AArch32 userspace. + Affected Cortex-A76/Neoverse-N1 cores (r0p0, r1p0, r2p0) could + cause register corruption when accessing the timer registers + from AArch32 userspace. If unsure, say Y. diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 9950bb0cbd5216..06f1c8aae1dcbf 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -603,6 +603,16 @@ static const struct midr_range workaround_clean_cache[] = { }; #endif +#ifdef CONFIG_ARM64_ERRATUM_1188873 +static const struct midr_range erratum_1188873_list[] = { + /* Cortex-A76 r0p0 to r2p0 */ + MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0), + /* Neoverse-N1 r0p0 to r2p0 */ + MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 0, 2, 0), + {}, +}; +#endif + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -725,10 +735,9 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_ARM64_ERRATUM_1188873 { - /* Cortex-A76 r0p0 to r2p0 */ .desc = "ARM erratum 1188873", .capability = ARM64_WORKAROUND_1188873, - ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0), + ERRATA_MIDR_RANGE_LIST(erratum_1188873_list), }, #endif #ifdef CONFIG_ARM64_ERRATUM_1165522 From 1f5b62f09f6b314c8d70b9de5182dae4de1f94da Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 8 Apr 2019 16:49:01 +0100 Subject: [PATCH 05/11] ARM: vdso: Remove dependency with the arch_timer driver internals The VDSO code uses the kernel helper that was originally designed to abstract the access between 32 and 64bit systems. It worked so far because this function is declared as 'inline'. As we're about to revamp that part of the code, the VDSO would break. Let's fix it by doing what should have been done from the start, a proper system register access. Reviewed-by: Mark Rutland Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm/include/asm/cp15.h | 2 ++ arch/arm/vdso/vgettimeofday.c | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/cp15.h b/arch/arm/include/asm/cp15.h index 07e27f212dc754..d2453e2d3f1f38 100644 --- a/arch/arm/include/asm/cp15.h +++ b/arch/arm/include/asm/cp15.h @@ -68,6 +68,8 @@ #define BPIALL __ACCESS_CP15(c7, 0, c5, 6) #define ICIALLU __ACCESS_CP15(c7, 0, c5, 0) +#define CNTVCT __ACCESS_CP15_64(1, c14) + extern unsigned long cr_alignment; /* defined in entry-armv.S */ static inline unsigned long get_cr(void) diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c index a9dd619c6c2900..7bdbf5d5c47d3c 100644 --- a/arch/arm/vdso/vgettimeofday.c +++ b/arch/arm/vdso/vgettimeofday.c @@ -18,9 +18,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -123,7 +123,8 @@ static notrace u64 get_ns(struct vdso_data *vdata) u64 cycle_now; u64 nsec; - cycle_now = arch_counter_get_cntvct(); + isb(); + cycle_now = read_sysreg(CNTVCT); cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask; From eae1ddc615bef9e1a1958a4b867a0a1678049bb3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 8 Apr 2019 16:49:02 +0100 Subject: [PATCH 06/11] watchdog/sbsa: Use arch_timer_read_counter instead of arch_counter_get_cntvct Only arch_timer_read_counter will guarantee that workarounds are applied. So let's use this one instead of arch_counter_get_cntvct. Acked-by: Mark Rutland Reviewed-by: Guenter Roeck Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- drivers/watchdog/sbsa_gwdt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/watchdog/sbsa_gwdt.c b/drivers/watchdog/sbsa_gwdt.c index e8bd9887c56638..e221e47396ab72 100644 --- a/drivers/watchdog/sbsa_gwdt.c +++ b/drivers/watchdog/sbsa_gwdt.c @@ -161,7 +161,7 @@ static unsigned int sbsa_gwdt_get_timeleft(struct watchdog_device *wdd) timeleft += readl(gwdt->control_base + SBSA_GWDT_WOR); timeleft += lo_hi_readq(gwdt->control_base + SBSA_GWDT_WCV) - - arch_counter_get_cntvct(); + arch_timer_read_counter(); do_div(timeleft, gwdt->clk); From dea86a80033f8b0fb25a805f46dde9f3b1a7c23a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 8 Apr 2019 16:49:03 +0100 Subject: [PATCH 07/11] arm64: Use arch_timer_read_counter instead of arch_counter_get_cntvct Only arch_timer_read_counter will guarantee that workarounds are applied. So let's use this one instead of arch_counter_get_cntvct. Acked-by: Mark Rutland Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/kernel/traps.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 8ad119c3f665d4..6190a60388cf3b 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -493,7 +493,7 @@ static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) { int rt = ESR_ELx_SYS64_ISS_RT(esr); - pt_regs_write_reg(regs, rt, arch_counter_get_cntvct()); + pt_regs_write_reg(regs, rt, arch_timer_read_counter()); arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); } @@ -665,7 +665,7 @@ static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs) { int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT; int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT; - u64 val = arch_counter_get_cntvct(); + u64 val = arch_timer_read_counter(); pt_regs_write_reg(regs, rt, lower_32_bits(val)); pt_regs_write_reg(regs, rt2, upper_32_bits(val)); From 5ef19a161cfa88a59508979e2f39d3d092c1d5c0 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 8 Apr 2019 16:49:04 +0100 Subject: [PATCH 08/11] clocksource/arm_arch_timer: Direcly assign set_next_event workaround When a given timer is affected by an erratum and requires an alternative implementation of set_next_event, we do a rather complicated dance to detect and call the workaround on each set_next_event call. This is clearly idiotic, as we can perfectly detect whether this CPU requires a workaround while setting up the clock event device. This only requires the CPU-specific detection to be done a bit earlier, and we can then safely override the set_next_event pointer if we have a workaround associated to that CPU. Acked-by: Mark Rutland Acked-by; Daniel Lezcano Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm/include/asm/arch_timer.h | 4 +++ arch/arm64/include/asm/arch_timer.h | 16 ++++++++++ drivers/clocksource/arm_arch_timer.c | 46 +++++----------------------- 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/arch/arm/include/asm/arch_timer.h b/arch/arm/include/asm/arch_timer.h index 0a8d7bba2cb012..3f0a0191f763d4 100644 --- a/arch/arm/include/asm/arch_timer.h +++ b/arch/arm/include/asm/arch_timer.h @@ -11,6 +11,10 @@ #include #ifdef CONFIG_ARM_ARCH_TIMER +/* 32bit ARM doesn't know anything about timer errata... */ +#define has_erratum_handler(h) (false) +#define erratum_handler(h) (arch_timer_##h) + int arch_timer_arch_init(void); /* diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index f2a234d6516cf5..c3762ffcc93307 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -31,10 +31,26 @@ #include #if IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND) +#define has_erratum_handler(h) \ + ({ \ + const struct arch_timer_erratum_workaround *__wa; \ + __wa = __this_cpu_read(timer_unstable_counter_workaround); \ + (__wa && __wa->h); \ + }) + +#define erratum_handler(h) \ + ({ \ + const struct arch_timer_erratum_workaround *__wa; \ + __wa = __this_cpu_read(timer_unstable_counter_workaround); \ + (__wa && __wa->h) ? __wa->h : arch_timer_##h; \ + }) + extern struct static_key_false arch_timer_read_ool_enabled; #define needs_unstable_timer_counter_workaround() \ static_branch_unlikely(&arch_timer_read_ool_enabled) #else +#define has_erratum_handler(h) false +#define erratum_handler(h) (arch_timer_##h) #define needs_unstable_timer_counter_workaround() false #endif diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index da11a9508b7783..b2a88a64aab492 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -598,36 +598,12 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t local ? "local" : "global", wa->desc); } -#define erratum_handler(fn, r, ...) \ -({ \ - bool __val; \ - if (needs_unstable_timer_counter_workaround()) { \ - const struct arch_timer_erratum_workaround *__wa; \ - __wa = __this_cpu_read(timer_unstable_counter_workaround); \ - if (__wa && __wa->fn) { \ - r = __wa->fn(__VA_ARGS__); \ - __val = true; \ - } else { \ - __val = false; \ - } \ - } else { \ - __val = false; \ - } \ - __val; \ -}) - static bool arch_timer_this_cpu_has_cntvct_wa(void) { - const struct arch_timer_erratum_workaround *wa; - - wa = __this_cpu_read(timer_unstable_counter_workaround); - return wa && wa->read_cntvct_el0; + return has_erratum_handler(read_cntvct_el0); } #else #define arch_timer_check_ool_workaround(t,a) do { } while(0) -#define erratum_set_next_event_tval_virt(...) ({BUG(); 0;}) -#define erratum_set_next_event_tval_phys(...) ({BUG(); 0;}) -#define erratum_handler(fn, r, ...) ({false;}) #define arch_timer_this_cpu_has_cntvct_wa() ({false;}) #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ @@ -721,11 +697,6 @@ static __always_inline void set_next_event(const int access, unsigned long evt, static int arch_timer_set_next_event_virt(unsigned long evt, struct clock_event_device *clk) { - int ret; - - if (erratum_handler(set_next_event_virt, ret, evt, clk)) - return ret; - set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0; } @@ -733,11 +704,6 @@ static int arch_timer_set_next_event_virt(unsigned long evt, static int arch_timer_set_next_event_phys(unsigned long evt, struct clock_event_device *clk) { - int ret; - - if (erratum_handler(set_next_event_phys, ret, evt, clk)) - return ret; - set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0; } @@ -762,6 +728,10 @@ static void __arch_timer_setup(unsigned type, clk->features = CLOCK_EVT_FEAT_ONESHOT; if (type == ARCH_TIMER_TYPE_CP15) { + typeof(clk->set_next_event) sne; + + arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL); + if (arch_timer_c3stop) clk->features |= CLOCK_EVT_FEAT_C3STOP; clk->name = "arch_sys_timer"; @@ -772,20 +742,20 @@ static void __arch_timer_setup(unsigned type, case ARCH_TIMER_VIRT_PPI: clk->set_state_shutdown = arch_timer_shutdown_virt; clk->set_state_oneshot_stopped = arch_timer_shutdown_virt; - clk->set_next_event = arch_timer_set_next_event_virt; + sne = erratum_handler(set_next_event_virt); break; case ARCH_TIMER_PHYS_SECURE_PPI: case ARCH_TIMER_PHYS_NONSECURE_PPI: case ARCH_TIMER_HYP_PPI: clk->set_state_shutdown = arch_timer_shutdown_phys; clk->set_state_oneshot_stopped = arch_timer_shutdown_phys; - clk->set_next_event = arch_timer_set_next_event_phys; + sne = erratum_handler(set_next_event_phys); break; default: BUG(); } - arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL); + clk->set_next_event = sne; } else { clk->features |= CLOCK_EVT_FEAT_DYNIRQ; clk->name = "arch_mem_timer"; From 57f27666f91a85431492b092f5db53ecab1a0739 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 8 Apr 2019 16:49:05 +0100 Subject: [PATCH 09/11] clocksource/arm_arch_timer: Drop use of static key in arch_timer_reg_read_stable Let's start with the removal of the arch_timer_read_ool_enabled static key in arch_timer_reg_read_stable. It is not a fast path, and we can simplify things a bit. Acked-by: Mark Rutland Acked-by: Daniel Lezcano Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/include/asm/arch_timer.h | 42 +++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index c3762ffcc93307..4a06d46def7ed1 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -77,23 +77,37 @@ struct arch_timer_erratum_workaround { DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround); +/* inline sysreg accessors that make erratum_handler() work */ +static inline notrace u32 arch_timer_read_cntp_tval_el0(void) +{ + return read_sysreg(cntp_tval_el0); +} + +static inline notrace u32 arch_timer_read_cntv_tval_el0(void) +{ + return read_sysreg(cntv_tval_el0); +} + +static inline notrace u64 arch_timer_read_cntpct_el0(void) +{ + return read_sysreg(cntpct_el0); +} + +static inline notrace u64 arch_timer_read_cntvct_el0(void) +{ + return read_sysreg(cntvct_el0); +} + #define arch_timer_reg_read_stable(reg) \ -({ \ - u64 _val; \ - if (needs_unstable_timer_counter_workaround()) { \ - const struct arch_timer_erratum_workaround *wa; \ + ({ \ + u64 _val; \ + \ preempt_disable_notrace(); \ - wa = __this_cpu_read(timer_unstable_counter_workaround); \ - if (wa && wa->read_##reg) \ - _val = wa->read_##reg(); \ - else \ - _val = read_sysreg(reg); \ + _val = erratum_handler(read_ ## reg)(); \ preempt_enable_notrace(); \ - } else { \ - _val = read_sysreg(reg); \ - } \ - _val; \ -}) + \ + _val; \ + }) /* * These register accessors are marked inline so the compiler can From a862fc2254bdbcee3b5da4f730984e5d8393a2f1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 8 Apr 2019 16:49:06 +0100 Subject: [PATCH 10/11] clocksource/arm_arch_timer: Remove use of workaround static key The use of a static key in a hotplug path has proved to be a real nightmare, and makes it impossible to have scream-free lockdep kernel. Let's remove the static key altogether, and focus on something saner. Acked-by: Mark Rutland Acked-by: Daniel Lezcano Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/include/asm/arch_timer.h | 4 ---- drivers/clocksource/arm_arch_timer.c | 25 +++++++------------------ 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index 4a06d46def7ed1..5502ea049b63c0 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -45,13 +45,9 @@ (__wa && __wa->h) ? __wa->h : arch_timer_##h; \ }) -extern struct static_key_false arch_timer_read_ool_enabled; -#define needs_unstable_timer_counter_workaround() \ - static_branch_unlikely(&arch_timer_read_ool_enabled) #else #define has_erratum_handler(h) false #define erratum_handler(h) (arch_timer_##h) -#define needs_unstable_timer_counter_workaround() false #endif enum arch_timer_erratum_match_type { diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index b2a88a64aab492..8f22976247c005 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -365,8 +365,6 @@ static u32 notrace sun50i_a64_read_cntv_tval_el0(void) DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround); EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); -DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); -EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); static void erratum_set_next_event_tval_generic(const int access, unsigned long evt, struct clock_event_device *clk) @@ -537,12 +535,6 @@ void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa per_cpu(timer_unstable_counter_workaround, i) = wa; } - /* - * Use the locked version, as we're called from the CPU - * hotplug framework. Otherwise, we end-up in deadlock-land. - */ - static_branch_enable_cpuslocked(&arch_timer_read_ool_enabled); - /* * Don't use the vdso fastpath if errata require using the * out-of-line counter accessor. We may change our mind pretty @@ -558,7 +550,7 @@ void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type, void *arg) { - const struct arch_timer_erratum_workaround *wa; + const struct arch_timer_erratum_workaround *wa, *__wa; ate_match_fn_t match_fn = NULL; bool local = false; @@ -582,16 +574,13 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t if (!wa) return; - if (needs_unstable_timer_counter_workaround()) { - const struct arch_timer_erratum_workaround *__wa; - __wa = __this_cpu_read(timer_unstable_counter_workaround); - if (__wa && wa != __wa) - pr_warn("Can't enable workaround for %s (clashes with %s\n)", - wa->desc, __wa->desc); + __wa = __this_cpu_read(timer_unstable_counter_workaround); + if (__wa && wa != __wa) + pr_warn("Can't enable workaround for %s (clashes with %s\n)", + wa->desc, __wa->desc); - if (__wa) - return; - } + if (__wa) + return; arch_timer_enable_workaround(wa, local); pr_info("Enabling %s workaround for %s\n", From 0ea415390cd345b7d09e8c9ebd4b68adfe873043 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 8 Apr 2019 16:49:07 +0100 Subject: [PATCH 11/11] clocksource/arm_arch_timer: Use arch_timer_read_counter to access stable counters Instead of always going via arch_counter_get_cntvct_stable to access the counter workaround, let's have arch_timer_read_counter point to the right method. For that, we need to track whether any CPU in the system has a workaround for the counter. This is done by having an atomic variable tracking this. Acked-by: Mark Rutland Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm/include/asm/arch_timer.h | 14 ++++++-- arch/arm64/include/asm/arch_timer.h | 16 ++++++++-- drivers/clocksource/arm_arch_timer.c | 48 +++++++++++++++++++++++++--- 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/arch/arm/include/asm/arch_timer.h b/arch/arm/include/asm/arch_timer.h index 3f0a0191f763d4..4b66ecd6be99df 100644 --- a/arch/arm/include/asm/arch_timer.h +++ b/arch/arm/include/asm/arch_timer.h @@ -83,7 +83,7 @@ static inline u32 arch_timer_get_cntfrq(void) return val; } -static inline u64 arch_counter_get_cntpct(void) +static inline u64 __arch_counter_get_cntpct(void) { u64 cval; @@ -92,7 +92,12 @@ static inline u64 arch_counter_get_cntpct(void) return cval; } -static inline u64 arch_counter_get_cntvct(void) +static inline u64 __arch_counter_get_cntpct_stable(void) +{ + return __arch_counter_get_cntpct(); +} + +static inline u64 __arch_counter_get_cntvct(void) { u64 cval; @@ -101,6 +106,11 @@ static inline u64 arch_counter_get_cntvct(void) return cval; } +static inline u64 __arch_counter_get_cntvct_stable(void) +{ + return __arch_counter_get_cntvct(); +} + static inline u32 arch_timer_get_cntkctl(void) { u32 cntkctl; diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index 5502ea049b63c0..48b2100f4aaad0 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -174,18 +174,30 @@ static inline void arch_timer_set_cntkctl(u32 cntkctl) isb(); } -static inline u64 arch_counter_get_cntpct(void) +static inline u64 __arch_counter_get_cntpct_stable(void) { isb(); return arch_timer_reg_read_stable(cntpct_el0); } -static inline u64 arch_counter_get_cntvct(void) +static inline u64 __arch_counter_get_cntpct(void) +{ + isb(); + return read_sysreg(cntpct_el0); +} + +static inline u64 __arch_counter_get_cntvct_stable(void) { isb(); return arch_timer_reg_read_stable(cntvct_el0); } +static inline u64 __arch_counter_get_cntvct(void) +{ + isb(); + return read_sysreg(cntvct_el0); +} + static inline int arch_timer_arch_init(void) { return 0; diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 8f22976247c005..27acc9eb0f7c49 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -152,6 +152,26 @@ u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, return val; } +static u64 arch_counter_get_cntpct_stable(void) +{ + return __arch_counter_get_cntpct_stable(); +} + +static u64 arch_counter_get_cntpct(void) +{ + return __arch_counter_get_cntpct(); +} + +static u64 arch_counter_get_cntvct_stable(void) +{ + return __arch_counter_get_cntvct_stable(); +} + +static u64 arch_counter_get_cntvct(void) +{ + return __arch_counter_get_cntvct(); +} + /* * Default to cp15 based access because arm64 uses this function for * sched_clock() before DT is probed and the cp15 method is guaranteed @@ -365,6 +385,7 @@ static u32 notrace sun50i_a64_read_cntv_tval_el0(void) DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround); EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); +static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0); static void erratum_set_next_event_tval_generic(const int access, unsigned long evt, struct clock_event_device *clk) @@ -535,6 +556,9 @@ void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa per_cpu(timer_unstable_counter_workaround, i) = wa; } + if (wa->read_cntvct_el0 || wa->read_cntpct_el0) + atomic_set(&timer_unstable_counter_workaround_in_use, 1); + /* * Don't use the vdso fastpath if errata require using the * out-of-line counter accessor. We may change our mind pretty @@ -591,9 +615,15 @@ static bool arch_timer_this_cpu_has_cntvct_wa(void) { return has_erratum_handler(read_cntvct_el0); } + +static bool arch_timer_counter_has_wa(void) +{ + return atomic_read(&timer_unstable_counter_workaround_in_use); +} #else #define arch_timer_check_ool_workaround(t,a) do { } while(0) #define arch_timer_this_cpu_has_cntvct_wa() ({false;}) +#define arch_timer_counter_has_wa() ({false;}) #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ static __always_inline irqreturn_t timer_handler(const int access, @@ -942,12 +972,22 @@ static void __init arch_counter_register(unsigned type) /* Register the CP15 based counter if we have one */ if (type & ARCH_TIMER_TYPE_CP15) { + u64 (*rd)(void); + if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) || - arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) - arch_timer_read_counter = arch_counter_get_cntvct; - else - arch_timer_read_counter = arch_counter_get_cntpct; + arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) { + if (arch_timer_counter_has_wa()) + rd = arch_counter_get_cntvct_stable; + else + rd = arch_counter_get_cntvct; + } else { + if (arch_timer_counter_has_wa()) + rd = arch_counter_get_cntpct_stable; + else + rd = arch_counter_get_cntpct; + } + arch_timer_read_counter = rd; clocksource_counter.archdata.vdso_direct = vdso_default; } else { arch_timer_read_counter = arch_counter_get_cntvct_mem;